From f974d95a6e710634100f6c11d05b15fb4f2766f0 Mon Sep 17 00:00:00 2001 From: Dong Bo Date: Tue, 25 Oct 2022 20:36:52 +0800 Subject: [PATCH] 8295698: AArch64: test/jdk/sun/security/ec/ed/EdDSATest.java failed with -XX:+UseSHA3Intrinsics --- .../cpu/aarch64/stubGenerator_aarch64.cpp | 67 ++++++++++--------- src/hotspot/share/opto/library_call.cpp | 30 ++++----- src/hotspot/share/opto/library_call.hpp | 2 +- src/hotspot/share/opto/runtime.cpp | 4 +- 4 files changed, 54 insertions(+), 49 deletions(-) diff --git a/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp b/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp index 58f5d4752c5..89ad0bb070c 100644 --- a/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp @@ -3824,8 +3824,8 @@ class StubGenerator: public StubCodeGenerator { // // Inputs: // c_rarg0 - byte[] source+offset - // c_rarg1 - byte[] SHA.state - // c_rarg2 - int digest_length + // c_rarg1 - byte[] SHA.state + // c_rarg2 - int block_size // c_rarg3 - int offset // c_rarg4 - int limit // @@ -3847,12 +3847,12 @@ class StubGenerator: public StubCodeGenerator { Register buf = c_rarg0; Register state = c_rarg1; - Register digest_length = c_rarg2; + Register block_size = c_rarg2; Register ofs = c_rarg3; Register limit = c_rarg4; Label sha3_loop, rounds24_loop; - Label sha3_512, sha3_384_or_224, sha3_256; + Label sha3_512_or_sha3_384, shake128; __ stpd(v8, v9, __ pre(sp, -64)); __ stpd(v10, v11, Address(sp, 16)); @@ -3888,46 +3888,54 @@ class StubGenerator: public StubCodeGenerator { __ eor(v5, __ T8B, v5, v30); __ eor(v6, __ T8B, v6, v31); - // digest_length == 64, SHA3-512 - __ tbnz(digest_length, 6, sha3_512); + // block_size == 72, SHA3-512; block_size == 104, SHA3-384 + __ tbz(block_size, 7, sha3_512_or_sha3_384); __ ld1(v25, v26, v27, v28, __ T8B, __ post(buf, 32)); - __ ld1(v29, v30, __ T8B, __ post(buf, 16)); + __ ld1(v29, v30, v31, __ T8B, __ post(buf, 24)); __ eor(v7, __ T8B, v7, v25); __ eor(v8, __ T8B, v8, v26); __ eor(v9, __ T8B, v9, v27); __ eor(v10, __ T8B, v10, v28); __ eor(v11, __ T8B, v11, v29); __ eor(v12, __ T8B, v12, v30); + __ eor(v13, __ T8B, v13, v31); - // digest_length == 28, SHA3-224; digest_length == 48, SHA3-384 - __ tbnz(digest_length, 4, sha3_384_or_224); + __ ld1(v25, v26, v27, __ T8B, __ post(buf, 24)); + __ eor(v14, __ T8B, v14, v25); + __ eor(v15, __ T8B, v15, v26); + __ eor(v16, __ T8B, v16, v27); - // SHA3-256 - __ ld1(v25, v26, v27, v28, __ T8B, __ post(buf, 32)); - __ eor(v13, __ T8B, v13, v25); - __ eor(v14, __ T8B, v14, v26); - __ eor(v15, __ T8B, v15, v27); - __ eor(v16, __ T8B, v16, v28); - __ b(rounds24_loop); + // block_size == 136, bit4 == 0 and bit5 == 0, SHA3-256 or SHAKE256 + __ andw(c_rarg5, block_size, 48); + __ cbzw(c_rarg5, rounds24_loop); - __ BIND(sha3_384_or_224); - __ tbz(digest_length, 2, rounds24_loop); // bit 2 cleared? SHA-384 + __ tbnz(block_size, 5, shake128); + // block_size == 144, bit5 == 0, SHA3-244 + __ ldrd(v28, __ post(buf, 8)); + __ eor(v17, __ T8B, v17, v28); + __ b(rounds24_loop); - // SHA3-224 - __ ld1(v25, v26, v27, v28, __ T8B, __ post(buf, 32)); - __ ld1(v29, __ T8B, __ post(buf, 8)); - __ eor(v13, __ T8B, v13, v25); - __ eor(v14, __ T8B, v14, v26); - __ eor(v15, __ T8B, v15, v27); - __ eor(v16, __ T8B, v16, v28); - __ eor(v17, __ T8B, v17, v29); + __ BIND(shake128); + __ ld1(v28, v29, v30, v31, __ T8B, __ post(buf, 32)); + __ eor(v17, __ T8B, v17, v28); + __ eor(v18, __ T8B, v18, v29); + __ eor(v19, __ T8B, v19, v30); + __ eor(v20, __ T8B, v20, v31); __ b(rounds24_loop); - __ BIND(sha3_512); + __ BIND(sha3_512_or_sha3_384); __ ld1(v25, v26, __ T8B, __ post(buf, 16)); __ eor(v7, __ T8B, v7, v25); __ eor(v8, __ T8B, v8, v26); + __ tbz(block_size, 5, rounds24_loop); + + // SHA3-384 + __ ld1(v27, v28, v29, v30, __ T8B, __ post(buf, 32)); + __ eor(v9, __ T8B, v9, v27); + __ eor(v10, __ T8B, v10, v28); + __ eor(v11, __ T8B, v11, v29); + __ eor(v12, __ T8B, v12, v30); __ BIND(rounds24_loop); __ subw(rscratch2, rscratch2, 1); @@ -4012,10 +4020,7 @@ class StubGenerator: public StubCodeGenerator { __ cbnzw(rscratch2, rounds24_loop); if (multi_block) { - // block_size = 200 - 2 * digest_length, ofs += block_size - __ add(ofs, ofs, 200); - __ sub(ofs, ofs, digest_length, Assembler::LSL, 1); - + __ add(ofs, ofs, block_size); __ cmp(ofs, limit); __ br(Assembler::LE, sha3_loop); __ mov(c_rarg0, ofs); // return ofs diff --git a/src/hotspot/share/opto/library_call.cpp b/src/hotspot/share/opto/library_call.cpp index 3767eec8b4d..574ad5c9b8c 100644 --- a/src/hotspot/share/opto/library_call.cpp +++ b/src/hotspot/share/opto/library_call.cpp @@ -7006,7 +7006,7 @@ bool LibraryCallKit::inline_digestBase_implCompress(vmIntrinsics::ID id) { src = must_be_not_null(src, true); Node* src_start = array_element_address(src, ofs, src_elem); Node* state = NULL; - Node* digest_length = NULL; + Node* block_size = NULL; address stubAddr; const char *stubName; @@ -7040,8 +7040,8 @@ bool LibraryCallKit::inline_digestBase_implCompress(vmIntrinsics::ID id) { state = get_state_from_digest_object(digestBase_obj, T_BYTE); stubAddr = StubRoutines::sha3_implCompress(); stubName = "sha3_implCompress"; - digest_length = get_digest_length_from_digest_object(digestBase_obj); - if (digest_length == NULL) return false; + block_size = get_block_size_from_digest_object(digestBase_obj); + if (block_size == NULL) return false; break; default: fatal_unexpected_iid(id); @@ -7054,14 +7054,14 @@ bool LibraryCallKit::inline_digestBase_implCompress(vmIntrinsics::ID id) { // Call the stub. Node* call; - if (digest_length == NULL) { + if (block_size == NULL) { call = make_runtime_call(RC_LEAF|RC_NO_FP, OptoRuntime::digestBase_implCompress_Type(false), stubAddr, stubName, TypePtr::BOTTOM, src_start, state); } else { call = make_runtime_call(RC_LEAF|RC_NO_FP, OptoRuntime::digestBase_implCompress_Type(true), stubAddr, stubName, TypePtr::BOTTOM, - src_start, state, digest_length); + src_start, state, block_size); } return true; @@ -7173,15 +7173,15 @@ bool LibraryCallKit::inline_digestBase_implCompressMB(Node* digestBase_obj, ciIn Node* state = get_state_from_digest_object(digest_obj, elem_type); if (state == NULL) return false; - Node* digest_length = NULL; + Node* block_size = NULL; if (strcmp("sha3_implCompressMB", stubName) == 0) { - digest_length = get_digest_length_from_digest_object(digest_obj); - if (digest_length == NULL) return false; + block_size = get_block_size_from_digest_object(digest_obj); + if (block_size == NULL) return false; } // Call the stub. Node* call; - if (digest_length == NULL) { + if (block_size == NULL) { call = make_runtime_call(RC_LEAF|RC_NO_FP, OptoRuntime::digestBase_implCompressMB_Type(false), stubAddr, stubName, TypePtr::BOTTOM, @@ -7190,7 +7190,7 @@ bool LibraryCallKit::inline_digestBase_implCompressMB(Node* digestBase_obj, ciIn call = make_runtime_call(RC_LEAF|RC_NO_FP, OptoRuntime::digestBase_implCompressMB_Type(true), stubAddr, stubName, TypePtr::BOTTOM, - src_start, state, digest_length, ofs, limit); + src_start, state, block_size, ofs, limit); } // return ofs (int) @@ -7347,11 +7347,11 @@ Node * LibraryCallKit::get_state_from_digest_object(Node *digest_object, BasicTy return state; } -//------------------------------get_digest_length_from_sha3_object---------------------------------- -Node * LibraryCallKit::get_digest_length_from_digest_object(Node *digest_object) { - Node* digest_length = load_field_from_object(digest_object, "digestLength", "I"); - assert (digest_length != NULL, "sanity"); - return digest_length; +//------------------------------get_block_size_from_sha3_object---------------------------------- +Node * LibraryCallKit::get_block_size_from_digest_object(Node *digest_object) { + Node* block_size = load_field_from_object(digest_object, "blockSize", "I"); + assert (block_size != NULL, "sanity"); + return block_size; } //----------------------------inline_digestBase_implCompressMB_predicate---------------------------- diff --git a/src/hotspot/share/opto/library_call.hpp b/src/hotspot/share/opto/library_call.hpp index 07235497740..1f660f7d987 100644 --- a/src/hotspot/share/opto/library_call.hpp +++ b/src/hotspot/share/opto/library_call.hpp @@ -299,7 +299,7 @@ class LibraryCallKit : public GraphKit { BasicType elem_type, address stubAddr, const char *stubName, Node* src_start, Node* ofs, Node* limit); Node* get_state_from_digest_object(Node *digestBase_object, BasicType elem_type); - Node* get_digest_length_from_digest_object(Node *digestBase_object); + Node* get_block_size_from_digest_object(Node *digestBase_object); Node* inline_digestBase_implCompressMB_predicate(int predicate); bool inline_encodeISOArray(bool ascii); bool inline_updateCRC32(); diff --git a/src/hotspot/share/opto/runtime.cpp b/src/hotspot/share/opto/runtime.cpp index 68408e198c7..b5326838194 100644 --- a/src/hotspot/share/opto/runtime.cpp +++ b/src/hotspot/share/opto/runtime.cpp @@ -1018,7 +1018,7 @@ const TypeFunc* OptoRuntime::digestBase_implCompress_Type(bool is_sha3) { int argp = TypeFunc::Parms; fields[argp++] = TypePtr::NOTNULL; // buf fields[argp++] = TypePtr::NOTNULL; // state - if (is_sha3) fields[argp++] = TypeInt::INT; // digest_length + if (is_sha3) fields[argp++] = TypeInt::INT; // block_size assert(argp == TypeFunc::Parms+argcnt, "correct decoding"); const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms+argcnt, fields); @@ -1040,7 +1040,7 @@ const TypeFunc* OptoRuntime::digestBase_implCompressMB_Type(bool is_sha3) { int argp = TypeFunc::Parms; fields[argp++] = TypePtr::NOTNULL; // buf fields[argp++] = TypePtr::NOTNULL; // state - if (is_sha3) fields[argp++] = TypeInt::INT; // digest_length + if (is_sha3) fields[argp++] = TypeInt::INT; // block_size fields[argp++] = TypeInt::INT; // ofs fields[argp++] = TypeInt::INT; // limit assert(argp == TypeFunc::Parms+argcnt, "correct decoding"); -- 2.30.0