       1 /*
       2  * Copyright (c) 1997, 2013, Oracle and/or its affiliates. All rights reserved.
       3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
       4  *
       5  * This code is free software; you can redistribute it and/or modify it
       6  * under the terms of the GNU General Public License version 2 only, as
       7  * published by the Free Software Foundation.
       8  *
       9  * This code is distributed in the hope that it will be useful, but WITHOUT
      10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
      11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
      12  * version 2 for more details (a copy is included in the LICENSE file that
      13  * accompanied this code).
      14  *
      15  * You should have received a copy of the GNU General Public License version
      16  * 2 along with this work; if not, write to the Free Software Foundation,
      17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
      18  *
      19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
      20  * or visit www.oracle.com if you need additional information or have any
      21  * questions.
      22  *
      23  */
      24
      25 #ifndef CPU_X86_VM_MACROASSEMBLER_X86_HPP
      26 #define CPU_X86_VM_MACROASSEMBLER_X86_HPP
      27
      28 #include "asm/assembler.hpp"
      29 #include "utilities/macros.hpp"
      30 #include "runtime/rtmLocking.hpp"
      31
      32
      33 // MacroAssembler extends Assembler by frequently used macros.
      34 //
      35 // Instructions for which a 'better' code sequence exists depending
      36 // on arguments should also go in here.
      37
      38 class MacroAssembler: public Assembler {
      39   friend class LIR_Assembler;
      40   friend class Runtime1;      // as_Address()
      41
      42  protected:
      43
      44   Address as_Address(AddressLiteral adr);
      45   Address as_Address(ArrayAddress adr);
      46
      47   // Support for VM calls
      48   //
      49   // This is the base routine called by the different versions of call_VM_leaf. The interpreter
      50   // may customize this version by overriding it for its purposes (e.g., to save/restore
      51   // additional registers when doing a VM call).
      52 #ifdef CC_INTERP
      53   // c++ interpreter never wants to use interp_masm version of call_VM
      54   #define VIRTUAL
      55 #else
      56   #define VIRTUAL virtual
      57 #endif
      58
      59   VIRTUAL void call_VM_leaf_base(
      60     address entry_point,               // the entry point
      61     int     number_of_arguments        // the number of arguments to pop after the call
      62   );
      63
      64   // This is the base routine called by the different versions of call_VM. The interpreter
      65   // may customize this version by overriding it for its purposes (e.g., to save/restore
      66   // additional registers when doing a VM call).
      67   //
      68   // If no java_thread register is specified (noreg) than rdi will be used instead. call_VM_base
      69   // returns the register which contains the thread upon return. If a thread register has been
      70   // specified, the return value will correspond to that register. If no last_java_sp is specified
      71   // (noreg) than rsp will be used instead.
      72   VIRTUAL void call_VM_base(           // returns the register containing the thread upon return
      73     Register oop_result,               // where an oop-result ends up if any; use noreg otherwise
      74     Register java_thread,              // the thread if computed before     ; use noreg otherwise
      75     Register last_java_sp,             // to set up last_Java_frame in stubs; use noreg otherwise
      76     address  entry_point,              // the entry point
      77     int      number_of_arguments,      // the number of arguments (w/o thread) to pop after the call
      78     bool     check_exceptions          // whether to check for pending exceptions after return
      79   );
      80
      81   // These routines should emit JVMTI PopFrame and ForceEarlyReturn handling code.
      82   // The implementation is only non-empty for the InterpreterMacroAssembler,
      83   // as only the interpreter handles PopFrame and ForceEarlyReturn requests.
      84   virtual void check_and_handle_popframe(Register java_thread);
      85   virtual void check_and_handle_earlyret(Register java_thread);
      86
      87   void call_VM_helper(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions = true);
      88
      89   // helpers for FPU flag access
      90   // tmp is a temporary register, if none is available use noreg
      91   void save_rax   (Register tmp);
      92   void restore_rax(Register tmp);
      93
      94  public:
      95   MacroAssembler(CodeBuffer* code) : Assembler(code) {}
      96
      97   // Support for NULL-checks
      98   //
      99   // Generates code that causes a NULL OS exception if the content of reg is NULL.
     100   // If the accessed location is M[reg + offset] and the offset is known, provide the
     101   // offset. No explicit code generation is needed if the offset is within a certain
     102   // range (0 <= offset <= page_size).
     103
     104   void null_check(Register reg, int offset = -1);
     105   static bool needs_explicit_null_check(intptr_t offset);
     106
     107   // Required platform-specific helpers for Label::patch_instructions.
     108   // They _shadow_ the declarations in AbstractAssembler, which are undefined.
     109   void pd_patch_instruction(address branch, address target) {
     110     unsigned char op = branch[0];
     111     assert(op == 0xE8 /* call */ ||
     112         op == 0xE9 /* jmp */ ||
     113         op == 0xEB /* short jmp */ ||
     114         (op & 0xF0) == 0x70 /* short jcc */ ||
     115         op == 0x0F && (branch[1] & 0xF0) == 0x80 /* jcc */ ||
     116         op == 0xC7 && branch[1] == 0xF8 /* xbegin */,
     117         "Invalid opcode at patch point");
     118
     119     if (op == 0xEB || (op & 0xF0) == 0x70) {
     120       // short offset operators (jmp and jcc)
     121       char* disp = (char*) &branch[1];
     122       int imm8 = target - (address) &disp[1];
     123       guarantee(this->is8bit(imm8), "Short forward jump exceeds 8-bit offset");
     124       *disp = imm8;
     125     } else {
     126       int* disp = (int*) &branch[(op == 0x0F || op == 0xC7)? 2: 1];
     127       int imm32 = target - (address) &disp[1];
     128       *disp = imm32;
     129     }
     130   }
     131
     132   // The following 4 methods return the offset of the appropriate move instruction
     133
     134   // Support for fast byte/short loading with zero extension (depending on particular CPU)
     135   int load_unsigned_byte(Register dst, Address src);
     136   int load_unsigned_short(Register dst, Address src);
     137
     138   // Support for fast byte/short loading with sign extension (depending on particular CPU)
     139   int load_signed_byte(Register dst, Address src);
     140   int load_signed_short(Register dst, Address src);
     141
     142   // Support for sign-extension (hi:lo = extend_sign(lo))
     143   void extend_sign(Register hi, Register lo);
     144
     145   // Load and store values by size and signed-ness
     146   void load_sized_value(Register dst, Address src, size_t size_in_bytes, bool is_signed, Register dst2 = noreg);
     147   void store_sized_value(Address dst, Register src, size_t size_in_bytes, Register src2 = noreg);
     148
     149   // Support for inc/dec with optimal instruction selection depending on value
     150
     151   void increment(Register reg, int value = 1) { LP64_ONLY(incrementq(reg, value)) NOT_LP64(incrementl(reg, value)) ; }
     152   void decrement(Register reg, int value = 1) { LP64_ONLY(decrementq(reg, value)) NOT_LP64(decrementl(reg, value)) ; }
     153
     154   void decrementl(Address dst, int value = 1);
     155   void decrementl(Register reg, int value = 1);
     156
     157   void decrementq(Register reg, int value = 1);
     158   void decrementq(Address dst, int value = 1);
     159
     160   void incrementl(Address dst, int value = 1);
     161   void incrementl(Register reg, int value = 1);
     162
     163   void incrementq(Register reg, int value = 1);
     164   void incrementq(Address dst, int value = 1);
     165
     166   // Support optimal SSE move instructions.
     167   void movflt(XMMRegister dst, XMMRegister src) {
     168     if (UseXmmRegToRegMoveAll) { movaps(dst, src); return; }
     169     else                       { movss (dst, src); return; }
     170   }
     171   void movflt(XMMRegister dst, Address src) { movss(dst, src); }
     172   void movflt(XMMRegister dst, AddressLiteral src);
     173   void movflt(Address dst, XMMRegister src) { movss(dst, src); }
     174
     175   void movdbl(XMMRegister dst, XMMRegister src) {
     176     if (UseXmmRegToRegMoveAll) { movapd(dst, src); return; }
     177     else                       { movsd (dst, src); return; }
     178   }
     179
     180   void movdbl(XMMRegister dst, AddressLiteral src);
     181
     182   void movdbl(XMMRegister dst, Address src) {
     183     if (UseXmmLoadAndClearUpper) { movsd (dst, src); return; }
     184     else                         { movlpd(dst, src); return; }
     185   }
     186   void movdbl(Address dst, XMMRegister src) { movsd(dst, src); }
     187
     188   void incrementl(AddressLiteral dst);
     189   void incrementl(ArrayAddress dst);
     190
     191   void incrementq(AddressLiteral dst);
     192
     193   // Alignment
     194   void align(int modulus);
     195
     196   // A 5 byte nop that is safe for patching (see patch_verified_entry)
     197   void fat_nop();
     198
     199   // Stack frame creation/removal
     200   void enter();
     201   void leave();
     202
     203   // Support for getting the JavaThread pointer (i.e.; a reference to thread-local information)
     204   // The pointer will be loaded into the thread register.
     205   void get_thread(Register thread);
     206
     207
     208   // Support for VM calls
     209   //
     210   // It is imperative that all calls into the VM are handled via the call_VM macros.
     211   // They make sure that the stack linkage is setup correctly. call_VM's correspond
     212   // to ENTRY/ENTRY_X entry points while call_VM_leaf's correspond to LEAF entry points.
     213
     214
     215   void call_VM(Register oop_result,
     216                address entry_point,
     217                bool check_exceptions = true);
     218   void call_VM(Register oop_result,
     219                address entry_point,
     220                Register arg_1,
     221                bool check_exceptions = true);
     222   void call_VM(Register oop_result,
     223                address entry_point,
     224                Register arg_1, Register arg_2,
     225                bool check_exceptions = true);
     226   void call_VM(Register oop_result,
     227                address entry_point,
     228                Register arg_1, Register arg_2, Register arg_3,
     229                bool check_exceptions = true);
     230
     231   // Overloadings with last_Java_sp
     232   void call_VM(Register oop_result,
     233                Register last_java_sp,
     234                address entry_point,
     235                int number_of_arguments = 0,
     236                bool check_exceptions = true);
     237   void call_VM(Register oop_result,
     238                Register last_java_sp,
     239                address entry_point,
     240                Register arg_1, bool
     241                check_exceptions = true);
     242   void call_VM(Register oop_result,
     243                Register last_java_sp,
     244                address entry_point,
     245                Register arg_1, Register arg_2,
     246                bool check_exceptions = true);
     247   void call_VM(Register oop_result,
     248                Register last_java_sp,
     249                address entry_point,
     250                Register arg_1, Register arg_2, Register arg_3,
     251                bool check_exceptions = true);
     252
     253   void get_vm_result  (Register oop_result, Register thread);
     254   void get_vm_result_2(Register metadata_result, Register thread);
     255
     256   // These always tightly bind to MacroAssembler::call_VM_base
     257   // bypassing the virtual implementation
     258   void super_call_VM(Register oop_result, Register last_java_sp, address entry_point, int number_of_arguments = 0, bool check_exceptions = true);
     259   void super_call_VM(Register oop_result, Register last_java_sp, address entry_point, Register arg_1, bool check_exceptions = true);
     260   void super_call_VM(Register oop_result, Register last_java_sp, address entry_point, Register arg_1, Register arg_2, bool check_exceptions = true);
     261   void super_call_VM(Register oop_result, Register last_java_sp, address entry_point, Register arg_1, Register arg_2, Register arg_3, bool check_exceptions = true);
     262   void super_call_VM(Register oop_result, Register last_java_sp, address entry_point, Register arg_1, Register arg_2, Register arg_3, Register arg_4, bool check_exceptions = true);
     263
     264   void call_VM_leaf(address entry_point,
     265                     int number_of_arguments = 0);
     266   void call_VM_leaf(address entry_point,
     267                     Register arg_1);
     268   void call_VM_leaf(address entry_point,
     269                     Register arg_1, Register arg_2);
     270   void call_VM_leaf(address entry_point,
     271                     Register arg_1, Register arg_2, Register arg_3);
     272
     273   // These always tightly bind to MacroAssembler::call_VM_leaf_base
     274   // bypassing the virtual implementation
     275   void super_call_VM_leaf(address entry_point);
     276   void super_call_VM_leaf(address entry_point, Register arg_1);
     277   void super_call_VM_leaf(address entry_point, Register arg_1, Register arg_2);
     278   void super_call_VM_leaf(address entry_point, Register arg_1, Register arg_2, Register arg_3);
     279   void super_call_VM_leaf(address entry_point, Register arg_1, Register arg_2, Register arg_3, Register arg_4);
     280
     281   // last Java Frame (fills frame anchor)
     282   void set_last_Java_frame(Register thread,
     283                            Register last_java_sp,
     284                            Register last_java_fp,
     285                            address last_java_pc);
     286
     287   // thread in the default location (r15_thread on 64bit)
     288   void set_last_Java_frame(Register last_java_sp,
     289                            Register last_java_fp,
     290                            address last_java_pc);
     291
     292   void reset_last_Java_frame(Register thread, bool clear_fp, bool clear_pc);
     293
     294   // thread in the default location (r15_thread on 64bit)
     295   void reset_last_Java_frame(bool clear_fp, bool clear_pc);
     296
     297   // Stores
     298   void store_check(Register obj);                // store check for obj - register is destroyed afterwards
     299   void store_check(Register obj, Address dst);   // same as above, dst is exact store location (reg. is destroyed)
     300
     301 #if INCLUDE_ALL_GCS
     302
     303   void g1_write_barrier_pre(Register obj,
     304                             Register pre_val,
     305                             Register thread,
     306                             Register tmp,
     307                             bool tosca_live,
     308                             bool expand_call);
     309
     310   void g1_write_barrier_post(Register store_addr,
     311                              Register new_val,
     312                              Register thread,
     313                              Register tmp,
     314                              Register tmp2);
     315
     316 #endif // INCLUDE_ALL_GCS
     317
     318   // split store_check(Register obj) to enhance instruction interleaving
     319   void store_check_part_1(Register obj);
     320   void store_check_part_2(Register obj);
     321
     322   // C 'boolean' to Java boolean: x == 0 ? 0 : 1
     323   void c2bool(Register x);
     324
     325   // C++ bool manipulation
     326
     327   void movbool(Register dst, Address src);
     328   void movbool(Address dst, bool boolconst);
     329   void movbool(Address dst, Register src);
     330   void testbool(Register dst);
     331
     332   // oop manipulations
     333   void load_klass(Register dst, Register src);
     334   void store_klass(Register dst, Register src);
     335
     336   void load_heap_oop(Register dst, Address src);
     337   void load_heap_oop_not_null(Register dst, Address src);
     338   void store_heap_oop(Address dst, Register src);
     339   void cmp_heap_oop(Register src1, Address src2, Register tmp = noreg);
     340
     341   // Used for storing NULL. All other oop constants should be
     342   // stored using routines that take a jobject.
     343   void store_heap_oop_null(Address dst);
     344
     345   void load_prototype_header(Register dst, Register src);
     346
     347 #ifdef _LP64
     348   void store_klass_gap(Register dst, Register src);
     349
     350   // This dummy is to prevent a call to store_heap_oop from
     351   // converting a zero (like NULL) into a Register by giving
     352   // the compiler two choices it can't resolve
     353
     354   void store_heap_oop(Address dst, void* dummy);
     355
     356   void encode_heap_oop(Register r);
     357   void decode_heap_oop(Register r);
     358   void encode_heap_oop_not_null(Register r);
     359   void decode_heap_oop_not_null(Register r);
     360   void encode_heap_oop_not_null(Register dst, Register src);
     361   void decode_heap_oop_not_null(Register dst, Register src);
     362
     363   void set_narrow_oop(Register dst, jobject obj);
     364   void set_narrow_oop(Address dst, jobject obj);
     365   void cmp_narrow_oop(Register dst, jobject obj);
     366   void cmp_narrow_oop(Address dst, jobject obj);
     367
     368   void encode_klass_not_null(Register r);
     369   void decode_klass_not_null(Register r);
     370   void encode_klass_not_null(Register dst, Register src);
     371   void decode_klass_not_null(Register dst, Register src);
     372   void set_narrow_klass(Register dst, Klass* k);
     373   void set_narrow_klass(Address dst, Klass* k);
     374   void cmp_narrow_klass(Register dst, Klass* k);
     375   void cmp_narrow_klass(Address dst, Klass* k);
     376
     377   // Returns the byte size of the instructions generated by decode_klass_not_null()
     378   // when compressed klass pointers are being used.
     379   static int instr_size_for_decode_klass_not_null();
     380
     381   // if heap base register is used - reinit it with the correct value
     382   void reinit_heapbase();
     383
     384   DEBUG_ONLY(void verify_heapbase(const char* msg);)
     385
     386 #endif // _LP64
     387
     388   // Int division/remainder for Java
     389   // (as idivl, but checks for special case as described in JVM spec.)
     390   // returns idivl instruction offset for implicit exception handling
     391   int corrected_idivl(Register reg);
     392
     393   // Long division/remainder for Java
     394   // (as idivq, but checks for special case as described in JVM spec.)
     395   // returns idivq instruction offset for implicit exception handling
     396   int corrected_idivq(Register reg);
     397
     398   void int3();
     399
     400   // Long operation macros for a 32bit cpu
     401   // Long negation for Java
     402   void lneg(Register hi, Register lo);
     403
     404   // Long multiplication for Java
     405   // (destroys contents of eax, ebx, ecx and edx)
     406   void lmul(int x_rsp_offset, int y_rsp_offset); // rdx:rax = x * y
     407
     408   // Long shifts for Java
     409   // (semantics as described in JVM spec.)
     410   void lshl(Register hi, Register lo);                               // hi:lo << (rcx & 0x3f)
     411   void lshr(Register hi, Register lo, bool sign_extension = false);  // hi:lo >> (rcx & 0x3f)
     412
     413   // Long compare for Java
     414   // (semantics as described in JVM spec.)
     415   void lcmp2int(Register x_hi, Register x_lo, Register y_hi, Register y_lo); // x_hi = lcmp(x, y)
     416
     417
     418   // misc
     419
     420   // Sign extension
     421   void sign_extend_short(Register reg);
     422   void sign_extend_byte(Register reg);
     423
     424   // Division by power of 2, rounding towards 0
     425   void division_with_shift(Register reg, int shift_value);
     426
     427   // Compares the top-most stack entries on the FPU stack and sets the eflags as follows:
     428   //
     429   // CF (corresponds to C0) if x < y
     430   // PF (corresponds to C2) if unordered
     431   // ZF (corresponds to C3) if x = y
     432   //
     433   // The arguments are in reversed order on the stack (i.e., top of stack is first argument).
     434   // tmp is a temporary register, if none is available use noreg (only matters for non-P6 code)
     435   void fcmp(Register tmp);
     436   // Variant of the above which allows y to be further down the stack
     437   // and which only pops x and y if specified. If pop_right is
     438   // specified then pop_left must also be specified.
     439   void fcmp(Register tmp, int index, bool pop_left, bool pop_right);
     440
     441   // Floating-point comparison for Java
     442   // Compares the top-most stack entries on the FPU stack and stores the result in dst.
     443   // The arguments are in reversed order on the stack (i.e., top of stack is first argument).
     444   // (semantics as described in JVM spec.)
     445   void fcmp2int(Register dst, bool unordered_is_less);
     446   // Variant of the above which allows y to be further down the stack
     447   // and which only pops x and y if specified. If pop_right is
     448   // specified then pop_left must also be specified.
     449   void fcmp2int(Register dst, bool unordered_is_less, int index, bool pop_left, bool pop_right);
     450
     451   // Floating-point remainder for Java (ST0 = ST0 fremr ST1, ST1 is empty afterwards)
     452   // tmp is a temporary register, if none is available use noreg
     453   void fremr(Register tmp);
     454
     455
     456   // same as fcmp2int, but using SSE2
     457   void cmpss2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less);
     458   void cmpsd2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less);
     459
     460   // Inlined sin/cos generator for Java; must not use CPU instruction
     461   // directly on Intel as it does not have high enough precision
     462   // outside of the range [-pi/4, pi/4]. Extra argument indicate the
     463   // number of FPU stack slots in use; all but the topmost will
     464   // require saving if a slow case is necessary. Assumes argument is
     465   // on FP TOS; result is on FP TOS.  No cpu registers are changed by
     466   // this code.
     467   void trigfunc(char trig, int num_fpu_regs_in_use = 1);
     468
     469   // branch to L if FPU flag C2 is set/not set
     470   // tmp is a temporary register, if none is available use noreg
     471   void jC2 (Register tmp, Label& L);
     472   void jnC2(Register tmp, Label& L);
     473
     474   // Pop ST (ffree & fincstp combined)
     475   void fpop();
     476
     477   // pushes double TOS element of FPU stack on CPU stack; pops from FPU stack
     478   void push_fTOS();
     479
     480   // pops double TOS element from CPU stack and pushes on FPU stack
     481   void pop_fTOS();
     482
     483   void empty_FPU_stack();
     484
     485   void push_IU_state();
     486   void pop_IU_state();
     487
     488   void push_FPU_state();
     489   void pop_FPU_state();
     490
     491   void push_CPU_state();
     492   void pop_CPU_state();
     493
     494   // Round up to a power of two
     495   void round_to(Register reg, int modulus);
     496
     497   // Callee saved registers handling
     498   void push_callee_saved_registers();
     499   void pop_callee_saved_registers();
     500
     501   // allocation
     502   void eden_allocate(
     503     Register obj,                      // result: pointer to object after successful allocation
     504     Register var_size_in_bytes,        // object size in bytes if unknown at compile time; invalid otherwise
     505     int      con_size_in_bytes,        // object size in bytes if   known at compile time
     506     Register t1,                       // temp register
     507     Label&   slow_case                 // continuation point if fast allocation fails
     508   );
     509   void tlab_allocate(
     510     Register obj,                      // result: pointer to object after successful allocation
     511     Register var_size_in_bytes,        // object size in bytes if unknown at compile time; invalid otherwise
     512     int      con_size_in_bytes,        // object size in bytes if   known at compile time
     513     Register t1,                       // temp register
     514     Register t2,                       // temp register
     515     Label&   slow_case                 // continuation point if fast allocation fails
     516   );
     517   Register tlab_refill(Label& retry_tlab, Label& try_eden, Label& slow_case); // returns TLS address
     518   void incr_allocated_bytes(Register thread,
     519                             Register var_size_in_bytes, int con_size_in_bytes,
     520                             Register t1 = noreg);
     521
     522   // interface method calling
     523   void lookup_interface_method(Register recv_klass,
     524                                Register intf_klass,
     525                                RegisterOrConstant itable_index,
     526                                Register method_result,
     527                                Register scan_temp,
     528                                Label& no_such_interface);
     529
     530   // virtual method calling
     531   void lookup_virtual_method(Register recv_klass,
     532                              RegisterOrConstant vtable_index,
     533                              Register method_result);
     534
     535   // Test sub_klass against super_klass, with fast and slow paths.
     536
     537   // The fast path produces a tri-state answer: yes / no / maybe-slow.
     538   // One of the three labels can be NULL, meaning take the fall-through.
     539   // If super_check_offset is -1, the value is loaded up from super_klass.
     540   // No registers are killed, except temp_reg.
     541   void check_klass_subtype_fast_path(Register sub_klass,
     542                                      Register super_klass,
     543                                      Register temp_reg,
     544                                      Label* L_success,
     545                                      Label* L_failure,
     546                                      Label* L_slow_path,
     547                 RegisterOrConstant super_check_offset = RegisterOrConstant(-1));
     548
     549   // The rest of the type check; must be wired to a corresponding fast path.
     550   // It does not repeat the fast path logic, so don't use it standalone.
     551   // The temp_reg and temp2_reg can be noreg, if no temps are available.
     552   // Updates the sub's secondary super cache as necessary.
     553   // If set_cond_codes, condition codes will be Z on success, NZ on failure.
     554   void check_klass_subtype_slow_path(Register sub_klass,
     555                                      Register super_klass,
     556                                      Register temp_reg,
     557                                      Register temp2_reg,
     558                                      Label* L_success,
     559                                      Label* L_failure,
     560                                      bool set_cond_codes = false);
     561
     562   // Simplified, combined version, good for typical uses.
     563   // Falls through on failure.
     564   void check_klass_subtype(Register sub_klass,
     565                            Register super_klass,
     566                            Register temp_reg,
     567                            Label& L_success);
     568
     569   // method handles (JSR 292)
     570   Address argument_address(RegisterOrConstant arg_slot, int extra_slot_offset = 0);
     571
     572   //----
     573   void set_word_if_not_zero(Register reg); // sets reg to 1 if not zero, otherwise 0
     574
     575   // Debugging
     576
     577   // only if +VerifyOops
     578   // TODO: Make these macros with file and line like sparc version!
     579   void verify_oop(Register reg, const char* s = "broken oop");
     580   void verify_oop_addr(Address addr, const char * s = "broken oop addr");
     581
     582   // TODO: verify method and klass metadata (compare against vptr?)
     583   void _verify_method_ptr(Register reg, const char * msg, const char * file, int line) {}
     584   void _verify_klass_ptr(Register reg, const char * msg, const char * file, int line){}
     585
     586 #define verify_method_ptr(reg) _verify_method_ptr(reg, "broken method " #reg, __FILE__, __LINE__)
     587 #define verify_klass_ptr(reg) _verify_klass_ptr(reg, "broken klass " #reg, __FILE__, __LINE__)
     588
     589   // only if +VerifyFPU
     590   void verify_FPU(int stack_depth, const char* s = "illegal FPU state");
     591
     592   // Verify or restore cpu control state after JNI call
     593   void restore_cpu_control_state_after_jni();
     594
     595   // prints msg, dumps registers and stops execution
     596   void stop(const char* msg);
     597
     598   // prints msg and continues
     599   void warn(const char* msg);
     600
     601   // dumps registers and other state
     602   void print_state();
     603
     604   static void debug32(int rdi, int rsi, int rbp, int rsp, int rbx, int rdx, int rcx, int rax, int eip, char* msg);
     605   static void debug64(char* msg, int64_t pc, int64_t regs[]);
     606   static void print_state32(int rdi, int rsi, int rbp, int rsp, int rbx, int rdx, int rcx, int rax, int eip);
     607   static void print_state64(int64_t pc, int64_t regs[]);
     608
     609   void os_breakpoint();
     610
     611   void untested()                                { stop("untested"); }
     612
     613   void unimplemented(const char* what = "")      { char* b = new char[1024];  jio_snprintf(b, 1024, "unimplemented: %s", what);  stop(b); }
     614
     615   void should_not_reach_here()                   { stop("should not reach here"); }
     616
     617   void print_CPU_state();
     618
     619   // Stack overflow checking
     620   void bang_stack_with_offset(int offset) {
     621     // stack grows down, caller passes positive offset
     622     assert(offset > 0, "must bang with negative offset");
     623     movl(Address(rsp, (-offset)), rax);
     624   }
     625
     626   // Writes to stack successive pages until offset reached to check for
     627   // stack overflow + shadow pages.  Also, clobbers tmp
     628   void bang_stack_size(Register size, Register tmp);
     629
     630   virtual RegisterOrConstant delayed_value_impl(intptr_t* delayed_value_addr,
     631                                                 Register tmp,
     632                                                 int offset);
     633
     634   // Support for serializing memory accesses between threads
     635   void serialize_memory(Register thread, Register tmp);
     636
     637   void verify_tlab();
     638
     639   // Biased locking support
     640   // lock_reg and obj_reg must be loaded up with the appropriate values.
     641   // swap_reg must be rax, and is killed.
     642   // tmp_reg is optional. If it is supplied (i.e., != noreg) it will
     643   // be killed; if not supplied, push/pop will be used internally to
     644   // allocate a temporary (inefficient, avoid if possible).
     645   // Optional slow case is for implementations (interpreter and C1) which branch to
     646   // slow case directly. Leaves condition codes set for C2's Fast_Lock node.
     647   // Returns offset of first potentially-faulting instruction for null
     648   // check info (currently consumed only by C1). If
     649   // swap_reg_contains_mark is true then returns -1 as it is assumed
     650   // the calling code has already passed any potential faults.
     651   int biased_locking_enter(Register lock_reg, Register obj_reg,
     652                            Register swap_reg, Register tmp_reg,
     653                            bool swap_reg_contains_mark,
     654                            Label& done, Label* slow_case = NULL,
     655                            BiasedLockingCounters* counters = NULL);
     656   void biased_locking_exit (Register obj_reg, Register temp_reg, Label& done);
     657 #ifdef COMPILER2
     658   // Code used by cmpFastLock and cmpFastUnlock mach instructions in .ad file.
     659   // See full desription in macroAssembler_x86.cpp.
     660   void fast_lock(Register obj, Register box, Register tmp,
     661                  Register scr, Register cx1, Register cx2,
     662                  BiasedLockingCounters* counters,
     663                  RTMLockingCounters* rtm_counters,
     664                  RTMLockingCounters* stack_rtm_counters,
     665                  Metadata* method_data,
     666                  bool use_rtm, bool profile_rtm);
     667   void fast_unlock(Register obj, Register box, Register tmp, bool use_rtm);
     668 #if INCLUDE_RTM_OPT
     669   void rtm_counters_update(Register abort_status, Register rtm_counters);
     670   void branch_on_random_using_rdtsc(Register tmp, Register scr, int count, Label& brLabel);
     671   void rtm_abort_ratio_calculation(Register tmp, Register rtm_counters_reg,
     672                                    RTMLockingCounters* rtm_counters,
     673                                    Metadata* method_data);
     674   void rtm_profiling(Register abort_status_Reg, Register rtm_counters_Reg,
     675                      RTMLockingCounters* rtm_counters, Metadata* method_data, bool profile_rtm);
     676   void rtm_retry_lock_on_abort(Register retry_count, Register abort_status, Label& retryLabel);
     677   void rtm_retry_lock_on_busy(Register retry_count, Register box, Register tmp, Register scr, Label& retryLabel);
     678   void rtm_stack_locking(Register obj, Register tmp, Register scr,
     679                          Register retry_on_abort_count,
     680                          RTMLockingCounters* stack_rtm_counters,
     681                          Metadata* method_data, bool profile_rtm,
     682                          Label& DONE_LABEL, Label& IsInflated);
     683   void rtm_inflated_locking(Register obj, Register box, Register tmp,
     684                             Register scr, Register retry_on_busy_count,
     685                             Register retry_on_abort_count,
     686                             RTMLockingCounters* rtm_counters,
     687                             Metadata* method_data, bool profile_rtm,
     688                             Label& DONE_LABEL);
     689 #endif
     690 #endif
     691
     692   Condition negate_condition(Condition cond);
     693
     694   // Instructions that use AddressLiteral operands. These instruction can handle 32bit/64bit
     695   // operands. In general the names are modified to avoid hiding the instruction in Assembler
     696   // so that we don't need to implement all the varieties in the Assembler with trivial wrappers
     697   // here in MacroAssembler. The major exception to this rule is call
     698
     699   // Arithmetics
     700
     701
     702   void addptr(Address dst, int32_t src) { LP64_ONLY(addq(dst, src)) NOT_LP64(addl(dst, src)) ; }
     703   void addptr(Address dst, Register src);
     704
     705   void addptr(Register dst, Address src) { LP64_ONLY(addq(dst, src)) NOT_LP64(addl(dst, src)); }
     706   void addptr(Register dst, int32_t src);
     707   void addptr(Register dst, Register src);
     708   void addptr(Register dst, RegisterOrConstant src) {
     709     if (src.is_constant()) addptr(dst, (int) src.as_constant());
     710     else                   addptr(dst,       src.as_register());
     711   }
     712
     713   void andptr(Register dst, int32_t src);
     714   void andptr(Register src1, Register src2) { LP64_ONLY(andq(src1, src2)) NOT_LP64(andl(src1, src2)) ; }
     715
     716   void cmp8(AddressLiteral src1, int imm);
     717
     718   // renamed to drag out the casting of address to int32_t/intptr_t
     719   void cmp32(Register src1, int32_t imm);
     720
     721   void cmp32(AddressLiteral src1, int32_t imm);
     722   // compare reg - mem, or reg - &mem
     723   void cmp32(Register src1, AddressLiteral src2);
     724
     725   void cmp32(Register src1, Address src2);
     726
     727 #ifndef _LP64
     728   void cmpklass(Address dst, Metadata* obj);
     729   void cmpklass(Register dst, Metadata* obj);
     730   void cmpoop(Address dst, jobject obj);
     731   void cmpoop(Register dst, jobject obj);
     732 #endif // _LP64
     733
     734   // NOTE src2 must be the lval. This is NOT an mem-mem compare
     735   void cmpptr(Address src1, AddressLiteral src2);
     736
     737   void cmpptr(Register src1, AddressLiteral src2);
     738
     739   void cmpptr(Register src1, Register src2) { LP64_ONLY(cmpq(src1, src2)) NOT_LP64(cmpl(src1, src2)) ; }
     740   void cmpptr(Register src1, Address src2) { LP64_ONLY(cmpq(src1, src2)) NOT_LP64(cmpl(src1, src2)) ; }
     741   // void cmpptr(Address src1, Register src2) { LP64_ONLY(cmpq(src1, src2)) NOT_LP64(cmpl(src1, src2)) ; }
     742
     743   void cmpptr(Register src1, int32_t src2) { LP64_ONLY(cmpq(src1, src2)) NOT_LP64(cmpl(src1, src2)) ; }
     744   void cmpptr(Address src1, int32_t src2) { LP64_ONLY(cmpq(src1, src2)) NOT_LP64(cmpl(src1, src2)) ; }
     745
     746   // cmp64 to avoild hiding cmpq
     747   void cmp64(Register src1, AddressLiteral src);
     748
     749   void cmpxchgptr(Register reg, Address adr);
     750
     751   void locked_cmpxchgptr(Register reg, AddressLiteral adr);
     752
     753
     754   void imulptr(Register dst, Register src) { LP64_ONLY(imulq(dst, src)) NOT_LP64(imull(dst, src)); }
     755   void imulptr(Register dst, Register src, int imm32) { LP64_ONLY(imulq(dst, src, imm32)) NOT_LP64(imull(dst, src, imm32)); }
     756
     757
     758   void negptr(Register dst) { LP64_ONLY(negq(dst)) NOT_LP64(negl(dst)); }
     759
     760   void notptr(Register dst) { LP64_ONLY(notq(dst)) NOT_LP64(notl(dst)); }
     761
     762   void shlptr(Register dst, int32_t shift);
     763   void shlptr(Register dst) { LP64_ONLY(shlq(dst)) NOT_LP64(shll(dst)); }
     764
     765   void shrptr(Register dst, int32_t shift);
     766   void shrptr(Register dst) { LP64_ONLY(shrq(dst)) NOT_LP64(shrl(dst)); }
     767
     768   void sarptr(Register dst) { LP64_ONLY(sarq(dst)) NOT_LP64(sarl(dst)); }
     769   void sarptr(Register dst, int32_t src) { LP64_ONLY(sarq(dst, src)) NOT_LP64(sarl(dst, src)); }
     770
     771   void subptr(Address dst, int32_t src) { LP64_ONLY(subq(dst, src)) NOT_LP64(subl(dst, src)); }
     772
     773   void subptr(Register dst, Address src) { LP64_ONLY(subq(dst, src)) NOT_LP64(subl(dst, src)); }
     774   void subptr(Register dst, int32_t src);
     775   // Force generation of a 4 byte immediate value even if it fits into 8bit
     776   void subptr_imm32(Register dst, int32_t src);
     777   void subptr(Register dst, Register src);
     778   void subptr(Register dst, RegisterOrConstant src) {
     779     if (src.is_constant()) subptr(dst, (int) src.as_constant());
     780     else                   subptr(dst,       src.as_register());
     781   }
     782
     783   void sbbptr(Address dst, int32_t src) { LP64_ONLY(sbbq(dst, src)) NOT_LP64(sbbl(dst, src)); }
     784   void sbbptr(Register dst, int32_t src) { LP64_ONLY(sbbq(dst, src)) NOT_LP64(sbbl(dst, src)); }
     785
     786   void xchgptr(Register src1, Register src2) { LP64_ONLY(xchgq(src1, src2)) NOT_LP64(xchgl(src1, src2)) ; }
     787   void xchgptr(Register src1, Address src2) { LP64_ONLY(xchgq(src1, src2)) NOT_LP64(xchgl(src1, src2)) ; }
     788
     789   void xaddptr(Address src1, Register src2) { LP64_ONLY(xaddq(src1, src2)) NOT_LP64(xaddl(src1, src2)) ; }
     790
     791
     792
     793   // Helper functions for statistics gathering.
     794   // Conditionally (atomically, on MPs) increments passed counter address, preserving condition codes.
     795   void cond_inc32(Condition cond, AddressLiteral counter_addr);
     796   // Unconditional atomic increment.
     797   void atomic_incl(Address counter_addr);
     798   void atomic_incl(AddressLiteral counter_addr, Register scr = rscratch1);
     799 #ifdef _LP64
     800   void atomic_incq(Address counter_addr);
     801   void atomic_incq(AddressLiteral counter_addr, Register scr = rscratch1);
     802 #endif
     803   void atomic_incptr(AddressLiteral counter_addr, Register scr = rscratch1) { LP64_ONLY(atomic_incq(counter_addr, scr)) NOT_LP64(atomic_incl(counter_addr, scr)) ; }
     804   void atomic_incptr(Address counter_addr) { LP64_ONLY(atomic_incq(counter_addr)) NOT_LP64(atomic_incl(counter_addr)) ; }
     805
     806   void lea(Register dst, AddressLiteral adr);
     807   void lea(Address dst, AddressLiteral adr);
     808   void lea(Register dst, Address adr) { Assembler::lea(dst, adr); }
     809
     810   void leal32(Register dst, Address src) { leal(dst, src); }
     811
     812   // Import other testl() methods from the parent class or else
     813   // they will be hidden by the following overriding declaration.
     814   using Assembler::testl;
     815   void testl(Register dst, AddressLiteral src);
     816
     817   void orptr(Register dst, Address src) { LP64_ONLY(orq(dst, src)) NOT_LP64(orl(dst, src)); }
     818   void orptr(Register dst, Register src) { LP64_ONLY(orq(dst, src)) NOT_LP64(orl(dst, src)); }
     819   void orptr(Register dst, int32_t src) { LP64_ONLY(orq(dst, src)) NOT_LP64(orl(dst, src)); }
     820   void orptr(Address dst, int32_t imm32) { LP64_ONLY(orq(dst, imm32)) NOT_LP64(orl(dst, imm32)); }
     821
     822   void testptr(Register src, int32_t imm32) {  LP64_ONLY(testq(src, imm32)) NOT_LP64(testl(src, imm32)); }
     823   void testptr(Register src1, Register src2);
     824
     825   void xorptr(Register dst, Register src) { LP64_ONLY(xorq(dst, src)) NOT_LP64(xorl(dst, src)); }
     826   void xorptr(Register dst, Address src) { LP64_ONLY(xorq(dst, src)) NOT_LP64(xorl(dst, src)); }
     827
     828   // Calls
     829
     830   void call(Label& L, relocInfo::relocType rtype);
     831   void call(Register entry);
     832
     833   // NOTE: this call tranfers to the effective address of entry NOT
     834   // the address contained by entry. This is because this is more natural
     835   // for jumps/calls.
     836   void call(AddressLiteral entry);
     837
     838   // Emit the CompiledIC call idiom
     839   void ic_call(address entry);
     840
     841   // Jumps
     842
     843   // NOTE: these jumps tranfer to the effective address of dst NOT
     844   // the address contained by dst. This is because this is more natural
     845   // for jumps/calls.
     846   void jump(AddressLiteral dst);
     847   void jump_cc(Condition cc, AddressLiteral dst);
     848
     849   // 32bit can do a case table jump in one instruction but we no longer allow the base
     850   // to be installed in the Address class. This jump will tranfers to the address
     851   // contained in the location described by entry (not the address of entry)
     852   void jump(ArrayAddress entry);
     853
     854   // Floating
     855
     856   void andpd(XMMRegister dst, Address src) { Assembler::andpd(dst, src); }
     857   void andpd(XMMRegister dst, AddressLiteral src);
     858
     859   void andps(XMMRegister dst, XMMRegister src) { Assembler::andps(dst, src); }
     860   void andps(XMMRegister dst, Address src) { Assembler::andps(dst, src); }
     861   void andps(XMMRegister dst, AddressLiteral src);
     862
     863   void comiss(XMMRegister dst, XMMRegister src) { Assembler::comiss(dst, src); }
     864   void comiss(XMMRegister dst, Address src) { Assembler::comiss(dst, src); }
     865   void comiss(XMMRegister dst, AddressLiteral src);
     866
     867   void comisd(XMMRegister dst, XMMRegister src) { Assembler::comisd(dst, src); }
     868   void comisd(XMMRegister dst, Address src) { Assembler::comisd(dst, src); }
     869   void comisd(XMMRegister dst, AddressLiteral src);
     870
     871   void fadd_s(Address src)        { Assembler::fadd_s(src); }
     872   void fadd_s(AddressLiteral src) { Assembler::fadd_s(as_Address(src)); }
     873
     874   void fldcw(Address src) { Assembler::fldcw(src); }
     875   void fldcw(AddressLiteral src);
     876
     877   void fld_s(int index)   { Assembler::fld_s(index); }
     878   void fld_s(Address src) { Assembler::fld_s(src); }
     879   void fld_s(AddressLiteral src);
     880
     881   void fld_d(Address src) { Assembler::fld_d(src); }
     882   void fld_d(AddressLiteral src);
     883
     884   void fld_x(Address src) { Assembler::fld_x(src); }
     885   void fld_x(AddressLiteral src);
     886
     887   void fmul_s(Address src)        { Assembler::fmul_s(src); }
     888   void fmul_s(AddressLiteral src) { Assembler::fmul_s(as_Address(src)); }
     889
     890   void ldmxcsr(Address src) { Assembler::ldmxcsr(src); }
     891   void ldmxcsr(AddressLiteral src);
     892
     893   // compute pow(x,y) and exp(x) with x86 instructions. Don't cover
     894   // all corner cases and may result in NaN and require fallback to a
     895   // runtime call.
     896   void fast_pow();
     897   void fast_exp();
     898   void increase_precision();
     899   void restore_precision();
     900
     901   // computes exp(x). Fallback to runtime call included.
     902   void exp_with_fallback(int num_fpu_regs_in_use) { pow_or_exp(true, num_fpu_regs_in_use); }
     903   // computes pow(x,y). Fallback to runtime call included.
     904   void pow_with_fallback(int num_fpu_regs_in_use) { pow_or_exp(false, num_fpu_regs_in_use); }
     905
     906 private:
     907
     908   // call runtime as a fallback for trig functions and pow/exp.
     909   void fp_runtime_fallback(address runtime_entry, int nb_args, int num_fpu_regs_in_use);
     910
     911   // computes 2^(Ylog2X); Ylog2X in ST(0)
     912   void pow_exp_core_encoding();
     913
     914   // computes pow(x,y) or exp(x). Fallback to runtime call included.
     915   void pow_or_exp(bool is_exp, int num_fpu_regs_in_use);
     916
     917   // these are private because users should be doing movflt/movdbl
     918
     919   void movss(Address dst, XMMRegister src)     { Assembler::movss(dst, src); }
     920   void movss(XMMRegister dst, XMMRegister src) { Assembler::movss(dst, src); }
     921   void movss(XMMRegister dst, Address src)     { Assembler::movss(dst, src); }
     922   void movss(XMMRegister dst, AddressLiteral src);
     923
     924   void movlpd(XMMRegister dst, Address src)    {Assembler::movlpd(dst, src); }
     925   void movlpd(XMMRegister dst, AddressLiteral src);
     926
     927 public:
     928
     929   void addsd(XMMRegister dst, XMMRegister src)    { Assembler::addsd(dst, src); }
     930   void addsd(XMMRegister dst, Address src)        { Assembler::addsd(dst, src); }
     931   void addsd(XMMRegister dst, AddressLiteral src);
     932
     933   void addss(XMMRegister dst, XMMRegister src)    { Assembler::addss(dst, src); }
     934   void addss(XMMRegister dst, Address src)        { Assembler::addss(dst, src); }
     935   void addss(XMMRegister dst, AddressLiteral src);
     936
     937   void divsd(XMMRegister dst, XMMRegister src)    { Assembler::divsd(dst, src); }
     938   void divsd(XMMRegister dst, Address src)        { Assembler::divsd(dst, src); }
     939   void divsd(XMMRegister dst, AddressLiteral src);
     940
     941   void divss(XMMRegister dst, XMMRegister src)    { Assembler::divss(dst, src); }
     942   void divss(XMMRegister dst, Address src)        { Assembler::divss(dst, src); }
     943   void divss(XMMRegister dst, AddressLiteral src);
     944
     945   // Move Unaligned Double Quadword
     946   void movdqu(Address     dst, XMMRegister src)   { Assembler::movdqu(dst, src); }
     947   void movdqu(XMMRegister dst, Address src)       { Assembler::movdqu(dst, src); }
     948   void movdqu(XMMRegister dst, XMMRegister src)   { Assembler::movdqu(dst, src); }
     949   void movdqu(XMMRegister dst, AddressLiteral src);
     950
     951   // Move Aligned Double Quadword
     952   void movdqa(XMMRegister dst, Address src)       { Assembler::movdqa(dst, src); }
     953   void movdqa(XMMRegister dst, XMMRegister src)   { Assembler::movdqa(dst, src); }
     954   void movdqa(XMMRegister dst, AddressLiteral src);
     955
     956   void movsd(XMMRegister dst, XMMRegister src) { Assembler::movsd(dst, src); }
     957   void movsd(Address dst, XMMRegister src)     { Assembler::movsd(dst, src); }
     958   void movsd(XMMRegister dst, Address src)     { Assembler::movsd(dst, src); }
     959   void movsd(XMMRegister dst, AddressLiteral src);
     960
     961   void mulsd(XMMRegister dst, XMMRegister src)    { Assembler::mulsd(dst, src); }
     962   void mulsd(XMMRegister dst, Address src)        { Assembler::mulsd(dst, src); }
     963   void mulsd(XMMRegister dst, AddressLiteral src);
     964
     965   void mulss(XMMRegister dst, XMMRegister src)    { Assembler::mulss(dst, src); }
     966   void mulss(XMMRegister dst, Address src)        { Assembler::mulss(dst, src); }
     967   void mulss(XMMRegister dst, AddressLiteral src);
     968
     969   // Carry-Less Multiplication Quadword
     970   void pclmulldq(XMMRegister dst, XMMRegister src) {
     971     // 0x00 - multiply lower 64 bits [0:63]
     972     Assembler::pclmulqdq(dst, src, 0x00);
     973   }
     974   void pclmulhdq(XMMRegister dst, XMMRegister src) {
     975     // 0x11 - multiply upper 64 bits [64:127]
     976     Assembler::pclmulqdq(dst, src, 0x11);
     977   }
     978
     979   void sqrtsd(XMMRegister dst, XMMRegister src)    { Assembler::sqrtsd(dst, src); }
     980   void sqrtsd(XMMRegister dst, Address src)        { Assembler::sqrtsd(dst, src); }
     981   void sqrtsd(XMMRegister dst, AddressLiteral src);
     982
     983   void sqrtss(XMMRegister dst, XMMRegister src)    { Assembler::sqrtss(dst, src); }
     984   void sqrtss(XMMRegister dst, Address src)        { Assembler::sqrtss(dst, src); }
     985   void sqrtss(XMMRegister dst, AddressLiteral src);
     986
     987   void subsd(XMMRegister dst, XMMRegister src)    { Assembler::subsd(dst, src); }
     988   void subsd(XMMRegister dst, Address src)        { Assembler::subsd(dst, src); }
     989   void subsd(XMMRegister dst, AddressLiteral src);
     990
     991   void subss(XMMRegister dst, XMMRegister src)    { Assembler::subss(dst, src); }
     992   void subss(XMMRegister dst, Address src)        { Assembler::subss(dst, src); }
     993   void subss(XMMRegister dst, AddressLiteral src);
     994
     995   void ucomiss(XMMRegister dst, XMMRegister src) { Assembler::ucomiss(dst, src); }
     996   void ucomiss(XMMRegister dst, Address src)     { Assembler::ucomiss(dst, src); }
     997   void ucomiss(XMMRegister dst, AddressLiteral src);
     998
     999   void ucomisd(XMMRegister dst, XMMRegister src) { Assembler::ucomisd(dst, src); }
    1000   void ucomisd(XMMRegister dst, Address src)     { Assembler::ucomisd(dst, src); }
    1001   void ucomisd(XMMRegister dst, AddressLiteral src);
    1002
    1003   // Bitwise Logical XOR of Packed Double-Precision Floating-Point Values
    1004   void xorpd(XMMRegister dst, XMMRegister src) { Assembler::xorpd(dst, src); }
    1005   void xorpd(XMMRegister dst, Address src)     { Assembler::xorpd(dst, src); }
    1006   void xorpd(XMMRegister dst, AddressLiteral src);
    1007
    1008   // Bitwise Logical XOR of Packed Single-Precision Floating-Point Values
    1009   void xorps(XMMRegister dst, XMMRegister src) { Assembler::xorps(dst, src); }
    1010   void xorps(XMMRegister dst, Address src)     { Assembler::xorps(dst, src); }
    1011   void xorps(XMMRegister dst, AddressLiteral src);
    1012
    1013   // Shuffle Bytes
    1014   void pshufb(XMMRegister dst, XMMRegister src) { Assembler::pshufb(dst, src); }
    1015   void pshufb(XMMRegister dst, Address src)     { Assembler::pshufb(dst, src); }
    1016   void pshufb(XMMRegister dst, AddressLiteral src);
    1017   // AVX 3-operands instructions
    1018
    1019   void vaddsd(XMMRegister dst, XMMRegister nds, XMMRegister src) { Assembler::vaddsd(dst, nds, src); }
    1020   void vaddsd(XMMRegister dst, XMMRegister nds, Address src)     { Assembler::vaddsd(dst, nds, src); }
    1021   void vaddsd(XMMRegister dst, XMMRegister nds, AddressLiteral src);
    1022
    1023   void vaddss(XMMRegister dst, XMMRegister nds, XMMRegister src) { Assembler::vaddss(dst, nds, src); }
    1024   void vaddss(XMMRegister dst, XMMRegister nds, Address src)     { Assembler::vaddss(dst, nds, src); }
    1025   void vaddss(XMMRegister dst, XMMRegister nds, AddressLiteral src);
    1026
    1027   void vandpd(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) { Assembler::vandpd(dst, nds, src, vector256); }
    1028   void vandpd(XMMRegister dst, XMMRegister nds, Address src, bool vector256)     { Assembler::vandpd(dst, nds, src, vector256); }
    1029   void vandpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, bool vector256);
    1030
    1031   void vandps(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) { Assembler::vandps(dst, nds, src, vector256); }
    1032   void vandps(XMMRegister dst, XMMRegister nds, Address src, bool vector256)     { Assembler::vandps(dst, nds, src, vector256); }
    1033   void vandps(XMMRegister dst, XMMRegister nds, AddressLiteral src, bool vector256);
    1034
    1035   void vdivsd(XMMRegister dst, XMMRegister nds, XMMRegister src) { Assembler::vdivsd(dst, nds, src); }
    1036   void vdivsd(XMMRegister dst, XMMRegister nds, Address src)     { Assembler::vdivsd(dst, nds, src); }
    1037   void vdivsd(XMMRegister dst, XMMRegister nds, AddressLiteral src);
    1038
    1039   void vdivss(XMMRegister dst, XMMRegister nds, XMMRegister src) { Assembler::vdivss(dst, nds, src); }
    1040   void vdivss(XMMRegister dst, XMMRegister nds, Address src)     { Assembler::vdivss(dst, nds, src); }
    1041   void vdivss(XMMRegister dst, XMMRegister nds, AddressLiteral src);
    1042
    1043   void vmulsd(XMMRegister dst, XMMRegister nds, XMMRegister src) { Assembler::vmulsd(dst, nds, src); }
    1044   void vmulsd(XMMRegister dst, XMMRegister nds, Address src)     { Assembler::vmulsd(dst, nds, src); }
    1045   void vmulsd(XMMRegister dst, XMMRegister nds, AddressLiteral src);
    1046
    1047   void vmulss(XMMRegister dst, XMMRegister nds, XMMRegister src) { Assembler::vmulss(dst, nds, src); }
    1048   void vmulss(XMMRegister dst, XMMRegister nds, Address src)     { Assembler::vmulss(dst, nds, src); }
    1049   void vmulss(XMMRegister dst, XMMRegister nds, AddressLiteral src);
    1050
    1051   void vsubsd(XMMRegister dst, XMMRegister nds, XMMRegister src) { Assembler::vsubsd(dst, nds, src); }
    1052   void vsubsd(XMMRegister dst, XMMRegister nds, Address src)     { Assembler::vsubsd(dst, nds, src); }
    1053   void vsubsd(XMMRegister dst, XMMRegister nds, AddressLiteral src);
    1054
    1055   void vsubss(XMMRegister dst, XMMRegister nds, XMMRegister src) { Assembler::vsubss(dst, nds, src); }
    1056   void vsubss(XMMRegister dst, XMMRegister nds, Address src)     { Assembler::vsubss(dst, nds, src); }
    1057   void vsubss(XMMRegister dst, XMMRegister nds, AddressLiteral src);
    1058
    1059   // AVX Vector instructions
    1060
    1061   void vxorpd(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) { Assembler::vxorpd(dst, nds, src, vector256); }
    1062   void vxorpd(XMMRegister dst, XMMRegister nds, Address src, bool vector256) { Assembler::vxorpd(dst, nds, src, vector256); }
    1063   void vxorpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, bool vector256);
    1064
    1065   void vxorps(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) { Assembler::vxorps(dst, nds, src, vector256); }
    1066   void vxorps(XMMRegister dst, XMMRegister nds, Address src, bool vector256) { Assembler::vxorps(dst, nds, src, vector256); }
    1067   void vxorps(XMMRegister dst, XMMRegister nds, AddressLiteral src, bool vector256);
    1068
    1069   void vpxor(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
    1070     if (UseAVX > 1 || !vector256) // vpxor 256 bit is available only in AVX2
    1071       Assembler::vpxor(dst, nds, src, vector256);
    1072     else
    1073       Assembler::vxorpd(dst, nds, src, vector256);
    1074   }
    1075   void vpxor(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
    1076     if (UseAVX > 1 || !vector256) // vpxor 256 bit is available only in AVX2
    1077       Assembler::vpxor(dst, nds, src, vector256);
    1078     else
    1079       Assembler::vxorpd(dst, nds, src, vector256);
    1080   }
    1081
    1082   // Simple version for AVX2 256bit vectors
    1083   void vpxor(XMMRegister dst, XMMRegister src) { Assembler::vpxor(dst, dst, src, true); }
    1084   void vpxor(XMMRegister dst, Address src) { Assembler::vpxor(dst, dst, src, true); }
    1085
    1086   // Move packed integer values from low 128 bit to hign 128 bit in 256 bit vector.
    1087   void vinserti128h(XMMRegister dst, XMMRegister nds, XMMRegister src) {
    1088     if (UseAVX > 1) // vinserti128h is available only in AVX2
    1089       Assembler::vinserti128h(dst, nds, src);
    1090     else
    1091       Assembler::vinsertf128h(dst, nds, src);
    1092   }
    1093
    1094   // Carry-Less Multiplication Quadword
    1095   void vpclmulldq(XMMRegister dst, XMMRegister nds, XMMRegister src) {
    1096     // 0x00 - multiply lower 64 bits [0:63]
    1097     Assembler::vpclmulqdq(dst, nds, src, 0x00);
    1098   }
    1099   void vpclmulhdq(XMMRegister dst, XMMRegister nds, XMMRegister src) {
    1100     // 0x11 - multiply upper 64 bits [64:127]
    1101     Assembler::vpclmulqdq(dst, nds, src, 0x11);
    1102   }
    1103
    1104   // Data
    1105
    1106   void cmov32( Condition cc, Register dst, Address  src);
    1107   void cmov32( Condition cc, Register dst, Register src);
    1108
    1109   void cmov(   Condition cc, Register dst, Register src) { cmovptr(cc, dst, src); }
    1110
    1111   void cmovptr(Condition cc, Register dst, Address  src) { LP64_ONLY(cmovq(cc, dst, src)) NOT_LP64(cmov32(cc, dst, src)); }
    1112   void cmovptr(Condition cc, Register dst, Register src) { LP64_ONLY(cmovq(cc, dst, src)) NOT_LP64(cmov32(cc, dst, src)); }
    1113
    1114   void movoop(Register dst, jobject obj);
    1115   void movoop(Address dst, jobject obj);
    1116
    1117   void mov_metadata(Register dst, Metadata* obj);
    1118   void mov_metadata(Address dst, Metadata* obj);
    1119
    1120   void movptr(ArrayAddress dst, Register src);
    1121   // can this do an lea?
    1122   void movptr(Register dst, ArrayAddress src);
    1123
    1124   void movptr(Register dst, Address src);
    1125
    1126 #ifdef _LP64
    1127   void movptr(Register dst, AddressLiteral src, Register scratch=rscratch1);
    1128 #else
    1129   void movptr(Register dst, AddressLiteral src, Register scratch=noreg); // Scratch reg is ignored in 32-bit
    1130 #endif
    1131
    1132   void movptr(Register dst, intptr_t src);
    1133   void movptr(Register dst, Register src);
    1134   void movptr(Address dst, intptr_t src);
    1135
    1136   void movptr(Address dst, Register src);
    1137
    1138   void movptr(Register dst, RegisterOrConstant src) {
    1139     if (src.is_constant()) movptr(dst, src.as_constant());
    1140     else                   movptr(dst, src.as_register());
    1141   }
    1142
    1143 #ifdef _LP64
    1144   // Generally the next two are only used for moving NULL
    1145   // Although there are situations in initializing the mark word where
    1146   // they could be used. They are dangerous.
    1147
    1148   // They only exist on LP64 so that int32_t and intptr_t are not the same
    1149   // and we have ambiguous declarations.
    1150
    1151   void movptr(Address dst, int32_t imm32);
    1152   void movptr(Register dst, int32_t imm32);
    1153 #endif // _LP64
    1154
    1155   // to avoid hiding movl
    1156   void mov32(AddressLiteral dst, Register src);
    1157   void mov32(Register dst, AddressLiteral src);
    1158
    1159   // to avoid hiding movb
    1160   void movbyte(ArrayAddress dst, int src);
    1161
    1162   // Import other mov() methods from the parent class or else
    1163   // they will be hidden by the following overriding declaration.
    1164   using Assembler::movdl;
    1165   using Assembler::movq;
    1166   void movdl(XMMRegister dst, AddressLiteral src);
    1167   void movq(XMMRegister dst, AddressLiteral src);
    1168
    1169   // Can push value or effective address
    1170   void pushptr(AddressLiteral src);
    1171
    1172   void pushptr(Address src) { LP64_ONLY(pushq(src)) NOT_LP64(pushl(src)); }
    1173   void popptr(Address src) { LP64_ONLY(popq(src)) NOT_LP64(popl(src)); }
    1174
    1175   void pushoop(jobject obj);
    1176   void pushklass(Metadata* obj);
    1177
    1178   // sign extend as need a l to ptr sized element
    1179   void movl2ptr(Register dst, Address src) { LP64_ONLY(movslq(dst, src)) NOT_LP64(movl(dst, src)); }
    1180   void movl2ptr(Register dst, Register src) { LP64_ONLY(movslq(dst, src)) NOT_LP64(if (dst != src) movl(dst, src)); }
    1181
    1182   // C2 compiled method's prolog code.
    1183   void verified_entry(int framesize, int stack_bang_size, bool fp_mode_24b);
    1184
    1185   // clear memory of size 'cnt' qwords, starting at 'base'.
    1186   void clear_mem(Register base, Register cnt, Register rtmp);
    1187
    1188   // IndexOf strings.
    1189   // Small strings are loaded through stack if they cross page boundary.
    1190   void string_indexof(Register str1, Register str2,
    1191                       Register cnt1, Register cnt2,
    1192                       int int_cnt2,  Register result,
    1193                       XMMRegister vec, Register tmp);
    1194
    1195   // IndexOf for constant substrings with size >= 8 elements
    1196   // which don't need to be loaded through stack.
    1197   void string_indexofC8(Register str1, Register str2,
    1198                       Register cnt1, Register cnt2,
    1199                       int int_cnt2,  Register result,
    1200                       XMMRegister vec, Register tmp);
    1201
    1202     // Smallest code: we don't need to load through stack,
    1203     // check string tail.
    1204
    1205   // Compare strings.
    1206   void string_compare(Register str1, Register str2,
    1207                       Register cnt1, Register cnt2, Register result,
    1208                       XMMRegister vec1);
    1209
    1210   // Compare char[] arrays.
    1211   void char_arrays_equals(bool is_array_equ, Register ary1, Register ary2,
    1212                           Register limit, Register result, Register chr,
    1213                           XMMRegister vec1, XMMRegister vec2);
    1214
    1215   // Fill primitive arrays
    1216   void generate_fill(BasicType t, bool aligned,
    1217                      Register to, Register value, Register count,
    1218                      Register rtmp, XMMRegister xtmp);
    1219
    1220   void encode_iso_array(Register src, Register dst, Register len,
    1221                         XMMRegister tmp1, XMMRegister tmp2, XMMRegister tmp3,
    1222                         XMMRegister tmp4, Register tmp5, Register result);
    1223
    1224 #ifdef _LP64
    1225   void add2_with_carry(Register dest_hi, Register dest_lo, Register src1, Register src2);
    1226   void multiply_64_x_64_loop(Register x, Register xstart, Register x_xstart,
    1227                              Register y, Register y_idx, Register z,
    1228                              Register carry, Register product,
    1229                              Register idx, Register kdx);
    1230   void multiply_add_128_x_128(Register x_xstart, Register y, Register z,
    1231                               Register yz_idx, Register idx,
    1232                               Register carry, Register product, int offset);
    1233   void multiply_128_x_128_bmi2_loop(Register y, Register z,
    1234                                     Register carry, Register carry2,
    1235                                     Register idx, Register jdx,
    1236                                     Register yz_idx1, Register yz_idx2,
    1237                                     Register tmp, Register tmp3, Register tmp4);
    1238   void multiply_128_x_128_loop(Register x_xstart, Register y, Register z,
    1239                                Register yz_idx, Register idx, Register jdx,
    1240                                Register carry, Register product,
    1241                                Register carry2);
    1242   void multiply_to_len(Register x, Register xlen, Register y, Register ylen, Register z, Register zlen,
    1243                        Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5);
    1244 #endif
    1245
    1246   // CRC32 code for java.util.zip.CRC32::updateBytes() instrinsic.
    1247   void update_byte_crc32(Register crc, Register val, Register table);
    1248   void kernel_crc32(Register crc, Register buf, Register len, Register table, Register tmp);
    1249   // Fold 128-bit data chunk
    1250   void fold_128bit_crc32(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, Register buf, int offset);
    1251   void fold_128bit_crc32(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, XMMRegister xbuf);
    1252   // Fold 8-bit data
    1253   void fold_8bit_crc32(Register crc, Register table, Register tmp);
    1254   void fold_8bit_crc32(XMMRegister crc, Register table, XMMRegister xtmp, Register tmp);
    1255
    1256 #undef VIRTUAL
    1257
    1258 };
    1259
    1260 /**
    1261  * class SkipIfEqual:
    1262  *
    1263  * Instantiating this class will result in assembly code being output that will
    1264  * jump around any code emitted between the creation of the instance and it's
    1265  * automatic destruction at the end of a scope block, depending on the value of
    1266  * the flag passed to the constructor, which will be checked at run-time.
    1267  */
    1268 class SkipIfEqual {
    1269  private:
    1270   MacroAssembler* _masm;
    1271   Label _label;
    1272
    1273  public:
    1274    SkipIfEqual(MacroAssembler*, const bool* flag_addr, bool value);
    1275    ~SkipIfEqual();
    1276 };
    1277
    1278 #endif // CPU_X86_VM_MACROASSEMBLER_X86_HPP