       1 /*
       2  * Copyright (c) 1997, 2014, Oracle and/or its affiliates. All rights reserved.
       3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
       4  *
       5  * This code is free software; you can redistribute it and/or modify it
       6  * under the terms of the GNU General Public License version 2 only, as
       7  * published by the Free Software Foundation.
       8  *
       9  * This code is distributed in the hope that it will be useful, but WITHOUT
      10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
      11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
      12  * version 2 for more details (a copy is included in the LICENSE file that
      13  * accompanied this code).
      14  *
      15  * You should have received a copy of the GNU General Public License version
      16  * 2 along with this work; if not, write to the Free Software Foundation,
      17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
      18  *
      19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
      20  * or visit www.oracle.com if you need additional information or have any
      21  * questions.
      22  *
      23  */
      24
      25 #include "precompiled.hpp"
      26 #include "asm/assembler.hpp"
      27 #include "asm/assembler.inline.hpp"
      28 #include "gc_interface/collectedHeap.inline.hpp"
      29 #include "interpreter/interpreter.hpp"
      30 #include "memory/cardTableModRefBS.hpp"
      31 #include "memory/resourceArea.hpp"
      32 #include "prims/methodHandles.hpp"
      33 #include "runtime/biasedLocking.hpp"
      34 #include "runtime/interfaceSupport.hpp"
      35 #include "runtime/objectMonitor.hpp"
      36 #include "runtime/os.hpp"
      37 #include "runtime/sharedRuntime.hpp"
      38 #include "runtime/stubRoutines.hpp"
      39 #include "utilities/macros.hpp"
      40 #if INCLUDE_ALL_GCS
      41 #include "gc_implementation/g1/g1CollectedHeap.inline.hpp"
      42 #include "gc_implementation/g1/g1SATBCardTableModRefBS.hpp"
      43 #include "gc_implementation/g1/heapRegion.hpp"
      44 #endif // INCLUDE_ALL_GCS
      45
      46 #ifdef PRODUCT
      47 #define BLOCK_COMMENT(str) /* nothing */
      48 #define STOP(error) stop(error)
      49 #else
      50 #define BLOCK_COMMENT(str) block_comment(str)
      51 #define STOP(error) block_comment(error); stop(error)
      52 #endif
      53
      54 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
      55 // Implementation of AddressLiteral
      56
      57 AddressLiteral::AddressLiteral(address target, relocInfo::relocType rtype) {
      58   _is_lval = false;
      59   _target = target;
      60   switch (rtype) {
      61   case relocInfo::oop_type:
      62   case relocInfo::metadata_type:
      63     // Oops are a special case. Normally they would be their own section
      64     // but in cases like icBuffer they are literals in the code stream that
      65     // we don't have a section for. We use none so that we get a literal address
      66     // which is always patchable.
      67     break;
      68   case relocInfo::external_word_type:
      69     _rspec = external_word_Relocation::spec(target);
      70     break;
      71   case relocInfo::internal_word_type:
      72     _rspec = internal_word_Relocation::spec(target);
      73     break;
      74   case relocInfo::opt_virtual_call_type:
      75     _rspec = opt_virtual_call_Relocation::spec();
      76     break;
      77   case relocInfo::static_call_type:
      78     _rspec = static_call_Relocation::spec();
      79     break;
      80   case relocInfo::runtime_call_type:
      81     _rspec = runtime_call_Relocation::spec();
      82     break;
      83   case relocInfo::poll_type:
      84   case relocInfo::poll_return_type:
      85     _rspec = Relocation::spec_simple(rtype);
      86     break;
      87   case relocInfo::none:
      88     break;
      89   default:
      90     ShouldNotReachHere();
      91     break;
      92   }
      93 }
      94
      95 // Implementation of Address
      96
      97 #ifdef _LP64
      98
      99 Address Address::make_array(ArrayAddress adr) {
     100   // Not implementable on 64bit machines
     101   // Should have been handled higher up the call chain.
     102   ShouldNotReachHere();
     103   return Address();
     104 }
     105
     106 // exceedingly dangerous constructor
     107 Address::Address(int disp, address loc, relocInfo::relocType rtype) {
     108   _base  = noreg;
     109   _index = noreg;
     110   _scale = no_scale;
     111   _disp  = disp;
     112   switch (rtype) {
     113     case relocInfo::external_word_type:
     114       _rspec = external_word_Relocation::spec(loc);
     115       break;
     116     case relocInfo::internal_word_type:
     117       _rspec = internal_word_Relocation::spec(loc);
     118       break;
     119     case relocInfo::runtime_call_type:
     120       // HMM
     121       _rspec = runtime_call_Relocation::spec();
     122       break;
     123     case relocInfo::poll_type:
     124     case relocInfo::poll_return_type:
     125       _rspec = Relocation::spec_simple(rtype);
     126       break;
     127     case relocInfo::none:
     128       break;
     129     default:
     130       ShouldNotReachHere();
     131   }
     132 }
     133 #else // LP64
     134
     135 Address Address::make_array(ArrayAddress adr) {
     136   AddressLiteral base = adr.base();
     137   Address index = adr.index();
     138   assert(index._disp == 0, "must not have disp"); // maybe it can?
     139   Address array(index._base, index._index, index._scale, (intptr_t) base.target());
     140   array._rspec = base._rspec;
     141   return array;
     142 }
     143
     144 // exceedingly dangerous constructor
     145 Address::Address(address loc, RelocationHolder spec) {
     146   _base  = noreg;
     147   _index = noreg;
     148   _scale = no_scale;
     149   _disp  = (intptr_t) loc;
     150   _rspec = spec;
     151 }
     152
     153 #endif // _LP64
     154
     155
     156
     157 // Convert the raw encoding form into the form expected by the constructor for
     158 // Address.  An index of 4 (rsp) corresponds to having no index, so convert
     159 // that to noreg for the Address constructor.
     160 Address Address::make_raw(int base, int index, int scale, int disp, relocInfo::relocType disp_reloc) {
     161   RelocationHolder rspec;
     162   if (disp_reloc != relocInfo::none) {
     163     rspec = Relocation::spec_simple(disp_reloc);
     164   }
     165   bool valid_index = index != rsp->encoding();
     166   if (valid_index) {
     167     Address madr(as_Register(base), as_Register(index), (Address::ScaleFactor)scale, in_ByteSize(disp));
     168     madr._rspec = rspec;
     169     return madr;
     170   } else {
     171     Address madr(as_Register(base), noreg, Address::no_scale, in_ByteSize(disp));
     172     madr._rspec = rspec;
     173     return madr;
     174   }
     175 }
     176
     177 // Implementation of Assembler
     178
     179 int AbstractAssembler::code_fill_byte() {
     180   return (u_char)'\xF4'; // hlt
     181 }
     182
     183 // make this go away someday
     184 void Assembler::emit_data(jint data, relocInfo::relocType rtype, int format) {
     185   if (rtype == relocInfo::none)
     186         emit_int32(data);
     187   else  emit_data(data, Relocation::spec_simple(rtype), format);
     188 }
     189
     190 void Assembler::emit_data(jint data, RelocationHolder const& rspec, int format) {
     191   assert(imm_operand == 0, "default format must be immediate in this file");
     192   assert(inst_mark() != NULL, "must be inside InstructionMark");
     193   if (rspec.type() !=  relocInfo::none) {
     194     #ifdef ASSERT
     195       check_relocation(rspec, format);
     196     #endif
     197     // Do not use AbstractAssembler::relocate, which is not intended for
     198     // embedded words.  Instead, relocate to the enclosing instruction.
     199
     200     // hack. call32 is too wide for mask so use disp32
     201     if (format == call32_operand)
     202       code_section()->relocate(inst_mark(), rspec, disp32_operand);
     203     else
     204       code_section()->relocate(inst_mark(), rspec, format);
     205   }
     206   emit_int32(data);
     207 }
     208
     209 static int encode(Register r) {
     210   int enc = r->encoding();
     211   if (enc >= 8) {
     212     enc -= 8;
     213   }
     214   return enc;
     215 }
     216
     217 void Assembler::emit_arith_b(int op1, int op2, Register dst, int imm8) {
     218   assert(dst->has_byte_register(), "must have byte register");
     219   assert(isByte(op1) && isByte(op2), "wrong opcode");
     220   assert(isByte(imm8), "not a byte");
     221   assert((op1 & 0x01) == 0, "should be 8bit operation");
     222   emit_int8(op1);
     223   emit_int8(op2 | encode(dst));
     224   emit_int8(imm8);
     225 }
     226
     227
     228 void Assembler::emit_arith(int op1, int op2, Register dst, int32_t imm32) {
     229   assert(isByte(op1) && isByte(op2), "wrong opcode");
     230   assert((op1 & 0x01) == 1, "should be 32bit operation");
     231   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
     232   if (is8bit(imm32)) {
     233     emit_int8(op1 | 0x02); // set sign bit
     234     emit_int8(op2 | encode(dst));
     235     emit_int8(imm32 & 0xFF);
     236   } else {
     237     emit_int8(op1);
     238     emit_int8(op2 | encode(dst));
     239     emit_int32(imm32);
     240   }
     241 }
     242
     243 // Force generation of a 4 byte immediate value even if it fits into 8bit
     244 void Assembler::emit_arith_imm32(int op1, int op2, Register dst, int32_t imm32) {
     245   assert(isByte(op1) && isByte(op2), "wrong opcode");
     246   assert((op1 & 0x01) == 1, "should be 32bit operation");
     247   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
     248   emit_int8(op1);
     249   emit_int8(op2 | encode(dst));
     250   emit_int32(imm32);
     251 }
     252
     253 // immediate-to-memory forms
     254 void Assembler::emit_arith_operand(int op1, Register rm, Address adr, int32_t imm32) {
     255   assert((op1 & 0x01) == 1, "should be 32bit operation");
     256   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
     257   if (is8bit(imm32)) {
     258     emit_int8(op1 | 0x02); // set sign bit
     259     emit_operand(rm, adr, 1);
     260     emit_int8(imm32 & 0xFF);
     261   } else {
     262     emit_int8(op1);
     263     emit_operand(rm, adr, 4);
     264     emit_int32(imm32);
     265   }
     266 }
     267
     268
     269 void Assembler::emit_arith(int op1, int op2, Register dst, Register src) {
     270   assert(isByte(op1) && isByte(op2), "wrong opcode");
     271   emit_int8(op1);
     272   emit_int8(op2 | encode(dst) << 3 | encode(src));
     273 }
     274
     275
     276 void Assembler::emit_operand(Register reg, Register base, Register index,
     277                              Address::ScaleFactor scale, int disp,
     278                              RelocationHolder const& rspec,
     279                              int rip_relative_correction) {
     280   relocInfo::relocType rtype = (relocInfo::relocType) rspec.type();
     281
     282   // Encode the registers as needed in the fields they are used in
     283
     284   int regenc = encode(reg) << 3;
     285   int indexenc = index->is_valid() ? encode(index) << 3 : 0;
     286   int baseenc = base->is_valid() ? encode(base) : 0;
     287
     288   if (base->is_valid()) {
     289     if (index->is_valid()) {
     290       assert(scale != Address::no_scale, "inconsistent address");
     291       // [base + index*scale + disp]
     292       if (disp == 0 && rtype == relocInfo::none  &&
     293           base != rbp LP64_ONLY(&& base != r13)) {
     294         // [base + index*scale]
     295         // [00 reg 100][ss index base]
     296         assert(index != rsp, "illegal addressing mode");
     297         emit_int8(0x04 | regenc);
     298         emit_int8(scale << 6 | indexenc | baseenc);
     299       } else if (is8bit(disp) && rtype == relocInfo::none) {
     300         // [base + index*scale + imm8]
     301         // [01 reg 100][ss index base] imm8
     302         assert(index != rsp, "illegal addressing mode");
     303         emit_int8(0x44 | regenc);
     304         emit_int8(scale << 6 | indexenc | baseenc);
     305         emit_int8(disp & 0xFF);
     306       } else {
     307         // [base + index*scale + disp32]
     308         // [10 reg 100][ss index base] disp32
     309         assert(index != rsp, "illegal addressing mode");
     310         emit_int8(0x84 | regenc);
     311         emit_int8(scale << 6 | indexenc | baseenc);
     312         emit_data(disp, rspec, disp32_operand);
     313       }
     314     } else if (base == rsp LP64_ONLY(|| base == r12)) {
     315       // [rsp + disp]
     316       if (disp == 0 && rtype == relocInfo::none) {
     317         // [rsp]
     318         // [00 reg 100][00 100 100]
     319         emit_int8(0x04 | regenc);
     320         emit_int8(0x24);
     321       } else if (is8bit(disp) && rtype == relocInfo::none) {
     322         // [rsp + imm8]
     323         // [01 reg 100][00 100 100] disp8
     324         emit_int8(0x44 | regenc);
     325         emit_int8(0x24);
     326         emit_int8(disp & 0xFF);
     327       } else {
     328         // [rsp + imm32]
     329         // [10 reg 100][00 100 100] disp32
     330         emit_int8(0x84 | regenc);
     331         emit_int8(0x24);
     332         emit_data(disp, rspec, disp32_operand);
     333       }
     334     } else {
     335       // [base + disp]
     336       assert(base != rsp LP64_ONLY(&& base != r12), "illegal addressing mode");
     337       if (disp == 0 && rtype == relocInfo::none &&
     338           base != rbp LP64_ONLY(&& base != r13)) {
     339         // [base]
     340         // [00 reg base]
     341         emit_int8(0x00 | regenc | baseenc);
     342       } else if (is8bit(disp) && rtype == relocInfo::none) {
     343         // [base + disp8]
     344         // [01 reg base] disp8
     345         emit_int8(0x40 | regenc | baseenc);
     346         emit_int8(disp & 0xFF);
     347       } else {
     348         // [base + disp32]
     349         // [10 reg base] disp32
     350         emit_int8(0x80 | regenc | baseenc);
     351         emit_data(disp, rspec, disp32_operand);
     352       }
     353     }
     354   } else {
     355     if (index->is_valid()) {
     356       assert(scale != Address::no_scale, "inconsistent address");
     357       // [index*scale + disp]
     358       // [00 reg 100][ss index 101] disp32
     359       assert(index != rsp, "illegal addressing mode");
     360       emit_int8(0x04 | regenc);
     361       emit_int8(scale << 6 | indexenc | 0x05);
     362       emit_data(disp, rspec, disp32_operand);
     363     } else if (rtype != relocInfo::none ) {
     364       // [disp] (64bit) RIP-RELATIVE (32bit) abs
     365       // [00 000 101] disp32
     366
     367       emit_int8(0x05 | regenc);
     368       // Note that the RIP-rel. correction applies to the generated
     369       // disp field, but _not_ to the target address in the rspec.
     370
     371       // disp was created by converting the target address minus the pc
     372       // at the start of the instruction. That needs more correction here.
     373       // intptr_t disp = target - next_ip;
     374       assert(inst_mark() != NULL, "must be inside InstructionMark");
     375       address next_ip = pc() + sizeof(int32_t) + rip_relative_correction;
     376       int64_t adjusted = disp;
     377       // Do rip-rel adjustment for 64bit
     378       LP64_ONLY(adjusted -=  (next_ip - inst_mark()));
     379       assert(is_simm32(adjusted),
     380              "must be 32bit offset (RIP relative address)");
     381       emit_data((int32_t) adjusted, rspec, disp32_operand);
     382
     383     } else {
     384       // 32bit never did this, did everything as the rip-rel/disp code above
     385       // [disp] ABSOLUTE
     386       // [00 reg 100][00 100 101] disp32
     387       emit_int8(0x04 | regenc);
     388       emit_int8(0x25);
     389       emit_data(disp, rspec, disp32_operand);
     390     }
     391   }
     392 }
     393
     394 void Assembler::emit_operand(XMMRegister reg, Register base, Register index,
     395                              Address::ScaleFactor scale, int disp,
     396                              RelocationHolder const& rspec) {
     397   emit_operand((Register)reg, base, index, scale, disp, rspec);
     398 }
     399
     400 // Secret local extension to Assembler::WhichOperand:
     401 #define end_pc_operand (_WhichOperand_limit)
     402
     403 address Assembler::locate_operand(address inst, WhichOperand which) {
     404   // Decode the given instruction, and return the address of
     405   // an embedded 32-bit operand word.
     406
     407   // If "which" is disp32_operand, selects the displacement portion
     408   // of an effective address specifier.
     409   // If "which" is imm64_operand, selects the trailing immediate constant.
     410   // If "which" is call32_operand, selects the displacement of a call or jump.
     411   // Caller is responsible for ensuring that there is such an operand,
     412   // and that it is 32/64 bits wide.
     413
     414   // If "which" is end_pc_operand, find the end of the instruction.
     415
     416   address ip = inst;
     417   bool is_64bit = false;
     418
     419   debug_only(bool has_disp32 = false);
     420   int tail_size = 0; // other random bytes (#32, #16, etc.) at end of insn
     421
     422   again_after_prefix:
     423   switch (0xFF & *ip++) {
     424
     425   // These convenience macros generate groups of "case" labels for the switch.
     426 #define REP4(x) (x)+0: case (x)+1: case (x)+2: case (x)+3
     427 #define REP8(x) (x)+0: case (x)+1: case (x)+2: case (x)+3: \
     428              case (x)+4: case (x)+5: case (x)+6: case (x)+7
     429 #define REP16(x) REP8((x)+0): \
     430               case REP8((x)+8)
     431
     432   case CS_segment:
     433   case SS_segment:
     434   case DS_segment:
     435   case ES_segment:
     436   case FS_segment:
     437   case GS_segment:
     438     // Seems dubious
     439     LP64_ONLY(assert(false, "shouldn't have that prefix"));
     440     assert(ip == inst+1, "only one prefix allowed");
     441     goto again_after_prefix;
     442
     443   case 0x67:
     444   case REX:
     445   case REX_B:
     446   case REX_X:
     447   case REX_XB:
     448   case REX_R:
     449   case REX_RB:
     450   case REX_RX:
     451   case REX_RXB:
     452     NOT_LP64(assert(false, "64bit prefixes"));
     453     goto again_after_prefix;
     454
     455   case REX_W:
     456   case REX_WB:
     457   case REX_WX:
     458   case REX_WXB:
     459   case REX_WR:
     460   case REX_WRB:
     461   case REX_WRX:
     462   case REX_WRXB:
     463     NOT_LP64(assert(false, "64bit prefixes"));
     464     is_64bit = true;
     465     goto again_after_prefix;
     466
     467   case 0xFF: // pushq a; decl a; incl a; call a; jmp a
     468   case 0x88: // movb a, r
     469   case 0x89: // movl a, r
     470   case 0x8A: // movb r, a
     471   case 0x8B: // movl r, a
     472   case 0x8F: // popl a
     473     debug_only(has_disp32 = true);
     474     break;
     475
     476   case 0x68: // pushq #32
     477     if (which == end_pc_operand) {
     478       return ip + 4;
     479     }
     480     assert(which == imm_operand && !is_64bit, "pushl has no disp32 or 64bit immediate");
     481     return ip;                  // not produced by emit_operand
     482
     483   case 0x66: // movw ... (size prefix)
     484     again_after_size_prefix2:
     485     switch (0xFF & *ip++) {
     486     case REX:
     487     case REX_B:
     488     case REX_X:
     489     case REX_XB:
     490     case REX_R:
     491     case REX_RB:
     492     case REX_RX:
     493     case REX_RXB:
     494     case REX_W:
     495     case REX_WB:
     496     case REX_WX:
     497     case REX_WXB:
     498     case REX_WR:
     499     case REX_WRB:
     500     case REX_WRX:
     501     case REX_WRXB:
     502       NOT_LP64(assert(false, "64bit prefix found"));
     503       goto again_after_size_prefix2;
     504     case 0x8B: // movw r, a
     505     case 0x89: // movw a, r
     506       debug_only(has_disp32 = true);
     507       break;
     508     case 0xC7: // movw a, #16
     509       debug_only(has_disp32 = true);
     510       tail_size = 2;  // the imm16
     511       break;
     512     case 0x0F: // several SSE/SSE2 variants
     513       ip--;    // reparse the 0x0F
     514       goto again_after_prefix;
     515     default:
     516       ShouldNotReachHere();
     517     }
     518     break;
     519
     520   case REP8(0xB8): // movl/q r, #32/#64(oop?)
     521     if (which == end_pc_operand)  return ip + (is_64bit ? 8 : 4);
     522     // these asserts are somewhat nonsensical
     523 #ifndef _LP64
     524     assert(which == imm_operand || which == disp32_operand,
     525            err_msg("which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, p2i(ip)));
     526 #else
     527     assert((which == call32_operand || which == imm_operand) && is_64bit ||
     528            which == narrow_oop_operand && !is_64bit,
     529            err_msg("which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, p2i(ip)));
     530 #endif // _LP64
     531     return ip;
     532
     533   case 0x69: // imul r, a, #32
     534   case 0xC7: // movl a, #32(oop?)
     535     tail_size = 4;
     536     debug_only(has_disp32 = true); // has both kinds of operands!
     537     break;
     538
     539   case 0x0F: // movx..., etc.
     540     switch (0xFF & *ip++) {
     541     case 0x3A: // pcmpestri
     542       tail_size = 1;
     543     case 0x38: // ptest, pmovzxbw
     544       ip++; // skip opcode
     545       debug_only(has_disp32 = true); // has both kinds of operands!
     546       break;
     547
     548     case 0x70: // pshufd r, r/a, #8
     549       debug_only(has_disp32 = true); // has both kinds of operands!
     550     case 0x73: // psrldq r, #8
     551       tail_size = 1;
     552       break;
     553
     554     case 0x12: // movlps
     555     case 0x28: // movaps
     556     case 0x2E: // ucomiss
     557     case 0x2F: // comiss
     558     case 0x54: // andps
     559     case 0x55: // andnps
     560     case 0x56: // orps
     561     case 0x57: // xorps
     562     case 0x6E: // movd
     563     case 0x7E: // movd
     564     case 0xAE: // ldmxcsr, stmxcsr, fxrstor, fxsave, clflush
     565       debug_only(has_disp32 = true);
     566       break;
     567
     568     case 0xAD: // shrd r, a, %cl
     569     case 0xAF: // imul r, a
     570     case 0xBE: // movsbl r, a (movsxb)
     571     case 0xBF: // movswl r, a (movsxw)
     572     case 0xB6: // movzbl r, a (movzxb)
     573     case 0xB7: // movzwl r, a (movzxw)
     574     case REP16(0x40): // cmovl cc, r, a
     575     case 0xB0: // cmpxchgb
     576     case 0xB1: // cmpxchg
     577     case 0xC1: // xaddl
     578     case 0xC7: // cmpxchg8
     579     case REP16(0x90): // setcc a
     580       debug_only(has_disp32 = true);
     581       // fall out of the switch to decode the address
     582       break;
     583
     584     case 0xC4: // pinsrw r, a, #8
     585       debug_only(has_disp32 = true);
     586     case 0xC5: // pextrw r, r, #8
     587       tail_size = 1;  // the imm8
     588       break;
     589
     590     case 0xAC: // shrd r, a, #8
     591       debug_only(has_disp32 = true);
     592       tail_size = 1;  // the imm8
     593       break;
     594
     595     case REP16(0x80): // jcc rdisp32
     596       if (which == end_pc_operand)  return ip + 4;
     597       assert(which == call32_operand, "jcc has no disp32 or imm");
     598       return ip;
     599     default:
     600       ShouldNotReachHere();
     601     }
     602     break;
     603
     604   case 0x81: // addl a, #32; addl r, #32
     605     // also: orl, adcl, sbbl, andl, subl, xorl, cmpl
     606     // on 32bit in the case of cmpl, the imm might be an oop
     607     tail_size = 4;
     608     debug_only(has_disp32 = true); // has both kinds of operands!
     609     break;
     610
     611   case 0x83: // addl a, #8; addl r, #8
     612     // also: orl, adcl, sbbl, andl, subl, xorl, cmpl
     613     debug_only(has_disp32 = true); // has both kinds of operands!
     614     tail_size = 1;
     615     break;
     616
     617   case 0x9B:
     618     switch (0xFF & *ip++) {
     619     case 0xD9: // fnstcw a
     620       debug_only(has_disp32 = true);
     621       break;
     622     default:
     623       ShouldNotReachHere();
     624     }
     625     break;
     626
     627   case REP4(0x00): // addb a, r; addl a, r; addb r, a; addl r, a
     628   case REP4(0x10): // adc...
     629   case REP4(0x20): // and...
     630   case REP4(0x30): // xor...
     631   case REP4(0x08): // or...
     632   case REP4(0x18): // sbb...
     633   case REP4(0x28): // sub...
     634   case 0xF7: // mull a
     635   case 0x8D: // lea r, a
     636   case 0x87: // xchg r, a
     637   case REP4(0x38): // cmp...
     638   case 0x85: // test r, a
     639     debug_only(has_disp32 = true); // has both kinds of operands!
     640     break;
     641
     642   case 0xC1: // sal a, #8; sar a, #8; shl a, #8; shr a, #8
     643   case 0xC6: // movb a, #8
     644   case 0x80: // cmpb a, #8
     645   case 0x6B: // imul r, a, #8
     646     debug_only(has_disp32 = true); // has both kinds of operands!
     647     tail_size = 1; // the imm8
     648     break;
     649
     650   case 0xC4: // VEX_3bytes
     651   case 0xC5: // VEX_2bytes
     652     assert((UseAVX > 0), "shouldn't have VEX prefix");
     653     assert(ip == inst+1, "no prefixes allowed");
     654     // C4 and C5 are also used as opcodes for PINSRW and PEXTRW instructions
     655     // but they have prefix 0x0F and processed when 0x0F processed above.
     656     //
     657     // In 32-bit mode the VEX first byte C4 and C5 alias onto LDS and LES
     658     // instructions (these instructions are not supported in 64-bit mode).
     659     // To distinguish them bits [7:6] are set in the VEX second byte since
     660     // ModRM byte can not be of the form 11xxxxxx in 32-bit mode. To set
     661     // those VEX bits REX and vvvv bits are inverted.
     662     //
     663     // Fortunately C2 doesn't generate these instructions so we don't need
     664     // to check for them in product version.
     665
     666     // Check second byte
     667     NOT_LP64(assert((0xC0 & *ip) == 0xC0, "shouldn't have LDS and LES instructions"));
     668
     669     // First byte
     670     if ((0xFF & *inst) == VEX_3bytes) {
     671       ip++; // third byte
     672       is_64bit = ((VEX_W & *ip) == VEX_W);
     673     }
     674     ip++; // opcode
     675     // To find the end of instruction (which == end_pc_operand).
     676     switch (0xFF & *ip) {
     677     case 0x61: // pcmpestri r, r/a, #8
     678     case 0x70: // pshufd r, r/a, #8
     679     case 0x73: // psrldq r, #8
     680       tail_size = 1;  // the imm8
     681       break;
     682     default:
     683       break;
     684     }
     685     ip++; // skip opcode
     686     debug_only(has_disp32 = true); // has both kinds of operands!
     687     break;
     688
     689   case 0xD1: // sal a, 1; sar a, 1; shl a, 1; shr a, 1
     690   case 0xD3: // sal a, %cl; sar a, %cl; shl a, %cl; shr a, %cl
     691   case 0xD9: // fld_s a; fst_s a; fstp_s a; fldcw a
     692   case 0xDD: // fld_d a; fst_d a; fstp_d a
     693   case 0xDB: // fild_s a; fistp_s a; fld_x a; fstp_x a
     694   case 0xDF: // fild_d a; fistp_d a
     695   case 0xD8: // fadd_s a; fsubr_s a; fmul_s a; fdivr_s a; fcomp_s a
     696   case 0xDC: // fadd_d a; fsubr_d a; fmul_d a; fdivr_d a; fcomp_d a
     697   case 0xDE: // faddp_d a; fsubrp_d a; fmulp_d a; fdivrp_d a; fcompp_d a
     698     debug_only(has_disp32 = true);
     699     break;
     700
     701   case 0xE8: // call rdisp32
     702   case 0xE9: // jmp  rdisp32
     703     if (which == end_pc_operand)  return ip + 4;
     704     assert(which == call32_operand, "call has no disp32 or imm");
     705     return ip;
     706
     707   case 0xF0:                    // Lock
     708     assert(os::is_MP(), "only on MP");
     709     goto again_after_prefix;
     710
     711   case 0xF3:                    // For SSE
     712   case 0xF2:                    // For SSE2
     713     switch (0xFF & *ip++) {
     714     case REX:
     715     case REX_B:
     716     case REX_X:
     717     case REX_XB:
     718     case REX_R:
     719     case REX_RB:
     720     case REX_RX:
     721     case REX_RXB:
     722     case REX_W:
     723     case REX_WB:
     724     case REX_WX:
     725     case REX_WXB:
     726     case REX_WR:
     727     case REX_WRB:
     728     case REX_WRX:
     729     case REX_WRXB:
     730       NOT_LP64(assert(false, "found 64bit prefix"));
     731       ip++;
     732     default:
     733       ip++;
     734     }
     735     debug_only(has_disp32 = true); // has both kinds of operands!
     736     break;
     737
     738   default:
     739     ShouldNotReachHere();
     740
     741 #undef REP8
     742 #undef REP16
     743   }
     744
     745   assert(which != call32_operand, "instruction is not a call, jmp, or jcc");
     746 #ifdef _LP64
     747   assert(which != imm_operand, "instruction is not a movq reg, imm64");
     748 #else
     749   // assert(which != imm_operand || has_imm32, "instruction has no imm32 field");
     750   assert(which != imm_operand || has_disp32, "instruction has no imm32 field");
     751 #endif // LP64
     752   assert(which != disp32_operand || has_disp32, "instruction has no disp32 field");
     753
     754   // parse the output of emit_operand
     755   int op2 = 0xFF & *ip++;
     756   int base = op2 & 0x07;
     757   int op3 = -1;
     758   const int b100 = 4;
     759   const int b101 = 5;
     760   if (base == b100 && (op2 >> 6) != 3) {
     761     op3 = 0xFF & *ip++;
     762     base = op3 & 0x07;   // refetch the base
     763   }
     764   // now ip points at the disp (if any)
     765
     766   switch (op2 >> 6) {
     767   case 0:
     768     // [00 reg  100][ss index base]
     769     // [00 reg  100][00   100  esp]
     770     // [00 reg base]
     771     // [00 reg  100][ss index  101][disp32]
     772     // [00 reg  101]               [disp32]
     773
     774     if (base == b101) {
     775       if (which == disp32_operand)
     776         return ip;              // caller wants the disp32
     777       ip += 4;                  // skip the disp32
     778     }
     779     break;
     780
     781   case 1:
     782     // [01 reg  100][ss index base][disp8]
     783     // [01 reg  100][00   100  esp][disp8]
     784     // [01 reg base]               [disp8]
     785     ip += 1;                    // skip the disp8
     786     break;
     787
     788   case 2:
     789     // [10 reg  100][ss index base][disp32]
     790     // [10 reg  100][00   100  esp][disp32]
     791     // [10 reg base]               [disp32]
     792     if (which == disp32_operand)
     793       return ip;                // caller wants the disp32
     794     ip += 4;                    // skip the disp32
     795     break;
     796
     797   case 3:
     798     // [11 reg base]  (not a memory addressing mode)
     799     break;
     800   }
     801
     802   if (which == end_pc_operand) {
     803     return ip + tail_size;
     804   }
     805
     806 #ifdef _LP64
     807   assert(which == narrow_oop_operand && !is_64bit, "instruction is not a movl adr, imm32");
     808 #else
     809   assert(which == imm_operand, "instruction has only an imm field");
     810 #endif // LP64
     811   return ip;
     812 }
     813
     814 address Assembler::locate_next_instruction(address inst) {
     815   // Secretly share code with locate_operand:
     816   return locate_operand(inst, end_pc_operand);
     817 }
     818
     819
     820 #ifdef ASSERT
     821 void Assembler::check_relocation(RelocationHolder const& rspec, int format) {
     822   address inst = inst_mark();
     823   assert(inst != NULL && inst < pc(), "must point to beginning of instruction");
     824   address opnd;
     825
     826   Relocation* r = rspec.reloc();
     827   if (r->type() == relocInfo::none) {
     828     return;
     829   } else if (r->is_call() || format == call32_operand) {
     830     // assert(format == imm32_operand, "cannot specify a nonzero format");
     831     opnd = locate_operand(inst, call32_operand);
     832   } else if (r->is_data()) {
     833     assert(format == imm_operand || format == disp32_operand
     834            LP64_ONLY(|| format == narrow_oop_operand), "format ok");
     835     opnd = locate_operand(inst, (WhichOperand)format);
     836   } else {
     837     assert(format == imm_operand, "cannot specify a format");
     838     return;
     839   }
     840   assert(opnd == pc(), "must put operand where relocs can find it");
     841 }
     842 #endif // ASSERT
     843
     844 void Assembler::emit_operand32(Register reg, Address adr) {
     845   assert(reg->encoding() < 8, "no extended registers");
     846   assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
     847   emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
     848                adr._rspec);
     849 }
     850
     851 void Assembler::emit_operand(Register reg, Address adr,
     852                              int rip_relative_correction) {
     853   emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
     854                adr._rspec,
     855                rip_relative_correction);
     856 }
     857
     858 void Assembler::emit_operand(XMMRegister reg, Address adr) {
     859   emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
     860                adr._rspec);
     861 }
     862
     863 // MMX operations
     864 void Assembler::emit_operand(MMXRegister reg, Address adr) {
     865   assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
     866   emit_operand((Register)reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec);
     867 }
     868
     869 // work around gcc (3.2.1-7a) bug
     870 void Assembler::emit_operand(Address adr, MMXRegister reg) {
     871   assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
     872   emit_operand((Register)reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec);
     873 }
     874
     875
     876 void Assembler::emit_farith(int b1, int b2, int i) {
     877   assert(isByte(b1) && isByte(b2), "wrong opcode");
     878   assert(0 <= i &&  i < 8, "illegal stack offset");
     879   emit_int8(b1);
     880   emit_int8(b2 + i);
     881 }
     882
     883
     884 // Now the Assembler instructions (identical for 32/64 bits)
     885
     886 void Assembler::adcl(Address dst, int32_t imm32) {
     887   InstructionMark im(this);
     888   prefix(dst);
     889   emit_arith_operand(0x81, rdx, dst, imm32);
     890 }
     891
     892 void Assembler::adcl(Address dst, Register src) {
     893   InstructionMark im(this);
     894   prefix(dst, src);
     895   emit_int8(0x11);
     896   emit_operand(src, dst);
     897 }
     898
     899 void Assembler::adcl(Register dst, int32_t imm32) {
     900   prefix(dst);
     901   emit_arith(0x81, 0xD0, dst, imm32);
     902 }
     903
     904 void Assembler::adcl(Register dst, Address src) {
     905   InstructionMark im(this);
     906   prefix(src, dst);
     907   emit_int8(0x13);
     908   emit_operand(dst, src);
     909 }
     910
     911 void Assembler::adcl(Register dst, Register src) {
     912   (void) prefix_and_encode(dst->encoding(), src->encoding());
     913   emit_arith(0x13, 0xC0, dst, src);
     914 }
     915
     916 void Assembler::addl(Address dst, int32_t imm32) {
     917   InstructionMark im(this);
     918   prefix(dst);
     919   emit_arith_operand(0x81, rax, dst, imm32);
     920 }
     921
     922 void Assembler::addl(Address dst, Register src) {
     923   InstructionMark im(this);
     924   prefix(dst, src);
     925   emit_int8(0x01);
     926   emit_operand(src, dst);
     927 }
     928
     929 void Assembler::addl(Register dst, int32_t imm32) {
     930   prefix(dst);
     931   emit_arith(0x81, 0xC0, dst, imm32);
     932 }
     933
     934 void Assembler::addl(Register dst, Address src) {
     935   InstructionMark im(this);
     936   prefix(src, dst);
     937   emit_int8(0x03);
     938   emit_operand(dst, src);
     939 }
     940
     941 void Assembler::addl(Register dst, Register src) {
     942   (void) prefix_and_encode(dst->encoding(), src->encoding());
     943   emit_arith(0x03, 0xC0, dst, src);
     944 }
     945
     946 void Assembler::addr_nop_4() {
     947   assert(UseAddressNop, "no CPU support");
     948   // 4 bytes: NOP DWORD PTR [EAX+0]
     949   emit_int8(0x0F);
     950   emit_int8(0x1F);
     951   emit_int8(0x40); // emit_rm(cbuf, 0x1, EAX_enc, EAX_enc);
     952   emit_int8(0);    // 8-bits offset (1 byte)
     953 }
     954
     955 void Assembler::addr_nop_5() {
     956   assert(UseAddressNop, "no CPU support");
     957   // 5 bytes: NOP DWORD PTR [EAX+EAX*0+0] 8-bits offset
     958   emit_int8(0x0F);
     959   emit_int8(0x1F);
     960   emit_int8(0x44); // emit_rm(cbuf, 0x1, EAX_enc, 0x4);
     961   emit_int8(0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
     962   emit_int8(0);    // 8-bits offset (1 byte)
     963 }
     964
     965 void Assembler::addr_nop_7() {
     966   assert(UseAddressNop, "no CPU support");
     967   // 7 bytes: NOP DWORD PTR [EAX+0] 32-bits offset
     968   emit_int8(0x0F);
     969   emit_int8(0x1F);
     970   emit_int8((unsigned char)0x80);
     971                    // emit_rm(cbuf, 0x2, EAX_enc, EAX_enc);
     972   emit_int32(0);   // 32-bits offset (4 bytes)
     973 }
     974
     975 void Assembler::addr_nop_8() {
     976   assert(UseAddressNop, "no CPU support");
     977   // 8 bytes: NOP DWORD PTR [EAX+EAX*0+0] 32-bits offset
     978   emit_int8(0x0F);
     979   emit_int8(0x1F);
     980   emit_int8((unsigned char)0x84);
     981                    // emit_rm(cbuf, 0x2, EAX_enc, 0x4);
     982   emit_int8(0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
     983   emit_int32(0);   // 32-bits offset (4 bytes)
     984 }
     985
     986 void Assembler::addsd(XMMRegister dst, XMMRegister src) {
     987   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
     988   emit_simd_arith(0x58, dst, src, VEX_SIMD_F2);
     989 }
     990
     991 void Assembler::addsd(XMMRegister dst, Address src) {
     992   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
     993   emit_simd_arith(0x58, dst, src, VEX_SIMD_F2);
     994 }
     995
     996 void Assembler::addss(XMMRegister dst, XMMRegister src) {
     997   NOT_LP64(assert(VM_Version::supports_sse(), ""));
     998   emit_simd_arith(0x58, dst, src, VEX_SIMD_F3);
     999 }
    1000
    1001 void Assembler::addss(XMMRegister dst, Address src) {
    1002   NOT_LP64(assert(VM_Version::supports_sse(), ""));
    1003   emit_simd_arith(0x58, dst, src, VEX_SIMD_F3);
    1004 }
    1005
    1006 void Assembler::aesdec(XMMRegister dst, Address src) {
    1007   assert(VM_Version::supports_aes(), "");
    1008   InstructionMark im(this);
    1009   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
    1010   emit_int8((unsigned char)0xDE);
    1011   emit_operand(dst, src);
    1012 }
    1013
    1014 void Assembler::aesdec(XMMRegister dst, XMMRegister src) {
    1015   assert(VM_Version::supports_aes(), "");
    1016   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
    1017   emit_int8((unsigned char)0xDE);
    1018   emit_int8(0xC0 | encode);
    1019 }
    1020
    1021 void Assembler::aesdeclast(XMMRegister dst, Address src) {
    1022   assert(VM_Version::supports_aes(), "");
    1023   InstructionMark im(this);
    1024   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
    1025   emit_int8((unsigned char)0xDF);
    1026   emit_operand(dst, src);
    1027 }
    1028
    1029 void Assembler::aesdeclast(XMMRegister dst, XMMRegister src) {
    1030   assert(VM_Version::supports_aes(), "");
    1031   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
    1032   emit_int8((unsigned char)0xDF);
    1033   emit_int8((unsigned char)(0xC0 | encode));
    1034 }
    1035
    1036 void Assembler::aesenc(XMMRegister dst, Address src) {
    1037   assert(VM_Version::supports_aes(), "");
    1038   InstructionMark im(this);
    1039   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
    1040   emit_int8((unsigned char)0xDC);
    1041   emit_operand(dst, src);
    1042 }
    1043
    1044 void Assembler::aesenc(XMMRegister dst, XMMRegister src) {
    1045   assert(VM_Version::supports_aes(), "");
    1046   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
    1047   emit_int8((unsigned char)0xDC);
    1048   emit_int8(0xC0 | encode);
    1049 }
    1050
    1051 void Assembler::aesenclast(XMMRegister dst, Address src) {
    1052   assert(VM_Version::supports_aes(), "");
    1053   InstructionMark im(this);
    1054   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
    1055   emit_int8((unsigned char)0xDD);
    1056   emit_operand(dst, src);
    1057 }
    1058
    1059 void Assembler::aesenclast(XMMRegister dst, XMMRegister src) {
    1060   assert(VM_Version::supports_aes(), "");
    1061   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
    1062   emit_int8((unsigned char)0xDD);
    1063   emit_int8((unsigned char)(0xC0 | encode));
    1064 }
    1065
    1066
    1067 void Assembler::andl(Address dst, int32_t imm32) {
    1068   InstructionMark im(this);
    1069   prefix(dst);
    1070   emit_int8((unsigned char)0x81);
    1071   emit_operand(rsp, dst, 4);
    1072   emit_int32(imm32);
    1073 }
    1074
    1075 void Assembler::andl(Register dst, int32_t imm32) {
    1076   prefix(dst);
    1077   emit_arith(0x81, 0xE0, dst, imm32);
    1078 }
    1079
    1080 void Assembler::andl(Register dst, Address src) {
    1081   InstructionMark im(this);
    1082   prefix(src, dst);
    1083   emit_int8(0x23);
    1084   emit_operand(dst, src);
    1085 }
    1086
    1087 void Assembler::andl(Register dst, Register src) {
    1088   (void) prefix_and_encode(dst->encoding(), src->encoding());
    1089   emit_arith(0x23, 0xC0, dst, src);
    1090 }
    1091
    1092 void Assembler::andnl(Register dst, Register src1, Register src2) {
    1093   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
    1094   int encode = vex_prefix_0F38_and_encode(dst, src1, src2);
    1095   emit_int8((unsigned char)0xF2);
    1096   emit_int8((unsigned char)(0xC0 | encode));
    1097 }
    1098
    1099 void Assembler::andnl(Register dst, Register src1, Address src2) {
    1100   InstructionMark im(this);
    1101   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
    1102   vex_prefix_0F38(dst, src1, src2);
    1103   emit_int8((unsigned char)0xF2);
    1104   emit_operand(dst, src2);
    1105 }
    1106
    1107 void Assembler::bsfl(Register dst, Register src) {
    1108   int encode = prefix_and_encode(dst->encoding(), src->encoding());
    1109   emit_int8(0x0F);
    1110   emit_int8((unsigned char)0xBC);
    1111   emit_int8((unsigned char)(0xC0 | encode));
    1112 }
    1113
    1114 void Assembler::bsrl(Register dst, Register src) {
    1115   int encode = prefix_and_encode(dst->encoding(), src->encoding());
    1116   emit_int8(0x0F);
    1117   emit_int8((unsigned char)0xBD);
    1118   emit_int8((unsigned char)(0xC0 | encode));
    1119 }
    1120
    1121 void Assembler::bswapl(Register reg) { // bswap
    1122   int encode = prefix_and_encode(reg->encoding());
    1123   emit_int8(0x0F);
    1124   emit_int8((unsigned char)(0xC8 | encode));
    1125 }
    1126
    1127 void Assembler::blsil(Register dst, Register src) {
    1128   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
    1129   int encode = vex_prefix_0F38_and_encode(rbx, dst, src);
    1130   emit_int8((unsigned char)0xF3);
    1131   emit_int8((unsigned char)(0xC0 | encode));
    1132 }
    1133
    1134 void Assembler::blsil(Register dst, Address src) {
    1135   InstructionMark im(this);
    1136   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
    1137   vex_prefix_0F38(rbx, dst, src);
    1138   emit_int8((unsigned char)0xF3);
    1139   emit_operand(rbx, src);
    1140 }
    1141
    1142 void Assembler::blsmskl(Register dst, Register src) {
    1143   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
    1144   int encode = vex_prefix_0F38_and_encode(rdx, dst, src);
    1145   emit_int8((unsigned char)0xF3);
    1146   emit_int8((unsigned char)(0xC0 | encode));
    1147 }
    1148
    1149 void Assembler::blsmskl(Register dst, Address src) {
    1150   InstructionMark im(this);
    1151   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
    1152   vex_prefix_0F38(rdx, dst, src);
    1153   emit_int8((unsigned char)0xF3);
    1154   emit_operand(rdx, src);
    1155 }
    1156
    1157 void Assembler::blsrl(Register dst, Register src) {
    1158   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
    1159   int encode = vex_prefix_0F38_and_encode(rcx, dst, src);
    1160   emit_int8((unsigned char)0xF3);
    1161   emit_int8((unsigned char)(0xC0 | encode));
    1162 }
    1163
    1164 void Assembler::blsrl(Register dst, Address src) {
    1165   InstructionMark im(this);
    1166   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
    1167   vex_prefix_0F38(rcx, dst, src);
    1168   emit_int8((unsigned char)0xF3);
    1169   emit_operand(rcx, src);
    1170 }
    1171
    1172 void Assembler::call(Label& L, relocInfo::relocType rtype) {
    1173   // suspect disp32 is always good
    1174   int operand = LP64_ONLY(disp32_operand) NOT_LP64(imm_operand);
    1175
    1176   if (L.is_bound()) {
    1177     const int long_size = 5;
    1178     int offs = (int)( target(L) - pc() );
    1179     assert(offs <= 0, "assembler error");
    1180     InstructionMark im(this);
    1181     // 1110 1000 #32-bit disp
    1182     emit_int8((unsigned char)0xE8);
    1183     emit_data(offs - long_size, rtype, operand);
    1184   } else {
    1185     InstructionMark im(this);
    1186     // 1110 1000 #32-bit disp
    1187     L.add_patch_at(code(), locator());
    1188
    1189     emit_int8((unsigned char)0xE8);
    1190     emit_data(int(0), rtype, operand);
    1191   }
    1192 }
    1193
    1194 void Assembler::call(Register dst) {
    1195   int encode = prefix_and_encode(dst->encoding());
    1196   emit_int8((unsigned char)0xFF);
    1197   emit_int8((unsigned char)(0xD0 | encode));
    1198 }
    1199
    1200
    1201 void Assembler::call(Address adr) {
    1202   InstructionMark im(this);
    1203   prefix(adr);
    1204   emit_int8((unsigned char)0xFF);
    1205   emit_operand(rdx, adr);
    1206 }
    1207
    1208 void Assembler::call_literal(address entry, RelocationHolder const& rspec) {
    1209   assert(entry != NULL, "call most probably wrong");
    1210   InstructionMark im(this);
    1211   emit_int8((unsigned char)0xE8);
    1212   intptr_t disp = entry - (pc() + sizeof(int32_t));
    1213   assert(is_simm32(disp), "must be 32bit offset (call2)");
    1214   // Technically, should use call32_operand, but this format is
    1215   // implied by the fact that we're emitting a call instruction.
    1216
    1217   int operand = LP64_ONLY(disp32_operand) NOT_LP64(call32_operand);
    1218   emit_data((int) disp, rspec, operand);
    1219 }
    1220
    1221 void Assembler::cdql() {
    1222   emit_int8((unsigned char)0x99);
    1223 }
    1224
    1225 void Assembler::cld() {
    1226   emit_int8((unsigned char)0xFC);
    1227 }
    1228
    1229 void Assembler::cmovl(Condition cc, Register dst, Register src) {
    1230   NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction"));
    1231   int encode = prefix_and_encode(dst->encoding(), src->encoding());
    1232   emit_int8(0x0F);
    1233   emit_int8(0x40 | cc);
    1234   emit_int8((unsigned char)(0xC0 | encode));
    1235 }
    1236
    1237
    1238 void Assembler::cmovl(Condition cc, Register dst, Address src) {
    1239   NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction"));
    1240   prefix(src, dst);
    1241   emit_int8(0x0F);
    1242   emit_int8(0x40 | cc);
    1243   emit_operand(dst, src);
    1244 }
    1245
    1246 void Assembler::cmpb(Address dst, int imm8) {
    1247   InstructionMark im(this);
    1248   prefix(dst);
    1249   emit_int8((unsigned char)0x80);
    1250   emit_operand(rdi, dst, 1);
    1251   emit_int8(imm8);
    1252 }
    1253
    1254 void Assembler::cmpl(Address dst, int32_t imm32) {
    1255   InstructionMark im(this);
    1256   prefix(dst);
    1257   emit_int8((unsigned char)0x81);
    1258   emit_operand(rdi, dst, 4);
    1259   emit_int32(imm32);
    1260 }
    1261
    1262 void Assembler::cmpl(Register dst, int32_t imm32) {
    1263   prefix(dst);
    1264   emit_arith(0x81, 0xF8, dst, imm32);
    1265 }
    1266
    1267 void Assembler::cmpl(Register dst, Register src) {
    1268   (void) prefix_and_encode(dst->encoding(), src->encoding());
    1269   emit_arith(0x3B, 0xC0, dst, src);
    1270 }
    1271
    1272
    1273 void Assembler::cmpl(Register dst, Address  src) {
    1274   InstructionMark im(this);
    1275   prefix(src, dst);
    1276   emit_int8((unsigned char)0x3B);
    1277   emit_operand(dst, src);
    1278 }
    1279
    1280 void Assembler::cmpw(Address dst, int imm16) {
    1281   InstructionMark im(this);
    1282   assert(!dst.base_needs_rex() && !dst.index_needs_rex(), "no extended registers");
    1283   emit_int8(0x66);
    1284   emit_int8((unsigned char)0x81);
    1285   emit_operand(rdi, dst, 2);
    1286   emit_int16(imm16);
    1287 }
    1288
    1289 // The 32-bit cmpxchg compares the value at adr with the contents of rax,
    1290 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
    1291 // The ZF is set if the compared values were equal, and cleared otherwise.
    1292 void Assembler::cmpxchgl(Register reg, Address adr) { // cmpxchg
    1293   InstructionMark im(this);
    1294   prefix(adr, reg);
    1295   emit_int8(0x0F);
    1296   emit_int8((unsigned char)0xB1);
    1297   emit_operand(reg, adr);
    1298 }
    1299
    1300 // The 8-bit cmpxchg compares the value at adr with the contents of rax,
    1301 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
    1302 // The ZF is set if the compared values were equal, and cleared otherwise.
    1303 void Assembler::cmpxchgb(Register reg, Address adr) { // cmpxchg
    1304   InstructionMark im(this);
    1305   prefix(adr, reg, true);
    1306   emit_int8(0x0F);
    1307   emit_int8((unsigned char)0xB0);
    1308   emit_operand(reg, adr);
    1309 }
    1310
    1311 void Assembler::comisd(XMMRegister dst, Address src) {
    1312   // NOTE: dbx seems to decode this as comiss even though the
    1313   // 0x66 is there. Strangly ucomisd comes out correct
    1314   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    1315   emit_simd_arith_nonds(0x2F, dst, src, VEX_SIMD_66);
    1316 }
    1317
    1318 void Assembler::comisd(XMMRegister dst, XMMRegister src) {
    1319   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    1320   emit_simd_arith_nonds(0x2F, dst, src, VEX_SIMD_66);
    1321 }
    1322
    1323 void Assembler::comiss(XMMRegister dst, Address src) {
    1324   NOT_LP64(assert(VM_Version::supports_sse(), ""));
    1325   emit_simd_arith_nonds(0x2F, dst, src, VEX_SIMD_NONE);
    1326 }
    1327
    1328 void Assembler::comiss(XMMRegister dst, XMMRegister src) {
    1329   NOT_LP64(assert(VM_Version::supports_sse(), ""));
    1330   emit_simd_arith_nonds(0x2F, dst, src, VEX_SIMD_NONE);
    1331 }
    1332
    1333 void Assembler::cpuid() {
    1334   emit_int8(0x0F);
    1335   emit_int8((unsigned char)0xA2);
    1336 }
    1337
    1338 void Assembler::cvtdq2pd(XMMRegister dst, XMMRegister src) {
    1339   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    1340   emit_simd_arith_nonds(0xE6, dst, src, VEX_SIMD_F3);
    1341 }
    1342
    1343 void Assembler::cvtdq2ps(XMMRegister dst, XMMRegister src) {
    1344   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    1345   emit_simd_arith_nonds(0x5B, dst, src, VEX_SIMD_NONE);
    1346 }
    1347
    1348 void Assembler::cvtsd2ss(XMMRegister dst, XMMRegister src) {
    1349   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    1350   emit_simd_arith(0x5A, dst, src, VEX_SIMD_F2);
    1351 }
    1352
    1353 void Assembler::cvtsd2ss(XMMRegister dst, Address src) {
    1354   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    1355   emit_simd_arith(0x5A, dst, src, VEX_SIMD_F2);
    1356 }
    1357
    1358 void Assembler::cvtsi2sdl(XMMRegister dst, Register src) {
    1359   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    1360   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2);
    1361   emit_int8(0x2A);
    1362   emit_int8((unsigned char)(0xC0 | encode));
    1363 }
    1364
    1365 void Assembler::cvtsi2sdl(XMMRegister dst, Address src) {
    1366   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    1367   emit_simd_arith(0x2A, dst, src, VEX_SIMD_F2);
    1368 }
    1369
    1370 void Assembler::cvtsi2ssl(XMMRegister dst, Register src) {
    1371   NOT_LP64(assert(VM_Version::supports_sse(), ""));
    1372   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3);
    1373   emit_int8(0x2A);
    1374   emit_int8((unsigned char)(0xC0 | encode));
    1375 }
    1376
    1377 void Assembler::cvtsi2ssl(XMMRegister dst, Address src) {
    1378   NOT_LP64(assert(VM_Version::supports_sse(), ""));
    1379   emit_simd_arith(0x2A, dst, src, VEX_SIMD_F3);
    1380 }
    1381
    1382 void Assembler::cvtss2sd(XMMRegister dst, XMMRegister src) {
    1383   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    1384   emit_simd_arith(0x5A, dst, src, VEX_SIMD_F3);
    1385 }
    1386
    1387 void Assembler::cvtss2sd(XMMRegister dst, Address src) {
    1388   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    1389   emit_simd_arith(0x5A, dst, src, VEX_SIMD_F3);
    1390 }
    1391
    1392
    1393 void Assembler::cvttsd2sil(Register dst, XMMRegister src) {
    1394   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    1395   int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_F2);
    1396   emit_int8(0x2C);
    1397   emit_int8((unsigned char)(0xC0 | encode));
    1398 }
    1399
    1400 void Assembler::cvttss2sil(Register dst, XMMRegister src) {
    1401   NOT_LP64(assert(VM_Version::supports_sse(), ""));
    1402   int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_F3);
    1403   emit_int8(0x2C);
    1404   emit_int8((unsigned char)(0xC0 | encode));
    1405 }
    1406
    1407 void Assembler::decl(Address dst) {
    1408   // Don't use it directly. Use MacroAssembler::decrement() instead.
    1409   InstructionMark im(this);
    1410   prefix(dst);
    1411   emit_int8((unsigned char)0xFF);
    1412   emit_operand(rcx, dst);
    1413 }
    1414
    1415 void Assembler::divsd(XMMRegister dst, Address src) {
    1416   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    1417   emit_simd_arith(0x5E, dst, src, VEX_SIMD_F2);
    1418 }
    1419
    1420 void Assembler::divsd(XMMRegister dst, XMMRegister src) {
    1421   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    1422   emit_simd_arith(0x5E, dst, src, VEX_SIMD_F2);
    1423 }
    1424
    1425 void Assembler::divss(XMMRegister dst, Address src) {
    1426   NOT_LP64(assert(VM_Version::supports_sse(), ""));
    1427   emit_simd_arith(0x5E, dst, src, VEX_SIMD_F3);
    1428 }
    1429
    1430 void Assembler::divss(XMMRegister dst, XMMRegister src) {
    1431   NOT_LP64(assert(VM_Version::supports_sse(), ""));
    1432   emit_simd_arith(0x5E, dst, src, VEX_SIMD_F3);
    1433 }
    1434
    1435 void Assembler::emms() {
    1436   NOT_LP64(assert(VM_Version::supports_mmx(), ""));
    1437   emit_int8(0x0F);
    1438   emit_int8(0x77);
    1439 }
    1440
    1441 void Assembler::hlt() {
    1442   emit_int8((unsigned char)0xF4);
    1443 }
    1444
    1445 void Assembler::idivl(Register src) {
    1446   int encode = prefix_and_encode(src->encoding());
    1447   emit_int8((unsigned char)0xF7);
    1448   emit_int8((unsigned char)(0xF8 | encode));
    1449 }
    1450
    1451 void Assembler::divl(Register src) { // Unsigned
    1452   int encode = prefix_and_encode(src->encoding());
    1453   emit_int8((unsigned char)0xF7);
    1454   emit_int8((unsigned char)(0xF0 | encode));
    1455 }
    1456
    1457 void Assembler::imull(Register dst, Register src) {
    1458   int encode = prefix_and_encode(dst->encoding(), src->encoding());
    1459   emit_int8(0x0F);
    1460   emit_int8((unsigned char)0xAF);
    1461   emit_int8((unsigned char)(0xC0 | encode));
    1462 }
    1463
    1464
    1465 void Assembler::imull(Register dst, Register src, int value) {
    1466   int encode = prefix_and_encode(dst->encoding(), src->encoding());
    1467   if (is8bit(value)) {
    1468     emit_int8(0x6B);
    1469     emit_int8((unsigned char)(0xC0 | encode));
    1470     emit_int8(value & 0xFF);
    1471   } else {
    1472     emit_int8(0x69);
    1473     emit_int8((unsigned char)(0xC0 | encode));
    1474     emit_int32(value);
    1475   }
    1476 }
    1477
    1478 void Assembler::imull(Register dst, Address src) {
    1479   InstructionMark im(this);
    1480   prefix(src, dst);
    1481   emit_int8(0x0F);
    1482   emit_int8((unsigned char) 0xAF);
    1483   emit_operand(dst, src);
    1484 }
    1485
    1486
    1487 void Assembler::incl(Address dst) {
    1488   // Don't use it directly. Use MacroAssembler::increment() instead.
    1489   InstructionMark im(this);
    1490   prefix(dst);
    1491   emit_int8((unsigned char)0xFF);
    1492   emit_operand(rax, dst);
    1493 }
    1494
    1495 void Assembler::jcc(Condition cc, Label& L, bool maybe_short) {
    1496   InstructionMark im(this);
    1497   assert((0 <= cc) && (cc < 16), "illegal cc");
    1498   if (L.is_bound()) {
    1499     address dst = target(L);
    1500     assert(dst != NULL, "jcc most probably wrong");
    1501
    1502     const int short_size = 2;
    1503     const int long_size = 6;
    1504     intptr_t offs = (intptr_t)dst - (intptr_t)pc();
    1505     if (maybe_short && is8bit(offs - short_size)) {
    1506       // 0111 tttn #8-bit disp
    1507       emit_int8(0x70 | cc);
    1508       emit_int8((offs - short_size) & 0xFF);
    1509     } else {
    1510       // 0000 1111 1000 tttn #32-bit disp
    1511       assert(is_simm32(offs - long_size),
    1512              "must be 32bit offset (call4)");
    1513       emit_int8(0x0F);
    1514       emit_int8((unsigned char)(0x80 | cc));
    1515       emit_int32(offs - long_size);
    1516     }
    1517   } else {
    1518     // Note: could eliminate cond. jumps to this jump if condition
    1519     //       is the same however, seems to be rather unlikely case.
    1520     // Note: use jccb() if label to be bound is very close to get
    1521     //       an 8-bit displacement
    1522     L.add_patch_at(code(), locator());
    1523     emit_int8(0x0F);
    1524     emit_int8((unsigned char)(0x80 | cc));
    1525     emit_int32(0);
    1526   }
    1527 }
    1528
    1529 void Assembler::jccb(Condition cc, Label& L) {
    1530   if (L.is_bound()) {
    1531     const int short_size = 2;
    1532     address entry = target(L);
    1533 #ifdef ASSERT
    1534     intptr_t dist = (intptr_t)entry - ((intptr_t)pc() + short_size);
    1535     intptr_t delta = short_branch_delta();
    1536     if (delta != 0) {
    1537       dist += (dist < 0 ? (-delta) :delta);
    1538     }
    1539     assert(is8bit(dist), "Dispacement too large for a short jmp");
    1540 #endif
    1541     intptr_t offs = (intptr_t)entry - (intptr_t)pc();
    1542     // 0111 tttn #8-bit disp
    1543     emit_int8(0x70 | cc);
    1544     emit_int8((offs - short_size) & 0xFF);
    1545   } else {
    1546     InstructionMark im(this);
    1547     L.add_patch_at(code(), locator());
    1548     emit_int8(0x70 | cc);
    1549     emit_int8(0);
    1550   }
    1551 }
    1552
    1553 void Assembler::jmp(Address adr) {
    1554   InstructionMark im(this);
    1555   prefix(adr);
    1556   emit_int8((unsigned char)0xFF);
    1557   emit_operand(rsp, adr);
    1558 }
    1559
    1560 void Assembler::jmp(Label& L, bool maybe_short) {
    1561   if (L.is_bound()) {
    1562     address entry = target(L);
    1563     assert(entry != NULL, "jmp most probably wrong");
    1564     InstructionMark im(this);
    1565     const int short_size = 2;
    1566     const int long_size = 5;
    1567     intptr_t offs = entry - pc();
    1568     if (maybe_short && is8bit(offs - short_size)) {
    1569       emit_int8((unsigned char)0xEB);
    1570       emit_int8((offs - short_size) & 0xFF);
    1571     } else {
    1572       emit_int8((unsigned char)0xE9);
    1573       emit_int32(offs - long_size);
    1574     }
    1575   } else {
    1576     // By default, forward jumps are always 32-bit displacements, since
    1577     // we can't yet know where the label will be bound.  If you're sure that
    1578     // the forward jump will not run beyond 256 bytes, use jmpb to
    1579     // force an 8-bit displacement.
    1580     InstructionMark im(this);
    1581     L.add_patch_at(code(), locator());
    1582     emit_int8((unsigned char)0xE9);
    1583     emit_int32(0);
    1584   }
    1585 }
    1586
    1587 void Assembler::jmp(Register entry) {
    1588   int encode = prefix_and_encode(entry->encoding());
    1589   emit_int8((unsigned char)0xFF);
    1590   emit_int8((unsigned char)(0xE0 | encode));
    1591 }
    1592
    1593 void Assembler::jmp_literal(address dest, RelocationHolder const& rspec) {
    1594   InstructionMark im(this);
    1595   emit_int8((unsigned char)0xE9);
    1596   assert(dest != NULL, "must have a target");
    1597   intptr_t disp = dest - (pc() + sizeof(int32_t));
    1598   assert(is_simm32(disp), "must be 32bit offset (jmp)");
    1599   emit_data(disp, rspec.reloc(), call32_operand);
    1600 }
    1601
    1602 void Assembler::jmpb(Label& L) {
    1603   if (L.is_bound()) {
    1604     const int short_size = 2;
    1605     address entry = target(L);
    1606     assert(entry != NULL, "jmp most probably wrong");
    1607 #ifdef ASSERT
    1608     intptr_t dist = (intptr_t)entry - ((intptr_t)pc() + short_size);
    1609     intptr_t delta = short_branch_delta();
    1610     if (delta != 0) {
    1611       dist += (dist < 0 ? (-delta) :delta);
    1612     }
    1613     assert(is8bit(dist), "Dispacement too large for a short jmp");
    1614 #endif
    1615     intptr_t offs = entry - pc();
    1616     emit_int8((unsigned char)0xEB);
    1617     emit_int8((offs - short_size) & 0xFF);
    1618   } else {
    1619     InstructionMark im(this);
    1620     L.add_patch_at(code(), locator());
    1621     emit_int8((unsigned char)0xEB);
    1622     emit_int8(0);
    1623   }
    1624 }
    1625
    1626 void Assembler::ldmxcsr( Address src) {
    1627   NOT_LP64(assert(VM_Version::supports_sse(), ""));
    1628   InstructionMark im(this);
    1629   prefix(src);
    1630   emit_int8(0x0F);
    1631   emit_int8((unsigned char)0xAE);
    1632   emit_operand(as_Register(2), src);
    1633 }
    1634
    1635 void Assembler::leal(Register dst, Address src) {
    1636   InstructionMark im(this);
    1637 #ifdef _LP64
    1638   emit_int8(0x67); // addr32
    1639   prefix(src, dst);
    1640 #endif // LP64
    1641   emit_int8((unsigned char)0x8D);
    1642   emit_operand(dst, src);
    1643 }
    1644
    1645 void Assembler::lfence() {
    1646   emit_int8(0x0F);
    1647   emit_int8((unsigned char)0xAE);
    1648   emit_int8((unsigned char)0xE8);
    1649 }
    1650
    1651 void Assembler::lock() {
    1652   emit_int8((unsigned char)0xF0);
    1653 }
    1654
    1655 void Assembler::lzcntl(Register dst, Register src) {
    1656   assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
    1657   emit_int8((unsigned char)0xF3);
    1658   int encode = prefix_and_encode(dst->encoding(), src->encoding());
    1659   emit_int8(0x0F);
    1660   emit_int8((unsigned char)0xBD);
    1661   emit_int8((unsigned char)(0xC0 | encode));
    1662 }
    1663
    1664 // Emit mfence instruction
    1665 void Assembler::mfence() {
    1666   NOT_LP64(assert(VM_Version::supports_sse2(), "unsupported");)
    1667   emit_int8(0x0F);
    1668   emit_int8((unsigned char)0xAE);
    1669   emit_int8((unsigned char)0xF0);
    1670 }
    1671
    1672 void Assembler::mov(Register dst, Register src) {
    1673   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
    1674 }
    1675
    1676 void Assembler::movapd(XMMRegister dst, XMMRegister src) {
    1677   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    1678   emit_simd_arith_nonds(0x28, dst, src, VEX_SIMD_66);
    1679 }
    1680
    1681 void Assembler::movaps(XMMRegister dst, XMMRegister src) {
    1682   NOT_LP64(assert(VM_Version::supports_sse(), ""));
    1683   emit_simd_arith_nonds(0x28, dst, src, VEX_SIMD_NONE);
    1684 }
    1685
    1686 void Assembler::movlhps(XMMRegister dst, XMMRegister src) {
    1687   NOT_LP64(assert(VM_Version::supports_sse(), ""));
    1688   int encode = simd_prefix_and_encode(dst, src, src, VEX_SIMD_NONE);
    1689   emit_int8(0x16);
    1690   emit_int8((unsigned char)(0xC0 | encode));
    1691 }
    1692
    1693 void Assembler::movb(Register dst, Address src) {
    1694   NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
    1695   InstructionMark im(this);
    1696   prefix(src, dst, true);
    1697   emit_int8((unsigned char)0x8A);
    1698   emit_operand(dst, src);
    1699 }
    1700
    1701
    1702 void Assembler::movb(Address dst, int imm8) {
    1703   InstructionMark im(this);
    1704    prefix(dst);
    1705   emit_int8((unsigned char)0xC6);
    1706   emit_operand(rax, dst, 1);
    1707   emit_int8(imm8);
    1708 }
    1709
    1710
    1711 void Assembler::movb(Address dst, Register src) {
    1712   assert(src->has_byte_register(), "must have byte register");
    1713   InstructionMark im(this);
    1714   prefix(dst, src, true);
    1715   emit_int8((unsigned char)0x88);
    1716   emit_operand(src, dst);
    1717 }
    1718
    1719 void Assembler::movdl(XMMRegister dst, Register src) {
    1720   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    1721   int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_66);
    1722   emit_int8(0x6E);
    1723   emit_int8((unsigned char)(0xC0 | encode));
    1724 }
    1725
    1726 void Assembler::movdl(Register dst, XMMRegister src) {
    1727   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    1728   // swap src/dst to get correct prefix
    1729   int encode = simd_prefix_and_encode(src, dst, VEX_SIMD_66);
    1730   emit_int8(0x7E);
    1731   emit_int8((unsigned char)(0xC0 | encode));
    1732 }
    1733
    1734 void Assembler::movdl(XMMRegister dst, Address src) {
    1735   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    1736   InstructionMark im(this);
    1737   simd_prefix(dst, src, VEX_SIMD_66);
    1738   emit_int8(0x6E);
    1739   emit_operand(dst, src);
    1740 }
    1741
    1742 void Assembler::movdl(Address dst, XMMRegister src) {
    1743   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    1744   InstructionMark im(this);
    1745   simd_prefix(dst, src, VEX_SIMD_66);
    1746   emit_int8(0x7E);
    1747   emit_operand(src, dst);
    1748 }
    1749
    1750 void Assembler::movdqa(XMMRegister dst, XMMRegister src) {
    1751   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    1752   emit_simd_arith_nonds(0x6F, dst, src, VEX_SIMD_66);
    1753 }
    1754
    1755 void Assembler::movdqa(XMMRegister dst, Address src) {
    1756   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    1757   emit_simd_arith_nonds(0x6F, dst, src, VEX_SIMD_66);
    1758 }
    1759
    1760 void Assembler::movdqu(XMMRegister dst, Address src) {
    1761   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    1762   emit_simd_arith_nonds(0x6F, dst, src, VEX_SIMD_F3);
    1763 }
    1764
    1765 void Assembler::movdqu(XMMRegister dst, XMMRegister src) {
    1766   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    1767   emit_simd_arith_nonds(0x6F, dst, src, VEX_SIMD_F3);
    1768 }
    1769
    1770 void Assembler::movdqu(Address dst, XMMRegister src) {
    1771   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    1772   InstructionMark im(this);
    1773   simd_prefix(dst, src, VEX_SIMD_F3);
    1774   emit_int8(0x7F);
    1775   emit_operand(src, dst);
    1776 }
    1777
    1778 // Move Unaligned 256bit Vector
    1779 void Assembler::vmovdqu(XMMRegister dst, XMMRegister src) {
    1780   assert(UseAVX > 0, "");
    1781   bool vector256 = true;
    1782   int encode = vex_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, vector256);
    1783   emit_int8(0x6F);
    1784   emit_int8((unsigned char)(0xC0 | encode));
    1785 }
    1786
    1787 void Assembler::vmovdqu(XMMRegister dst, Address src) {
    1788   assert(UseAVX > 0, "");
    1789   InstructionMark im(this);
    1790   bool vector256 = true;
    1791   vex_prefix(dst, xnoreg, src, VEX_SIMD_F3, vector256);
    1792   emit_int8(0x6F);
    1793   emit_operand(dst, src);
    1794 }
    1795
    1796 void Assembler::vmovdqu(Address dst, XMMRegister src) {
    1797   assert(UseAVX > 0, "");
    1798   InstructionMark im(this);
    1799   bool vector256 = true;
    1800   // swap src<->dst for encoding
    1801   assert(src != xnoreg, "sanity");
    1802   vex_prefix(src, xnoreg, dst, VEX_SIMD_F3, vector256);
    1803   emit_int8(0x7F);
    1804   emit_operand(src, dst);
    1805 }
    1806
    1807 // Uses zero extension on 64bit
    1808
    1809 void Assembler::movl(Register dst, int32_t imm32) {
    1810   int encode = prefix_and_encode(dst->encoding());
    1811   emit_int8((unsigned char)(0xB8 | encode));
    1812   emit_int32(imm32);
    1813 }
    1814
    1815 void Assembler::movl(Register dst, Register src) {
    1816   int encode = prefix_and_encode(dst->encoding(), src->encoding());
    1817   emit_int8((unsigned char)0x8B);
    1818   emit_int8((unsigned char)(0xC0 | encode));
    1819 }
    1820
    1821 void Assembler::movl(Register dst, Address src) {
    1822   InstructionMark im(this);
    1823   prefix(src, dst);
    1824   emit_int8((unsigned char)0x8B);
    1825   emit_operand(dst, src);
    1826 }
    1827
    1828 void Assembler::movl(Address dst, int32_t imm32) {
    1829   InstructionMark im(this);
    1830   prefix(dst);
    1831   emit_int8((unsigned char)0xC7);
    1832   emit_operand(rax, dst, 4);
    1833   emit_int32(imm32);
    1834 }
    1835
    1836 void Assembler::movl(Address dst, Register src) {
    1837   InstructionMark im(this);
    1838   prefix(dst, src);
    1839   emit_int8((unsigned char)0x89);
    1840   emit_operand(src, dst);
    1841 }
    1842
    1843 // New cpus require to use movsd and movss to avoid partial register stall
    1844 // when loading from memory. But for old Opteron use movlpd instead of movsd.
    1845 // The selection is done in MacroAssembler::movdbl() and movflt().
    1846 void Assembler::movlpd(XMMRegister dst, Address src) {
    1847   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    1848   emit_simd_arith(0x12, dst, src, VEX_SIMD_66);
    1849 }
    1850
    1851 void Assembler::movq( MMXRegister dst, Address src ) {
    1852   assert( VM_Version::supports_mmx(), "" );
    1853   emit_int8(0x0F);
    1854   emit_int8(0x6F);
    1855   emit_operand(dst, src);
    1856 }
    1857
    1858 void Assembler::movq( Address dst, MMXRegister src ) {
    1859   assert( VM_Version::supports_mmx(), "" );
    1860   emit_int8(0x0F);
    1861   emit_int8(0x7F);
    1862   // workaround gcc (3.2.1-7a) bug
    1863   // In that version of gcc with only an emit_operand(MMX, Address)
    1864   // gcc will tail jump and try and reverse the parameters completely
    1865   // obliterating dst in the process. By having a version available
    1866   // that doesn't need to swap the args at the tail jump the bug is
    1867   // avoided.
    1868   emit_operand(dst, src);
    1869 }
    1870
    1871 void Assembler::movq(XMMRegister dst, Address src) {
    1872   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    1873   InstructionMark im(this);
    1874   simd_prefix(dst, src, VEX_SIMD_F3);
    1875   emit_int8(0x7E);
    1876   emit_operand(dst, src);
    1877 }
    1878
    1879 void Assembler::movq(Address dst, XMMRegister src) {
    1880   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    1881   InstructionMark im(this);
    1882   simd_prefix(dst, src, VEX_SIMD_66);
    1883   emit_int8((unsigned char)0xD6);
    1884   emit_operand(src, dst);
    1885 }
    1886
    1887 void Assembler::movsbl(Register dst, Address src) { // movsxb
    1888   InstructionMark im(this);
    1889   prefix(src, dst);
    1890   emit_int8(0x0F);
    1891   emit_int8((unsigned char)0xBE);
    1892   emit_operand(dst, src);
    1893 }
    1894
    1895 void Assembler::movsbl(Register dst, Register src) { // movsxb
    1896   NOT_LP64(assert(src->has_byte_register(), "must have byte register"));
    1897   int encode = prefix_and_encode(dst->encoding(), src->encoding(), true);
    1898   emit_int8(0x0F);
    1899   emit_int8((unsigned char)0xBE);
    1900   emit_int8((unsigned char)(0xC0 | encode));
    1901 }
    1902
    1903 void Assembler::movsd(XMMRegister dst, XMMRegister src) {
    1904   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    1905   emit_simd_arith(0x10, dst, src, VEX_SIMD_F2);
    1906 }
    1907
    1908 void Assembler::movsd(XMMRegister dst, Address src) {
    1909   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    1910   emit_simd_arith_nonds(0x10, dst, src, VEX_SIMD_F2);
    1911 }
    1912
    1913 void Assembler::movsd(Address dst, XMMRegister src) {
    1914   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    1915   InstructionMark im(this);
    1916   simd_prefix(dst, src, VEX_SIMD_F2);
    1917   emit_int8(0x11);
    1918   emit_operand(src, dst);
    1919 }
    1920
    1921 void Assembler::movss(XMMRegister dst, XMMRegister src) {
    1922   NOT_LP64(assert(VM_Version::supports_sse(), ""));
    1923   emit_simd_arith(0x10, dst, src, VEX_SIMD_F3);
    1924 }
    1925
    1926 void Assembler::movss(XMMRegister dst, Address src) {
    1927   NOT_LP64(assert(VM_Version::supports_sse(), ""));
    1928   emit_simd_arith_nonds(0x10, dst, src, VEX_SIMD_F3);
    1929 }
    1930
    1931 void Assembler::movss(Address dst, XMMRegister src) {
    1932   NOT_LP64(assert(VM_Version::supports_sse(), ""));
    1933   InstructionMark im(this);
    1934   simd_prefix(dst, src, VEX_SIMD_F3);
    1935   emit_int8(0x11);
    1936   emit_operand(src, dst);
    1937 }
    1938
    1939 void Assembler::movswl(Register dst, Address src) { // movsxw
    1940   InstructionMark im(this);
    1941   prefix(src, dst);
    1942   emit_int8(0x0F);
    1943   emit_int8((unsigned char)0xBF);
    1944   emit_operand(dst, src);
    1945 }
    1946
    1947 void Assembler::movswl(Register dst, Register src) { // movsxw
    1948   int encode = prefix_and_encode(dst->encoding(), src->encoding());
    1949   emit_int8(0x0F);
    1950   emit_int8((unsigned char)0xBF);
    1951   emit_int8((unsigned char)(0xC0 | encode));
    1952 }
    1953
    1954 void Assembler::movw(Address dst, int imm16) {
    1955   InstructionMark im(this);
    1956
    1957   emit_int8(0x66); // switch to 16-bit mode
    1958   prefix(dst);
    1959   emit_int8((unsigned char)0xC7);
    1960   emit_operand(rax, dst, 2);
    1961   emit_int16(imm16);
    1962 }
    1963
    1964 void Assembler::movw(Register dst, Address src) {
    1965   InstructionMark im(this);
    1966   emit_int8(0x66);
    1967   prefix(src, dst);
    1968   emit_int8((unsigned char)0x8B);
    1969   emit_operand(dst, src);
    1970 }
    1971
    1972 void Assembler::movw(Address dst, Register src) {
    1973   InstructionMark im(this);
    1974   emit_int8(0x66);
    1975   prefix(dst, src);
    1976   emit_int8((unsigned char)0x89);
    1977   emit_operand(src, dst);
    1978 }
    1979
    1980 void Assembler::movzbl(Register dst, Address src) { // movzxb
    1981   InstructionMark im(this);
    1982   prefix(src, dst);
    1983   emit_int8(0x0F);
    1984   emit_int8((unsigned char)0xB6);
    1985   emit_operand(dst, src);
    1986 }
    1987
    1988 void Assembler::movzbl(Register dst, Register src) { // movzxb
    1989   NOT_LP64(assert(src->has_byte_register(), "must have byte register"));
    1990   int encode = prefix_and_encode(dst->encoding(), src->encoding(), true);
    1991   emit_int8(0x0F);
    1992   emit_int8((unsigned char)0xB6);
    1993   emit_int8(0xC0 | encode);
    1994 }
    1995
    1996 void Assembler::movzwl(Register dst, Address src) { // movzxw
    1997   InstructionMark im(this);
    1998   prefix(src, dst);
    1999   emit_int8(0x0F);
    2000   emit_int8((unsigned char)0xB7);
    2001   emit_operand(dst, src);
    2002 }
    2003
    2004 void Assembler::movzwl(Register dst, Register src) { // movzxw
    2005   int encode = prefix_and_encode(dst->encoding(), src->encoding());
    2006   emit_int8(0x0F);
    2007   emit_int8((unsigned char)0xB7);
    2008   emit_int8(0xC0 | encode);
    2009 }
    2010
    2011 void Assembler::mull(Address src) {
    2012   InstructionMark im(this);
    2013   prefix(src);
    2014   emit_int8((unsigned char)0xF7);
    2015   emit_operand(rsp, src);
    2016 }
    2017
    2018 void Assembler::mull(Register src) {
    2019   int encode = prefix_and_encode(src->encoding());
    2020   emit_int8((unsigned char)0xF7);
    2021   emit_int8((unsigned char)(0xE0 | encode));
    2022 }
    2023
    2024 void Assembler::mulsd(XMMRegister dst, Address src) {
    2025   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    2026   emit_simd_arith(0x59, dst, src, VEX_SIMD_F2);
    2027 }
    2028
    2029 void Assembler::mulsd(XMMRegister dst, XMMRegister src) {
    2030   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    2031   emit_simd_arith(0x59, dst, src, VEX_SIMD_F2);
    2032 }
    2033
    2034 void Assembler::mulss(XMMRegister dst, Address src) {
    2035   NOT_LP64(assert(VM_Version::supports_sse(), ""));
    2036   emit_simd_arith(0x59, dst, src, VEX_SIMD_F3);
    2037 }
    2038
    2039 void Assembler::mulss(XMMRegister dst, XMMRegister src) {
    2040   NOT_LP64(assert(VM_Version::supports_sse(), ""));
    2041   emit_simd_arith(0x59, dst, src, VEX_SIMD_F3);
    2042 }
    2043
    2044 void Assembler::negl(Register dst) {
    2045   int encode = prefix_and_encode(dst->encoding());
    2046   emit_int8((unsigned char)0xF7);
    2047   emit_int8((unsigned char)(0xD8 | encode));
    2048 }
    2049
    2050 void Assembler::nop(int i) {
    2051 #ifdef ASSERT
    2052   assert(i > 0, " ");
    2053   // The fancy nops aren't currently recognized by debuggers making it a
    2054   // pain to disassemble code while debugging. If asserts are on clearly
    2055   // speed is not an issue so simply use the single byte traditional nop
    2056   // to do alignment.
    2057
    2058   for (; i > 0 ; i--) emit_int8((unsigned char)0x90);
    2059   return;
    2060
    2061 #endif // ASSERT
    2062
    2063   if (UseAddressNop && VM_Version::is_intel()) {
    2064     //
    2065     // Using multi-bytes nops "0x0F 0x1F [address]" for Intel
    2066     //  1: 0x90
    2067     //  2: 0x66 0x90
    2068     //  3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
    2069     //  4: 0x0F 0x1F 0x40 0x00
    2070     //  5: 0x0F 0x1F 0x44 0x00 0x00
    2071     //  6: 0x66 0x0F 0x1F 0x44 0x00 0x00
    2072     //  7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
    2073     //  8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
    2074     //  9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
    2075     // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
    2076     // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
    2077
    2078     // The rest coding is Intel specific - don't use consecutive address nops
    2079
    2080     // 12: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
    2081     // 13: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
    2082     // 14: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
    2083     // 15: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
    2084
    2085     while(i >= 15) {
    2086       // For Intel don't generate consecutive addess nops (mix with regular nops)
    2087       i -= 15;
    2088       emit_int8(0x66);   // size prefix
    2089       emit_int8(0x66);   // size prefix
    2090       emit_int8(0x66);   // size prefix
    2091       addr_nop_8();
    2092       emit_int8(0x66);   // size prefix
    2093       emit_int8(0x66);   // size prefix
    2094       emit_int8(0x66);   // size prefix
    2095       emit_int8((unsigned char)0x90);
    2096                          // nop
    2097     }
    2098     switch (i) {
    2099       case 14:
    2100         emit_int8(0x66); // size prefix
    2101       case 13:
    2102         emit_int8(0x66); // size prefix
    2103       case 12:
    2104         addr_nop_8();
    2105         emit_int8(0x66); // size prefix
    2106         emit_int8(0x66); // size prefix
    2107         emit_int8(0x66); // size prefix
    2108         emit_int8((unsigned char)0x90);
    2109                          // nop
    2110         break;
    2111       case 11:
    2112         emit_int8(0x66); // size prefix
    2113       case 10:
    2114         emit_int8(0x66); // size prefix
    2115       case 9:
    2116         emit_int8(0x66); // size prefix
    2117       case 8:
    2118         addr_nop_8();
    2119         break;
    2120       case 7:
    2121         addr_nop_7();
    2122         break;
    2123       case 6:
    2124         emit_int8(0x66); // size prefix
    2125       case 5:
    2126         addr_nop_5();
    2127         break;
    2128       case 4:
    2129         addr_nop_4();
    2130         break;
    2131       case 3:
    2132         // Don't use "0x0F 0x1F 0x00" - need patching safe padding
    2133         emit_int8(0x66); // size prefix
    2134       case 2:
    2135         emit_int8(0x66); // size prefix
    2136       case 1:
    2137         emit_int8((unsigned char)0x90);
    2138                          // nop
    2139         break;
    2140       default:
    2141         assert(i == 0, " ");
    2142     }
    2143     return;
    2144   }
    2145   if (UseAddressNop && VM_Version::is_amd()) {
    2146     //
    2147     // Using multi-bytes nops "0x0F 0x1F [address]" for AMD.
    2148     //  1: 0x90
    2149     //  2: 0x66 0x90
    2150     //  3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
    2151     //  4: 0x0F 0x1F 0x40 0x00
    2152     //  5: 0x0F 0x1F 0x44 0x00 0x00
    2153     //  6: 0x66 0x0F 0x1F 0x44 0x00 0x00
    2154     //  7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
    2155     //  8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
    2156     //  9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
    2157     // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
    2158     // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
    2159
    2160     // The rest coding is AMD specific - use consecutive address nops
    2161
    2162     // 12: 0x66 0x0F 0x1F 0x44 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
    2163     // 13: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
    2164     // 14: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
    2165     // 15: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
    2166     // 16: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
    2167     //     Size prefixes (0x66) are added for larger sizes
    2168
    2169     while(i >= 22) {
    2170       i -= 11;
    2171       emit_int8(0x66); // size prefix
    2172       emit_int8(0x66); // size prefix
    2173       emit_int8(0x66); // size prefix
    2174       addr_nop_8();
    2175     }
    2176     // Generate first nop for size between 21-12
    2177     switch (i) {
    2178       case 21:
    2179         i -= 1;
    2180         emit_int8(0x66); // size prefix
    2181       case 20:
    2182       case 19:
    2183         i -= 1;
    2184         emit_int8(0x66); // size prefix
    2185       case 18:
    2186       case 17:
    2187         i -= 1;
    2188         emit_int8(0x66); // size prefix
    2189       case 16:
    2190       case 15:
    2191         i -= 8;
    2192         addr_nop_8();
    2193         break;
    2194       case 14:
    2195       case 13:
    2196         i -= 7;
    2197         addr_nop_7();
    2198         break;
    2199       case 12:
    2200         i -= 6;
    2201         emit_int8(0x66); // size prefix
    2202         addr_nop_5();
    2203         break;
    2204       default:
    2205         assert(i < 12, " ");
    2206     }
    2207
    2208     // Generate second nop for size between 11-1
    2209     switch (i) {
    2210       case 11:
    2211         emit_int8(0x66); // size prefix
    2212       case 10:
    2213         emit_int8(0x66); // size prefix
    2214       case 9:
    2215         emit_int8(0x66); // size prefix
    2216       case 8:
    2217         addr_nop_8();
    2218         break;
    2219       case 7:
    2220         addr_nop_7();
    2221         break;
    2222       case 6:
    2223         emit_int8(0x66); // size prefix
    2224       case 5:
    2225         addr_nop_5();
    2226         break;
    2227       case 4:
    2228         addr_nop_4();
    2229         break;
    2230       case 3:
    2231         // Don't use "0x0F 0x1F 0x00" - need patching safe padding
    2232         emit_int8(0x66); // size prefix
    2233       case 2:
    2234         emit_int8(0x66); // size prefix
    2235       case 1:
    2236         emit_int8((unsigned char)0x90);
    2237                          // nop
    2238         break;
    2239       default:
    2240         assert(i == 0, " ");
    2241     }
    2242     return;
    2243   }
    2244
    2245   // Using nops with size prefixes "0x66 0x90".
    2246   // From AMD Optimization Guide:
    2247   //  1: 0x90
    2248   //  2: 0x66 0x90
    2249   //  3: 0x66 0x66 0x90
    2250   //  4: 0x66 0x66 0x66 0x90
    2251   //  5: 0x66 0x66 0x90 0x66 0x90
    2252   //  6: 0x66 0x66 0x90 0x66 0x66 0x90
    2253   //  7: 0x66 0x66 0x66 0x90 0x66 0x66 0x90
    2254   //  8: 0x66 0x66 0x66 0x90 0x66 0x66 0x66 0x90
    2255   //  9: 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
    2256   // 10: 0x66 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
    2257   //
    2258   while(i > 12) {
    2259     i -= 4;
    2260     emit_int8(0x66); // size prefix
    2261     emit_int8(0x66);
    2262     emit_int8(0x66);
    2263     emit_int8((unsigned char)0x90);
    2264                      // nop
    2265   }
    2266   // 1 - 12 nops
    2267   if(i > 8) {
    2268     if(i > 9) {
    2269       i -= 1;
    2270       emit_int8(0x66);
    2271     }
    2272     i -= 3;
    2273     emit_int8(0x66);
    2274     emit_int8(0x66);
    2275     emit_int8((unsigned char)0x90);
    2276   }
    2277   // 1 - 8 nops
    2278   if(i > 4) {
    2279     if(i > 6) {
    2280       i -= 1;
    2281       emit_int8(0x66);
    2282     }
    2283     i -= 3;
    2284     emit_int8(0x66);
    2285     emit_int8(0x66);
    2286     emit_int8((unsigned char)0x90);
    2287   }
    2288   switch (i) {
    2289     case 4:
    2290       emit_int8(0x66);
    2291     case 3:
    2292       emit_int8(0x66);
    2293     case 2:
    2294       emit_int8(0x66);
    2295     case 1:
    2296       emit_int8((unsigned char)0x90);
    2297       break;
    2298     default:
    2299       assert(i == 0, " ");
    2300   }
    2301 }
    2302
    2303 void Assembler::notl(Register dst) {
    2304   int encode = prefix_and_encode(dst->encoding());
    2305   emit_int8((unsigned char)0xF7);
    2306   emit_int8((unsigned char)(0xD0 | encode));
    2307 }
    2308
    2309 void Assembler::orl(Address dst, int32_t imm32) {
    2310   InstructionMark im(this);
    2311   prefix(dst);
    2312   emit_arith_operand(0x81, rcx, dst, imm32);
    2313 }
    2314
    2315 void Assembler::orl(Register dst, int32_t imm32) {
    2316   prefix(dst);
    2317   emit_arith(0x81, 0xC8, dst, imm32);
    2318 }
    2319
    2320 void Assembler::orl(Register dst, Address src) {
    2321   InstructionMark im(this);
    2322   prefix(src, dst);
    2323   emit_int8(0x0B);
    2324   emit_operand(dst, src);
    2325 }
    2326
    2327 void Assembler::orl(Register dst, Register src) {
    2328   (void) prefix_and_encode(dst->encoding(), src->encoding());
    2329   emit_arith(0x0B, 0xC0, dst, src);
    2330 }
    2331
    2332 void Assembler::orl(Address dst, Register src) {
    2333   InstructionMark im(this);
    2334   prefix(dst, src);
    2335   emit_int8(0x09);
    2336   emit_operand(src, dst);
    2337 }
    2338
    2339 void Assembler::packuswb(XMMRegister dst, Address src) {
    2340   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    2341   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
    2342   emit_simd_arith(0x67, dst, src, VEX_SIMD_66);
    2343 }
    2344
    2345 void Assembler::packuswb(XMMRegister dst, XMMRegister src) {
    2346   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    2347   emit_simd_arith(0x67, dst, src, VEX_SIMD_66);
    2348 }
    2349
    2350 void Assembler::vpackuswb(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
    2351   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
    2352   emit_vex_arith(0x67, dst, nds, src, VEX_SIMD_66, vector256);
    2353 }
    2354
    2355 void Assembler::vpermq(XMMRegister dst, XMMRegister src, int imm8, bool vector256) {
    2356   assert(VM_Version::supports_avx2(), "");
    2357   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, true, vector256);
    2358   emit_int8(0x00);
    2359   emit_int8(0xC0 | encode);
    2360   emit_int8(imm8);
    2361 }
    2362
    2363 void Assembler::pause() {
    2364   emit_int8((unsigned char)0xF3);
    2365   emit_int8((unsigned char)0x90);
    2366 }
    2367
    2368 void Assembler::pcmpestri(XMMRegister dst, Address src, int imm8) {
    2369   assert(VM_Version::supports_sse4_2(), "");
    2370   InstructionMark im(this);
    2371   simd_prefix(dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A);
    2372   emit_int8(0x61);
    2373   emit_operand(dst, src);
    2374   emit_int8(imm8);
    2375 }
    2376
    2377 void Assembler::pcmpestri(XMMRegister dst, XMMRegister src, int imm8) {
    2378   assert(VM_Version::supports_sse4_2(), "");
    2379   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A);
    2380   emit_int8(0x61);
    2381   emit_int8((unsigned char)(0xC0 | encode));
    2382   emit_int8(imm8);
    2383 }
    2384
    2385 void Assembler::pextrd(Register dst, XMMRegister src, int imm8) {
    2386   assert(VM_Version::supports_sse4_1(), "");
    2387   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, false);
    2388   emit_int8(0x16);
    2389   emit_int8((unsigned char)(0xC0 | encode));
    2390   emit_int8(imm8);
    2391 }
    2392
    2393 void Assembler::pextrq(Register dst, XMMRegister src, int imm8) {
    2394   assert(VM_Version::supports_sse4_1(), "");
    2395   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, true);
    2396   emit_int8(0x16);
    2397   emit_int8((unsigned char)(0xC0 | encode));
    2398   emit_int8(imm8);
    2399 }
    2400
    2401 void Assembler::pinsrd(XMMRegister dst, Register src, int imm8) {
    2402   assert(VM_Version::supports_sse4_1(), "");
    2403   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, false);
    2404   emit_int8(0x22);
    2405   emit_int8((unsigned char)(0xC0 | encode));
    2406   emit_int8(imm8);
    2407 }
    2408
    2409 void Assembler::pinsrq(XMMRegister dst, Register src, int imm8) {
    2410   assert(VM_Version::supports_sse4_1(), "");
    2411   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, true);
    2412   emit_int8(0x22);
    2413   emit_int8((unsigned char)(0xC0 | encode));
    2414   emit_int8(imm8);
    2415 }
    2416
    2417 void Assembler::pmovzxbw(XMMRegister dst, Address src) {
    2418   assert(VM_Version::supports_sse4_1(), "");
    2419   InstructionMark im(this);
    2420   simd_prefix(dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
    2421   emit_int8(0x30);
    2422   emit_operand(dst, src);
    2423 }
    2424
    2425 void Assembler::pmovzxbw(XMMRegister dst, XMMRegister src) {
    2426   assert(VM_Version::supports_sse4_1(), "");
    2427   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
    2428   emit_int8(0x30);
    2429   emit_int8((unsigned char)(0xC0 | encode));
    2430 }
    2431
    2432 // generic
    2433 void Assembler::pop(Register dst) {
    2434   int encode = prefix_and_encode(dst->encoding());
    2435   emit_int8(0x58 | encode);
    2436 }
    2437
    2438 void Assembler::popcntl(Register dst, Address src) {
    2439   assert(VM_Version::supports_popcnt(), "must support");
    2440   InstructionMark im(this);
    2441   emit_int8((unsigned char)0xF3);
    2442   prefix(src, dst);
    2443   emit_int8(0x0F);
    2444   emit_int8((unsigned char)0xB8);
    2445   emit_operand(dst, src);
    2446 }
    2447
    2448 void Assembler::popcntl(Register dst, Register src) {
    2449   assert(VM_Version::supports_popcnt(), "must support");
    2450   emit_int8((unsigned char)0xF3);
    2451   int encode = prefix_and_encode(dst->encoding(), src->encoding());
    2452   emit_int8(0x0F);
    2453   emit_int8((unsigned char)0xB8);
    2454   emit_int8((unsigned char)(0xC0 | encode));
    2455 }
    2456
    2457 void Assembler::popf() {
    2458   emit_int8((unsigned char)0x9D);
    2459 }
    2460
    2461 #ifndef _LP64 // no 32bit push/pop on amd64
    2462 void Assembler::popl(Address dst) {
    2463   // NOTE: this will adjust stack by 8byte on 64bits
    2464   InstructionMark im(this);
    2465   prefix(dst);
    2466   emit_int8((unsigned char)0x8F);
    2467   emit_operand(rax, dst);
    2468 }
    2469 #endif
    2470
    2471 void Assembler::prefetch_prefix(Address src) {
    2472   prefix(src);
    2473   emit_int8(0x0F);
    2474 }
    2475
    2476 void Assembler::prefetchnta(Address src) {
    2477   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
    2478   InstructionMark im(this);
    2479   prefetch_prefix(src);
    2480   emit_int8(0x18);
    2481   emit_operand(rax, src); // 0, src
    2482 }
    2483
    2484 void Assembler::prefetchr(Address src) {
    2485   assert(VM_Version::supports_3dnow_prefetch(), "must support");
    2486   InstructionMark im(this);
    2487   prefetch_prefix(src);
    2488   emit_int8(0x0D);
    2489   emit_operand(rax, src); // 0, src
    2490 }
    2491
    2492 void Assembler::prefetcht0(Address src) {
    2493   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
    2494   InstructionMark im(this);
    2495   prefetch_prefix(src);
    2496   emit_int8(0x18);
    2497   emit_operand(rcx, src); // 1, src
    2498 }
    2499
    2500 void Assembler::prefetcht1(Address src) {
    2501   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
    2502   InstructionMark im(this);
    2503   prefetch_prefix(src);
    2504   emit_int8(0x18);
    2505   emit_operand(rdx, src); // 2, src
    2506 }
    2507
    2508 void Assembler::prefetcht2(Address src) {
    2509   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
    2510   InstructionMark im(this);
    2511   prefetch_prefix(src);
    2512   emit_int8(0x18);
    2513   emit_operand(rbx, src); // 3, src
    2514 }
    2515
    2516 void Assembler::prefetchw(Address src) {
    2517   assert(VM_Version::supports_3dnow_prefetch(), "must support");
    2518   InstructionMark im(this);
    2519   prefetch_prefix(src);
    2520   emit_int8(0x0D);
    2521   emit_operand(rcx, src); // 1, src
    2522 }
    2523
    2524 void Assembler::prefix(Prefix p) {
    2525   emit_int8(p);
    2526 }
    2527
    2528 void Assembler::pshufb(XMMRegister dst, XMMRegister src) {
    2529   assert(VM_Version::supports_ssse3(), "");
    2530   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
    2531   emit_int8(0x00);
    2532   emit_int8((unsigned char)(0xC0 | encode));
    2533 }
    2534
    2535 void Assembler::pshufb(XMMRegister dst, Address src) {
    2536   assert(VM_Version::supports_ssse3(), "");
    2537   InstructionMark im(this);
    2538   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
    2539   emit_int8(0x00);
    2540   emit_operand(dst, src);
    2541 }
    2542
    2543 void Assembler::pshufd(XMMRegister dst, XMMRegister src, int mode) {
    2544   assert(isByte(mode), "invalid value");
    2545   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    2546   emit_simd_arith_nonds(0x70, dst, src, VEX_SIMD_66);
    2547   emit_int8(mode & 0xFF);
    2548
    2549 }
    2550
    2551 void Assembler::pshufd(XMMRegister dst, Address src, int mode) {
    2552   assert(isByte(mode), "invalid value");
    2553   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    2554   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
    2555   InstructionMark im(this);
    2556   simd_prefix(dst, src, VEX_SIMD_66);
    2557   emit_int8(0x70);
    2558   emit_operand(dst, src);
    2559   emit_int8(mode & 0xFF);
    2560 }
    2561
    2562 void Assembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) {
    2563   assert(isByte(mode), "invalid value");
    2564   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    2565   emit_simd_arith_nonds(0x70, dst, src, VEX_SIMD_F2);
    2566   emit_int8(mode & 0xFF);
    2567 }
    2568
    2569 void Assembler::pshuflw(XMMRegister dst, Address src, int mode) {
    2570   assert(isByte(mode), "invalid value");
    2571   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    2572   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
    2573   InstructionMark im(this);
    2574   simd_prefix(dst, src, VEX_SIMD_F2);
    2575   emit_int8(0x70);
    2576   emit_operand(dst, src);
    2577   emit_int8(mode & 0xFF);
    2578 }
    2579
    2580 void Assembler::psrldq(XMMRegister dst, int shift) {
    2581   // Shift 128 bit value in xmm register by number of bytes.
    2582   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    2583   int encode = simd_prefix_and_encode(xmm3, dst, dst, VEX_SIMD_66);
    2584   emit_int8(0x73);
    2585   emit_int8((unsigned char)(0xC0 | encode));
    2586   emit_int8(shift);
    2587 }
    2588
    2589 void Assembler::ptest(XMMRegister dst, Address src) {
    2590   assert(VM_Version::supports_sse4_1(), "");
    2591   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
    2592   InstructionMark im(this);
    2593   simd_prefix(dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
    2594   emit_int8(0x17);
    2595   emit_operand(dst, src);
    2596 }
    2597
    2598 void Assembler::ptest(XMMRegister dst, XMMRegister src) {
    2599   assert(VM_Version::supports_sse4_1(), "");
    2600   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
    2601   emit_int8(0x17);
    2602   emit_int8((unsigned char)(0xC0 | encode));
    2603 }
    2604
    2605 void Assembler::vptest(XMMRegister dst, Address src) {
    2606   assert(VM_Version::supports_avx(), "");
    2607   InstructionMark im(this);
    2608   bool vector256 = true;
    2609   assert(dst != xnoreg, "sanity");
    2610   int dst_enc = dst->encoding();
    2611   // swap src<->dst for encoding
    2612   vex_prefix(src, 0, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, false, vector256);
    2613   emit_int8(0x17);
    2614   emit_operand(dst, src);
    2615 }
    2616
    2617 void Assembler::vptest(XMMRegister dst, XMMRegister src) {
    2618   assert(VM_Version::supports_avx(), "");
    2619   bool vector256 = true;
    2620   int encode = vex_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, vector256, VEX_OPCODE_0F_38);
    2621   emit_int8(0x17);
    2622   emit_int8((unsigned char)(0xC0 | encode));
    2623 }
    2624
    2625 void Assembler::punpcklbw(XMMRegister dst, Address src) {
    2626   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    2627   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
    2628   emit_simd_arith(0x60, dst, src, VEX_SIMD_66);
    2629 }
    2630
    2631 void Assembler::punpcklbw(XMMRegister dst, XMMRegister src) {
    2632   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    2633   emit_simd_arith(0x60, dst, src, VEX_SIMD_66);
    2634 }
    2635
    2636 void Assembler::punpckldq(XMMRegister dst, Address src) {
    2637   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    2638   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
    2639   emit_simd_arith(0x62, dst, src, VEX_SIMD_66);
    2640 }
    2641
    2642 void Assembler::punpckldq(XMMRegister dst, XMMRegister src) {
    2643   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    2644   emit_simd_arith(0x62, dst, src, VEX_SIMD_66);
    2645 }
    2646
    2647 void Assembler::punpcklqdq(XMMRegister dst, XMMRegister src) {
    2648   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    2649   emit_simd_arith(0x6C, dst, src, VEX_SIMD_66);
    2650 }
    2651
    2652 void Assembler::push(int32_t imm32) {
    2653   // in 64bits we push 64bits onto the stack but only
    2654   // take a 32bit immediate
    2655   emit_int8(0x68);
    2656   emit_int32(imm32);
    2657 }
    2658
    2659 void Assembler::push(Register src) {
    2660   int encode = prefix_and_encode(src->encoding());
    2661
    2662   emit_int8(0x50 | encode);
    2663 }
    2664
    2665 void Assembler::pushf() {
    2666   emit_int8((unsigned char)0x9C);
    2667 }
    2668
    2669 #ifndef _LP64 // no 32bit push/pop on amd64
    2670 void Assembler::pushl(Address src) {
    2671   // Note this will push 64bit on 64bit
    2672   InstructionMark im(this);
    2673   prefix(src);
    2674   emit_int8((unsigned char)0xFF);
    2675   emit_operand(rsi, src);
    2676 }
    2677 #endif
    2678
    2679 void Assembler::rcll(Register dst, int imm8) {
    2680   assert(isShiftCount(imm8), "illegal shift count");
    2681   int encode = prefix_and_encode(dst->encoding());
    2682   if (imm8 == 1) {
    2683     emit_int8((unsigned char)0xD1);
    2684     emit_int8((unsigned char)(0xD0 | encode));
    2685   } else {
    2686     emit_int8((unsigned char)0xC1);
    2687     emit_int8((unsigned char)0xD0 | encode);
    2688     emit_int8(imm8);
    2689   }
    2690 }
    2691
    2692 void Assembler::rdtsc() {
    2693   emit_int8((unsigned char)0x0F);
    2694   emit_int8((unsigned char)0x31);
    2695 }
    2696
    2697 // copies data from [esi] to [edi] using rcx pointer sized words
    2698 // generic
    2699 void Assembler::rep_mov() {
    2700   emit_int8((unsigned char)0xF3);
    2701   // MOVSQ
    2702   LP64_ONLY(prefix(REX_W));
    2703   emit_int8((unsigned char)0xA5);
    2704 }
    2705
    2706 // sets rcx bytes with rax, value at [edi]
    2707 void Assembler::rep_stosb() {
    2708   emit_int8((unsigned char)0xF3); // REP
    2709   LP64_ONLY(prefix(REX_W));
    2710   emit_int8((unsigned char)0xAA); // STOSB
    2711 }
    2712
    2713 // sets rcx pointer sized words with rax, value at [edi]
    2714 // generic
    2715 void Assembler::rep_stos() {
    2716   emit_int8((unsigned char)0xF3); // REP
    2717   LP64_ONLY(prefix(REX_W));       // LP64:STOSQ, LP32:STOSD
    2718   emit_int8((unsigned char)0xAB);
    2719 }
    2720
    2721 // scans rcx pointer sized words at [edi] for occurance of rax,
    2722 // generic
    2723 void Assembler::repne_scan() { // repne_scan
    2724   emit_int8((unsigned char)0xF2);
    2725   // SCASQ
    2726   LP64_ONLY(prefix(REX_W));
    2727   emit_int8((unsigned char)0xAF);
    2728 }
    2729
    2730 #ifdef _LP64
    2731 // scans rcx 4 byte words at [edi] for occurance of rax,
    2732 // generic
    2733 void Assembler::repne_scanl() { // repne_scan
    2734   emit_int8((unsigned char)0xF2);
    2735   // SCASL
    2736   emit_int8((unsigned char)0xAF);
    2737 }
    2738 #endif
    2739
    2740 void Assembler::ret(int imm16) {
    2741   if (imm16 == 0) {
    2742     emit_int8((unsigned char)0xC3);
    2743   } else {
    2744     emit_int8((unsigned char)0xC2);
    2745     emit_int16(imm16);
    2746   }
    2747 }
    2748
    2749 void Assembler::sahf() {
    2750 #ifdef _LP64
    2751   // Not supported in 64bit mode
    2752   ShouldNotReachHere();
    2753 #endif
    2754   emit_int8((unsigned char)0x9E);
    2755 }
    2756
    2757 void Assembler::sarl(Register dst, int imm8) {
    2758   int encode = prefix_and_encode(dst->encoding());
    2759   assert(isShiftCount(imm8), "illegal shift count");
    2760   if (imm8 == 1) {
    2761     emit_int8((unsigned char)0xD1);
    2762     emit_int8((unsigned char)(0xF8 | encode));
    2763   } else {
    2764     emit_int8((unsigned char)0xC1);
    2765     emit_int8((unsigned char)(0xF8 | encode));
    2766     emit_int8(imm8);
    2767   }
    2768 }
    2769
    2770 void Assembler::sarl(Register dst) {
    2771   int encode = prefix_and_encode(dst->encoding());
    2772   emit_int8((unsigned char)0xD3);
    2773   emit_int8((unsigned char)(0xF8 | encode));
    2774 }
    2775
    2776 void Assembler::sbbl(Address dst, int32_t imm32) {
    2777   InstructionMark im(this);
    2778   prefix(dst);
    2779   emit_arith_operand(0x81, rbx, dst, imm32);
    2780 }
    2781
    2782 void Assembler::sbbl(Register dst, int32_t imm32) {
    2783   prefix(dst);
    2784   emit_arith(0x81, 0xD8, dst, imm32);
    2785 }
    2786
    2787
    2788 void Assembler::sbbl(Register dst, Address src) {
    2789   InstructionMark im(this);
    2790   prefix(src, dst);
    2791   emit_int8(0x1B);
    2792   emit_operand(dst, src);
    2793 }
    2794
    2795 void Assembler::sbbl(Register dst, Register src) {
    2796   (void) prefix_and_encode(dst->encoding(), src->encoding());
    2797   emit_arith(0x1B, 0xC0, dst, src);
    2798 }
    2799
    2800 void Assembler::setb(Condition cc, Register dst) {
    2801   assert(0 <= cc && cc < 16, "illegal cc");
    2802   int encode = prefix_and_encode(dst->encoding(), true);
    2803   emit_int8(0x0F);
    2804   emit_int8((unsigned char)0x90 | cc);
    2805   emit_int8((unsigned char)(0xC0 | encode));
    2806 }
    2807
    2808 void Assembler::shll(Register dst, int imm8) {
    2809   assert(isShiftCount(imm8), "illegal shift count");
    2810   int encode = prefix_and_encode(dst->encoding());
    2811   if (imm8 == 1 ) {
    2812     emit_int8((unsigned char)0xD1);
    2813     emit_int8((unsigned char)(0xE0 | encode));
    2814   } else {
    2815     emit_int8((unsigned char)0xC1);
    2816     emit_int8((unsigned char)(0xE0 | encode));
    2817     emit_int8(imm8);
    2818   }
    2819 }
    2820
    2821 void Assembler::shll(Register dst) {
    2822   int encode = prefix_and_encode(dst->encoding());
    2823   emit_int8((unsigned char)0xD3);
    2824   emit_int8((unsigned char)(0xE0 | encode));
    2825 }
    2826
    2827 void Assembler::shrl(Register dst, int imm8) {
    2828   assert(isShiftCount(imm8), "illegal shift count");
    2829   int encode = prefix_and_encode(dst->encoding());
    2830   emit_int8((unsigned char)0xC1);
    2831   emit_int8((unsigned char)(0xE8 | encode));
    2832   emit_int8(imm8);
    2833 }
    2834
    2835 void Assembler::shrl(Register dst) {
    2836   int encode = prefix_and_encode(dst->encoding());
    2837   emit_int8((unsigned char)0xD3);
    2838   emit_int8((unsigned char)(0xE8 | encode));
    2839 }
    2840
    2841 // copies a single word from [esi] to [edi]
    2842 void Assembler::smovl() {
    2843   emit_int8((unsigned char)0xA5);
    2844 }
    2845
    2846 void Assembler::sqrtsd(XMMRegister dst, XMMRegister src) {
    2847   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    2848   emit_simd_arith(0x51, dst, src, VEX_SIMD_F2);
    2849 }
    2850
    2851 void Assembler::sqrtsd(XMMRegister dst, Address src) {
    2852   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    2853   emit_simd_arith(0x51, dst, src, VEX_SIMD_F2);
    2854 }
    2855
    2856 void Assembler::sqrtss(XMMRegister dst, XMMRegister src) {
    2857   NOT_LP64(assert(VM_Version::supports_sse(), ""));
    2858   emit_simd_arith(0x51, dst, src, VEX_SIMD_F3);
    2859 }
    2860
    2861 void Assembler::std() {
    2862   emit_int8((unsigned char)0xFD);
    2863 }
    2864
    2865 void Assembler::sqrtss(XMMRegister dst, Address src) {
    2866   NOT_LP64(assert(VM_Version::supports_sse(), ""));
    2867   emit_simd_arith(0x51, dst, src, VEX_SIMD_F3);
    2868 }
    2869
    2870 void Assembler::stmxcsr( Address dst) {
    2871   NOT_LP64(assert(VM_Version::supports_sse(), ""));
    2872   InstructionMark im(this);
    2873   prefix(dst);
    2874   emit_int8(0x0F);
    2875   emit_int8((unsigned char)0xAE);
    2876   emit_operand(as_Register(3), dst);
    2877 }
    2878
    2879 void Assembler::subl(Address dst, int32_t imm32) {
    2880   InstructionMark im(this);
    2881   prefix(dst);
    2882   emit_arith_operand(0x81, rbp, dst, imm32);
    2883 }
    2884
    2885 void Assembler::subl(Address dst, Register src) {
    2886   InstructionMark im(this);
    2887   prefix(dst, src);
    2888   emit_int8(0x29);
    2889   emit_operand(src, dst);
    2890 }
    2891
    2892 void Assembler::subl(Register dst, int32_t imm32) {
    2893   prefix(dst);
    2894   emit_arith(0x81, 0xE8, dst, imm32);
    2895 }
    2896
    2897 // Force generation of a 4 byte immediate value even if it fits into 8bit
    2898 void Assembler::subl_imm32(Register dst, int32_t imm32) {
    2899   prefix(dst);
    2900   emit_arith_imm32(0x81, 0xE8, dst, imm32);
    2901 }
    2902
    2903 void Assembler::subl(Register dst, Address src) {
    2904   InstructionMark im(this);
    2905   prefix(src, dst);
    2906   emit_int8(0x2B);
    2907   emit_operand(dst, src);
    2908 }
    2909
    2910 void Assembler::subl(Register dst, Register src) {
    2911   (void) prefix_and_encode(dst->encoding(), src->encoding());
    2912   emit_arith(0x2B, 0xC0, dst, src);
    2913 }
    2914
    2915 void Assembler::subsd(XMMRegister dst, XMMRegister src) {
    2916   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    2917   emit_simd_arith(0x5C, dst, src, VEX_SIMD_F2);
    2918 }
    2919
    2920 void Assembler::subsd(XMMRegister dst, Address src) {
    2921   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    2922   emit_simd_arith(0x5C, dst, src, VEX_SIMD_F2);
    2923 }
    2924
    2925 void Assembler::subss(XMMRegister dst, XMMRegister src) {
    2926   NOT_LP64(assert(VM_Version::supports_sse(), ""));
    2927   emit_simd_arith(0x5C, dst, src, VEX_SIMD_F3);
    2928 }
    2929
    2930 void Assembler::subss(XMMRegister dst, Address src) {
    2931   NOT_LP64(assert(VM_Version::supports_sse(), ""));
    2932   emit_simd_arith(0x5C, dst, src, VEX_SIMD_F3);
    2933 }
    2934
    2935 void Assembler::testb(Register dst, int imm8) {
    2936   NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
    2937   (void) prefix_and_encode(dst->encoding(), true);
    2938   emit_arith_b(0xF6, 0xC0, dst, imm8);
    2939 }
    2940
    2941 void Assembler::testl(Register dst, int32_t imm32) {
    2942   // not using emit_arith because test
    2943   // doesn't support sign-extension of
    2944   // 8bit operands
    2945   int encode = dst->encoding();
    2946   if (encode == 0) {
    2947     emit_int8((unsigned char)0xA9);
    2948   } else {
    2949     encode = prefix_and_encode(encode);
    2950     emit_int8((unsigned char)0xF7);
    2951     emit_int8((unsigned char)(0xC0 | encode));
    2952   }
    2953   emit_int32(imm32);
    2954 }
    2955
    2956 void Assembler::testl(Register dst, Register src) {
    2957   (void) prefix_and_encode(dst->encoding(), src->encoding());
    2958   emit_arith(0x85, 0xC0, dst, src);
    2959 }
    2960
    2961 void Assembler::testl(Register dst, Address  src) {
    2962   InstructionMark im(this);
    2963   prefix(src, dst);
    2964   emit_int8((unsigned char)0x85);
    2965   emit_operand(dst, src);
    2966 }
    2967
    2968 void Assembler::tzcntl(Register dst, Register src) {
    2969   assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
    2970   emit_int8((unsigned char)0xF3);
    2971   int encode = prefix_and_encode(dst->encoding(), src->encoding());
    2972   emit_int8(0x0F);
    2973   emit_int8((unsigned char)0xBC);
    2974   emit_int8((unsigned char)0xC0 | encode);
    2975 }
    2976
    2977 void Assembler::tzcntq(Register dst, Register src) {
    2978   assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
    2979   emit_int8((unsigned char)0xF3);
    2980   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
    2981   emit_int8(0x0F);
    2982   emit_int8((unsigned char)0xBC);
    2983   emit_int8((unsigned char)(0xC0 | encode));
    2984 }
    2985
    2986 void Assembler::ucomisd(XMMRegister dst, Address src) {
    2987   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    2988   emit_simd_arith_nonds(0x2E, dst, src, VEX_SIMD_66);
    2989 }
    2990
    2991 void Assembler::ucomisd(XMMRegister dst, XMMRegister src) {
    2992   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    2993   emit_simd_arith_nonds(0x2E, dst, src, VEX_SIMD_66);
    2994 }
    2995
    2996 void Assembler::ucomiss(XMMRegister dst, Address src) {
    2997   NOT_LP64(assert(VM_Version::supports_sse(), ""));
    2998   emit_simd_arith_nonds(0x2E, dst, src, VEX_SIMD_NONE);
    2999 }
    3000
    3001 void Assembler::ucomiss(XMMRegister dst, XMMRegister src) {
    3002   NOT_LP64(assert(VM_Version::supports_sse(), ""));
    3003   emit_simd_arith_nonds(0x2E, dst, src, VEX_SIMD_NONE);
    3004 }
    3005
    3006 void Assembler::xabort(int8_t imm8) {
    3007   emit_int8((unsigned char)0xC6);
    3008   emit_int8((unsigned char)0xF8);
    3009   emit_int8((unsigned char)(imm8 & 0xFF));
    3010 }
    3011
    3012 void Assembler::xaddl(Address dst, Register src) {
    3013   InstructionMark im(this);
    3014   prefix(dst, src);
    3015   emit_int8(0x0F);
    3016   emit_int8((unsigned char)0xC1);
    3017   emit_operand(src, dst);
    3018 }
    3019
    3020 void Assembler::xbegin(Label& abort, relocInfo::relocType rtype) {
    3021   InstructionMark im(this);
    3022   relocate(rtype);
    3023   if (abort.is_bound()) {
    3024     address entry = target(abort);
    3025     assert(entry != NULL, "abort entry NULL");
    3026     intptr_t offset = entry - pc();
    3027     emit_int8((unsigned char)0xC7);
    3028     emit_int8((unsigned char)0xF8);
    3029     emit_int32(offset - 6); // 2 opcode + 4 address
    3030   } else {
    3031     abort.add_patch_at(code(), locator());
    3032     emit_int8((unsigned char)0xC7);
    3033     emit_int8((unsigned char)0xF8);
    3034     emit_int32(0);
    3035   }
    3036 }
    3037
    3038 void Assembler::xchgl(Register dst, Address src) { // xchg
    3039   InstructionMark im(this);
    3040   prefix(src, dst);
    3041   emit_int8((unsigned char)0x87);
    3042   emit_operand(dst, src);
    3043 }
    3044
    3045 void Assembler::xchgl(Register dst, Register src) {
    3046   int encode = prefix_and_encode(dst->encoding(), src->encoding());
    3047   emit_int8((unsigned char)0x87);
    3048   emit_int8((unsigned char)(0xC0 | encode));
    3049 }
    3050
    3051 void Assembler::xend() {
    3052   emit_int8((unsigned char)0x0F);
    3053   emit_int8((unsigned char)0x01);
    3054   emit_int8((unsigned char)0xD5);
    3055 }
    3056
    3057 void Assembler::xgetbv() {
    3058   emit_int8(0x0F);
    3059   emit_int8(0x01);
    3060   emit_int8((unsigned char)0xD0);
    3061 }
    3062
    3063 void Assembler::xorl(Register dst, int32_t imm32) {
    3064   prefix(dst);
    3065   emit_arith(0x81, 0xF0, dst, imm32);
    3066 }
    3067
    3068 void Assembler::xorl(Register dst, Address src) {
    3069   InstructionMark im(this);
    3070   prefix(src, dst);
    3071   emit_int8(0x33);
    3072   emit_operand(dst, src);
    3073 }
    3074
    3075 void Assembler::xorl(Register dst, Register src) {
    3076   (void) prefix_and_encode(dst->encoding(), src->encoding());
    3077   emit_arith(0x33, 0xC0, dst, src);
    3078 }
    3079
    3080
    3081 // AVX 3-operands scalar float-point arithmetic instructions
    3082
    3083 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, Address src) {
    3084   assert(VM_Version::supports_avx(), "");
    3085   emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_F2, /* vector256 */ false);
    3086 }
    3087
    3088 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
    3089   assert(VM_Version::supports_avx(), "");
    3090   emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_F2, /* vector256 */ false);
    3091 }
    3092
    3093 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, Address src) {
    3094   assert(VM_Version::supports_avx(), "");
    3095   emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_F3, /* vector256 */ false);
    3096 }
    3097
    3098 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
    3099   assert(VM_Version::supports_avx(), "");
    3100   emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_F3, /* vector256 */ false);
    3101 }
    3102
    3103 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, Address src) {
    3104   assert(VM_Version::supports_avx(), "");
    3105   emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_F2, /* vector256 */ false);
    3106 }
    3107
    3108 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
    3109   assert(VM_Version::supports_avx(), "");
    3110   emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_F2, /* vector256 */ false);
    3111 }
    3112
    3113 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, Address src) {
    3114   assert(VM_Version::supports_avx(), "");
    3115   emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_F3, /* vector256 */ false);
    3116 }
    3117
    3118 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
    3119   assert(VM_Version::supports_avx(), "");
    3120   emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_F3, /* vector256 */ false);
    3121 }
    3122
    3123 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, Address src) {
    3124   assert(VM_Version::supports_avx(), "");
    3125   emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_F2, /* vector256 */ false);
    3126 }
    3127
    3128 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
    3129   assert(VM_Version::supports_avx(), "");
    3130   emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_F2, /* vector256 */ false);
    3131 }
    3132
    3133 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, Address src) {
    3134   assert(VM_Version::supports_avx(), "");
    3135   emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_F3, /* vector256 */ false);
    3136 }
    3137
    3138 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
    3139   assert(VM_Version::supports_avx(), "");
    3140   emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_F3, /* vector256 */ false);
    3141 }
    3142
    3143 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, Address src) {
    3144   assert(VM_Version::supports_avx(), "");
    3145   emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_F2, /* vector256 */ false);
    3146 }
    3147
    3148 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
    3149   assert(VM_Version::supports_avx(), "");
    3150   emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_F2, /* vector256 */ false);
    3151 }
    3152
    3153 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, Address src) {
    3154   assert(VM_Version::supports_avx(), "");
    3155   emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_F3, /* vector256 */ false);
    3156 }
    3157
    3158 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
    3159   assert(VM_Version::supports_avx(), "");
    3160   emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_F3, /* vector256 */ false);
    3161 }
    3162
    3163 //====================VECTOR ARITHMETIC=====================================
    3164
    3165 // Float-point vector arithmetic
    3166
    3167 void Assembler::addpd(XMMRegister dst, XMMRegister src) {
    3168   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    3169   emit_simd_arith(0x58, dst, src, VEX_SIMD_66);
    3170 }
    3171
    3172 void Assembler::addps(XMMRegister dst, XMMRegister src) {
    3173   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    3174   emit_simd_arith(0x58, dst, src, VEX_SIMD_NONE);
    3175 }
    3176
    3177 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
    3178   assert(VM_Version::supports_avx(), "");
    3179   emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_66, vector256);
    3180 }
    3181
    3182 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
    3183   assert(VM_Version::supports_avx(), "");
    3184   emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_NONE, vector256);
    3185 }
    3186
    3187 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
    3188   assert(VM_Version::supports_avx(), "");
    3189   emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_66, vector256);
    3190 }
    3191
    3192 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
    3193   assert(VM_Version::supports_avx(), "");
    3194   emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_NONE, vector256);
    3195 }
    3196
    3197 void Assembler::subpd(XMMRegister dst, XMMRegister src) {
    3198   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    3199   emit_simd_arith(0x5C, dst, src, VEX_SIMD_66);
    3200 }
    3201
    3202 void Assembler::subps(XMMRegister dst, XMMRegister src) {
    3203   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    3204   emit_simd_arith(0x5C, dst, src, VEX_SIMD_NONE);
    3205 }
    3206
    3207 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
    3208   assert(VM_Version::supports_avx(), "");
    3209   emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_66, vector256);
    3210 }
    3211
    3212 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
    3213   assert(VM_Version::supports_avx(), "");
    3214   emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_NONE, vector256);
    3215 }
    3216
    3217 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
    3218   assert(VM_Version::supports_avx(), "");
    3219   emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_66, vector256);
    3220 }
    3221
    3222 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
    3223   assert(VM_Version::supports_avx(), "");
    3224   emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_NONE, vector256);
    3225 }
    3226
    3227 void Assembler::mulpd(XMMRegister dst, XMMRegister src) {
    3228   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    3229   emit_simd_arith(0x59, dst, src, VEX_SIMD_66);
    3230 }
    3231
    3232 void Assembler::mulps(XMMRegister dst, XMMRegister src) {
    3233   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    3234   emit_simd_arith(0x59, dst, src, VEX_SIMD_NONE);
    3235 }
    3236
    3237 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
    3238   assert(VM_Version::supports_avx(), "");
    3239   emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_66, vector256);
    3240 }
    3241
    3242 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
    3243   assert(VM_Version::supports_avx(), "");
    3244   emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_NONE, vector256);
    3245 }
    3246
    3247 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
    3248   assert(VM_Version::supports_avx(), "");
    3249   emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_66, vector256);
    3250 }
    3251
    3252 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
    3253   assert(VM_Version::supports_avx(), "");
    3254   emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_NONE, vector256);
    3255 }
    3256
    3257 void Assembler::divpd(XMMRegister dst, XMMRegister src) {
    3258   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    3259   emit_simd_arith(0x5E, dst, src, VEX_SIMD_66);
    3260 }
    3261
    3262 void Assembler::divps(XMMRegister dst, XMMRegister src) {
    3263   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    3264   emit_simd_arith(0x5E, dst, src, VEX_SIMD_NONE);
    3265 }
    3266
    3267 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
    3268   assert(VM_Version::supports_avx(), "");
    3269   emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_66, vector256);
    3270 }
    3271
    3272 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
    3273   assert(VM_Version::supports_avx(), "");
    3274   emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_NONE, vector256);
    3275 }
    3276
    3277 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
    3278   assert(VM_Version::supports_avx(), "");
    3279   emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_66, vector256);
    3280 }
    3281
    3282 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
    3283   assert(VM_Version::supports_avx(), "");
    3284   emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_NONE, vector256);
    3285 }
    3286
    3287 void Assembler::andpd(XMMRegister dst, XMMRegister src) {
    3288   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    3289   emit_simd_arith(0x54, dst, src, VEX_SIMD_66);
    3290 }
    3291
    3292 void Assembler::andps(XMMRegister dst, XMMRegister src) {
    3293   NOT_LP64(assert(VM_Version::supports_sse(), ""));
    3294   emit_simd_arith(0x54, dst, src, VEX_SIMD_NONE);
    3295 }
    3296
    3297 void Assembler::andps(XMMRegister dst, Address src) {
    3298   NOT_LP64(assert(VM_Version::supports_sse(), ""));
    3299   emit_simd_arith(0x54, dst, src, VEX_SIMD_NONE);
    3300 }
    3301
    3302 void Assembler::andpd(XMMRegister dst, Address src) {
    3303   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    3304   emit_simd_arith(0x54, dst, src, VEX_SIMD_66);
    3305 }
    3306
    3307 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
    3308   assert(VM_Version::supports_avx(), "");
    3309   emit_vex_arith(0x54, dst, nds, src, VEX_SIMD_66, vector256);
    3310 }
    3311
    3312 void Assembler::vandps(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
    3313   assert(VM_Version::supports_avx(), "");
    3314   emit_vex_arith(0x54, dst, nds, src, VEX_SIMD_NONE, vector256);
    3315 }
    3316
    3317 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
    3318   assert(VM_Version::supports_avx(), "");
    3319   emit_vex_arith(0x54, dst, nds, src, VEX_SIMD_66, vector256);
    3320 }
    3321
    3322 void Assembler::vandps(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
    3323   assert(VM_Version::supports_avx(), "");
    3324   emit_vex_arith(0x54, dst, nds, src, VEX_SIMD_NONE, vector256);
    3325 }
    3326
    3327 void Assembler::xorpd(XMMRegister dst, XMMRegister src) {
    3328   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    3329   emit_simd_arith(0x57, dst, src, VEX_SIMD_66);
    3330 }
    3331
    3332 void Assembler::xorps(XMMRegister dst, XMMRegister src) {
    3333   NOT_LP64(assert(VM_Version::supports_sse(), ""));
    3334   emit_simd_arith(0x57, dst, src, VEX_SIMD_NONE);
    3335 }
    3336
    3337 void Assembler::xorpd(XMMRegister dst, Address src) {
    3338   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    3339   emit_simd_arith(0x57, dst, src, VEX_SIMD_66);
    3340 }
    3341
    3342 void Assembler::xorps(XMMRegister dst, Address src) {
    3343   NOT_LP64(assert(VM_Version::supports_sse(), ""));
    3344   emit_simd_arith(0x57, dst, src, VEX_SIMD_NONE);
    3345 }
    3346
    3347 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
    3348   assert(VM_Version::supports_avx(), "");
    3349   emit_vex_arith(0x57, dst, nds, src, VEX_SIMD_66, vector256);
    3350 }
    3351
    3352 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
    3353   assert(VM_Version::supports_avx(), "");
    3354   emit_vex_arith(0x57, dst, nds, src, VEX_SIMD_NONE, vector256);
    3355 }
    3356
    3357 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
    3358   assert(VM_Version::supports_avx(), "");
    3359   emit_vex_arith(0x57, dst, nds, src, VEX_SIMD_66, vector256);
    3360 }
    3361
    3362 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
    3363   assert(VM_Version::supports_avx(), "");
    3364   emit_vex_arith(0x57, dst, nds, src, VEX_SIMD_NONE, vector256);
    3365 }
    3366
    3367
    3368 // Integer vector arithmetic
    3369 void Assembler::paddb(XMMRegister dst, XMMRegister src) {
    3370   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    3371   emit_simd_arith(0xFC, dst, src, VEX_SIMD_66);
    3372 }
    3373
    3374 void Assembler::paddw(XMMRegister dst, XMMRegister src) {
    3375   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    3376   emit_simd_arith(0xFD, dst, src, VEX_SIMD_66);
    3377 }
    3378
    3379 void Assembler::paddd(XMMRegister dst, XMMRegister src) {
    3380   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    3381   emit_simd_arith(0xFE, dst, src, VEX_SIMD_66);
    3382 }
    3383
    3384 void Assembler::paddq(XMMRegister dst, XMMRegister src) {
    3385   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    3386   emit_simd_arith(0xD4, dst, src, VEX_SIMD_66);
    3387 }
    3388
    3389 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
    3390   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
    3391   emit_vex_arith(0xFC, dst, nds, src, VEX_SIMD_66, vector256);
    3392 }
    3393
    3394 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
    3395   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
    3396   emit_vex_arith(0xFD, dst, nds, src, VEX_SIMD_66, vector256);
    3397 }
    3398
    3399 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
    3400   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
    3401   emit_vex_arith(0xFE, dst, nds, src, VEX_SIMD_66, vector256);
    3402 }
    3403
    3404 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
    3405   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
    3406   emit_vex_arith(0xD4, dst, nds, src, VEX_SIMD_66, vector256);
    3407 }
    3408
    3409 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
    3410   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
    3411   emit_vex_arith(0xFC, dst, nds, src, VEX_SIMD_66, vector256);
    3412 }
    3413
    3414 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
    3415   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
    3416   emit_vex_arith(0xFD, dst, nds, src, VEX_SIMD_66, vector256);
    3417 }
    3418
    3419 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
    3420   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
    3421   emit_vex_arith(0xFE, dst, nds, src, VEX_SIMD_66, vector256);
    3422 }
    3423
    3424 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
    3425   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
    3426   emit_vex_arith(0xD4, dst, nds, src, VEX_SIMD_66, vector256);
    3427 }
    3428
    3429 void Assembler::psubb(XMMRegister dst, XMMRegister src) {
    3430   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    3431   emit_simd_arith(0xF8, dst, src, VEX_SIMD_66);
    3432 }
    3433
    3434 void Assembler::psubw(XMMRegister dst, XMMRegister src) {
    3435   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    3436   emit_simd_arith(0xF9, dst, src, VEX_SIMD_66);
    3437 }
    3438
    3439 void Assembler::psubd(XMMRegister dst, XMMRegister src) {
    3440   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    3441   emit_simd_arith(0xFA, dst, src, VEX_SIMD_66);
    3442 }
    3443
    3444 void Assembler::psubq(XMMRegister dst, XMMRegister src) {
    3445   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    3446   emit_simd_arith(0xFB, dst, src, VEX_SIMD_66);
    3447 }
    3448
    3449 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
    3450   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
    3451   emit_vex_arith(0xF8, dst, nds, src, VEX_SIMD_66, vector256);
    3452 }
    3453
    3454 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
    3455   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
    3456   emit_vex_arith(0xF9, dst, nds, src, VEX_SIMD_66, vector256);
    3457 }
    3458
    3459 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
    3460   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
    3461   emit_vex_arith(0xFA, dst, nds, src, VEX_SIMD_66, vector256);
    3462 }
    3463
    3464 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
    3465   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
    3466   emit_vex_arith(0xFB, dst, nds, src, VEX_SIMD_66, vector256);
    3467 }
    3468
    3469 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
    3470   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
    3471   emit_vex_arith(0xF8, dst, nds, src, VEX_SIMD_66, vector256);
    3472 }
    3473
    3474 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
    3475   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
    3476   emit_vex_arith(0xF9, dst, nds, src, VEX_SIMD_66, vector256);
    3477 }
    3478
    3479 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
    3480   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
    3481   emit_vex_arith(0xFA, dst, nds, src, VEX_SIMD_66, vector256);
    3482 }
    3483
    3484 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
    3485   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
    3486   emit_vex_arith(0xFB, dst, nds, src, VEX_SIMD_66, vector256);
    3487 }
    3488
    3489 void Assembler::pmullw(XMMRegister dst, XMMRegister src) {
    3490   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    3491   emit_simd_arith(0xD5, dst, src, VEX_SIMD_66);
    3492 }
    3493
    3494 void Assembler::pmulld(XMMRegister dst, XMMRegister src) {
    3495   assert(VM_Version::supports_sse4_1(), "");
    3496   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
    3497   emit_int8(0x40);
    3498   emit_int8((unsigned char)(0xC0 | encode));
    3499 }
    3500
    3501 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
    3502   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
    3503   emit_vex_arith(0xD5, dst, nds, src, VEX_SIMD_66, vector256);
    3504 }
    3505
    3506 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
    3507   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
    3508   int encode = vex_prefix_and_encode(dst, nds, src, VEX_SIMD_66, vector256, VEX_OPCODE_0F_38);
    3509   emit_int8(0x40);
    3510   emit_int8((unsigned char)(0xC0 | encode));
    3511 }
    3512
    3513 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
    3514   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
    3515   emit_vex_arith(0xD5, dst, nds, src, VEX_SIMD_66, vector256);
    3516 }
    3517
    3518 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
    3519   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
    3520   InstructionMark im(this);
    3521   int dst_enc = dst->encoding();
    3522   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
    3523   vex_prefix(src, nds_enc, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, false, vector256);
    3524   emit_int8(0x40);
    3525   emit_operand(dst, src);
    3526 }
    3527
    3528 // Shift packed integers left by specified number of bits.
    3529 void Assembler::psllw(XMMRegister dst, int shift) {
    3530   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    3531   // XMM6 is for /6 encoding: 66 0F 71 /6 ib
    3532   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66);
    3533   emit_int8(0x71);
    3534   emit_int8((unsigned char)(0xC0 | encode));
    3535   emit_int8(shift & 0xFF);
    3536 }
    3537
    3538 void Assembler::pslld(XMMRegister dst, int shift) {
    3539   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    3540   // XMM6 is for /6 encoding: 66 0F 72 /6 ib
    3541   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66);
    3542   emit_int8(0x72);
    3543   emit_int8((unsigned char)(0xC0 | encode));
    3544   emit_int8(shift & 0xFF);
    3545 }
    3546
    3547 void Assembler::psllq(XMMRegister dst, int shift) {
    3548   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    3549   // XMM6 is for /6 encoding: 66 0F 73 /6 ib
    3550   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66);
    3551   emit_int8(0x73);
    3552   emit_int8((unsigned char)(0xC0 | encode));
    3553   emit_int8(shift & 0xFF);
    3554 }
    3555
    3556 void Assembler::psllw(XMMRegister dst, XMMRegister shift) {
    3557   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    3558   emit_simd_arith(0xF1, dst, shift, VEX_SIMD_66);
    3559 }
    3560
    3561 void Assembler::pslld(XMMRegister dst, XMMRegister shift) {
    3562   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    3563   emit_simd_arith(0xF2, dst, shift, VEX_SIMD_66);
    3564 }
    3565
    3566 void Assembler::psllq(XMMRegister dst, XMMRegister shift) {
    3567   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    3568   emit_simd_arith(0xF3, dst, shift, VEX_SIMD_66);
    3569 }
    3570
    3571 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, int shift, bool vector256) {
    3572   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
    3573   // XMM6 is for /6 encoding: 66 0F 71 /6 ib
    3574   emit_vex_arith(0x71, xmm6, dst, src, VEX_SIMD_66, vector256);
    3575   emit_int8(shift & 0xFF);
    3576 }
    3577
    3578 void Assembler::vpslld(XMMRegister dst, XMMRegister src, int shift, bool vector256) {
    3579   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
    3580   // XMM6 is for /6 encoding: 66 0F 72 /6 ib
    3581   emit_vex_arith(0x72, xmm6, dst, src, VEX_SIMD_66, vector256);
    3582   emit_int8(shift & 0xFF);
    3583 }
    3584
    3585 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, int shift, bool vector256) {
    3586   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
    3587   // XMM6 is for /6 encoding: 66 0F 73 /6 ib
    3588   emit_vex_arith(0x73, xmm6, dst, src, VEX_SIMD_66, vector256);
    3589   emit_int8(shift & 0xFF);
    3590 }
    3591
    3592 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, XMMRegister shift, bool vector256) {
    3593   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
    3594   emit_vex_arith(0xF1, dst, src, shift, VEX_SIMD_66, vector256);
    3595 }
    3596
    3597 void Assembler::vpslld(XMMRegister dst, XMMRegister src, XMMRegister shift, bool vector256) {
    3598   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
    3599   emit_vex_arith(0xF2, dst, src, shift, VEX_SIMD_66, vector256);
    3600 }
    3601
    3602 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, XMMRegister shift, bool vector256) {
    3603   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
    3604   emit_vex_arith(0xF3, dst, src, shift, VEX_SIMD_66, vector256);
    3605 }
    3606
    3607 // Shift packed integers logically right by specified number of bits.
    3608 void Assembler::psrlw(XMMRegister dst, int shift) {
    3609   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    3610   // XMM2 is for /2 encoding: 66 0F 71 /2 ib
    3611   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66);
    3612   emit_int8(0x71);
    3613   emit_int8((unsigned char)(0xC0 | encode));
    3614   emit_int8(shift & 0xFF);
    3615 }
    3616
    3617 void Assembler::psrld(XMMRegister dst, int shift) {
    3618   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    3619   // XMM2 is for /2 encoding: 66 0F 72 /2 ib
    3620   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66);
    3621   emit_int8(0x72);
    3622   emit_int8((unsigned char)(0xC0 | encode));
    3623   emit_int8(shift & 0xFF);
    3624 }
    3625
    3626 void Assembler::psrlq(XMMRegister dst, int shift) {
    3627   // Do not confuse it with psrldq SSE2 instruction which
    3628   // shifts 128 bit value in xmm register by number of bytes.
    3629   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    3630   // XMM2 is for /2 encoding: 66 0F 73 /2 ib
    3631   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66);
    3632   emit_int8(0x73);
    3633   emit_int8((unsigned char)(0xC0 | encode));
    3634   emit_int8(shift & 0xFF);
    3635 }
    3636
    3637 void Assembler::psrlw(XMMRegister dst, XMMRegister shift) {
    3638   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    3639   emit_simd_arith(0xD1, dst, shift, VEX_SIMD_66);
    3640 }
    3641
    3642 void Assembler::psrld(XMMRegister dst, XMMRegister shift) {
    3643   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    3644   emit_simd_arith(0xD2, dst, shift, VEX_SIMD_66);
    3645 }
    3646
    3647 void Assembler::psrlq(XMMRegister dst, XMMRegister shift) {
    3648   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    3649   emit_simd_arith(0xD3, dst, shift, VEX_SIMD_66);
    3650 }
    3651
    3652 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, int shift, bool vector256) {
    3653   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
    3654   // XMM2 is for /2 encoding: 66 0F 73 /2 ib
    3655   emit_vex_arith(0x71, xmm2, dst, src, VEX_SIMD_66, vector256);
    3656   emit_int8(shift & 0xFF);
    3657 }
    3658
    3659 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, int shift, bool vector256) {
    3660   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
    3661   // XMM2 is for /2 encoding: 66 0F 73 /2 ib
    3662   emit_vex_arith(0x72, xmm2, dst, src, VEX_SIMD_66, vector256);
    3663   emit_int8(shift & 0xFF);
    3664 }
    3665
    3666 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, int shift, bool vector256) {
    3667   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
    3668   // XMM2 is for /2 encoding: 66 0F 73 /2 ib
    3669   emit_vex_arith(0x73, xmm2, dst, src, VEX_SIMD_66, vector256);
    3670   emit_int8(shift & 0xFF);
    3671 }
    3672
    3673 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, XMMRegister shift, bool vector256) {
    3674   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
    3675   emit_vex_arith(0xD1, dst, src, shift, VEX_SIMD_66, vector256);
    3676 }
    3677
    3678 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, XMMRegister shift, bool vector256) {
    3679   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
    3680   emit_vex_arith(0xD2, dst, src, shift, VEX_SIMD_66, vector256);
    3681 }
    3682
    3683 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, XMMRegister shift, bool vector256) {
    3684   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
    3685   emit_vex_arith(0xD3, dst, src, shift, VEX_SIMD_66, vector256);
    3686 }
    3687
    3688 // Shift packed integers arithmetically right by specified number of bits.
    3689 void Assembler::psraw(XMMRegister dst, int shift) {
    3690   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    3691   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
    3692   int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66);
    3693   emit_int8(0x71);
    3694   emit_int8((unsigned char)(0xC0 | encode));
    3695   emit_int8(shift & 0xFF);
    3696 }
    3697
    3698 void Assembler::psrad(XMMRegister dst, int shift) {
    3699   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    3700   // XMM4 is for /4 encoding: 66 0F 72 /4 ib
    3701   int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66);
    3702   emit_int8(0x72);
    3703   emit_int8((unsigned char)(0xC0 | encode));
    3704   emit_int8(shift & 0xFF);
    3705 }
    3706
    3707 void Assembler::psraw(XMMRegister dst, XMMRegister shift) {
    3708   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    3709   emit_simd_arith(0xE1, dst, shift, VEX_SIMD_66);
    3710 }
    3711
    3712 void Assembler::psrad(XMMRegister dst, XMMRegister shift) {
    3713   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    3714   emit_simd_arith(0xE2, dst, shift, VEX_SIMD_66);
    3715 }
    3716
    3717 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, int shift, bool vector256) {
    3718   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
    3719   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
    3720   emit_vex_arith(0x71, xmm4, dst, src, VEX_SIMD_66, vector256);
    3721   emit_int8(shift & 0xFF);
    3722 }
    3723
    3724 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, int shift, bool vector256) {
    3725   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
    3726   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
    3727   emit_vex_arith(0x72, xmm4, dst, src, VEX_SIMD_66, vector256);
    3728   emit_int8(shift & 0xFF);
    3729 }
    3730
    3731 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, XMMRegister shift, bool vector256) {
    3732   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
    3733   emit_vex_arith(0xE1, dst, src, shift, VEX_SIMD_66, vector256);
    3734 }
    3735
    3736 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, XMMRegister shift, bool vector256) {
    3737   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
    3738   emit_vex_arith(0xE2, dst, src, shift, VEX_SIMD_66, vector256);
    3739 }
    3740
    3741
    3742 // AND packed integers
    3743 void Assembler::pand(XMMRegister dst, XMMRegister src) {
    3744   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    3745   emit_simd_arith(0xDB, dst, src, VEX_SIMD_66);
    3746 }
    3747
    3748 void Assembler::vpand(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
    3749   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
    3750   emit_vex_arith(0xDB, dst, nds, src, VEX_SIMD_66, vector256);
    3751 }
    3752
    3753 void Assembler::vpand(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
    3754   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
    3755   emit_vex_arith(0xDB, dst, nds, src, VEX_SIMD_66, vector256);
    3756 }
    3757
    3758 void Assembler::por(XMMRegister dst, XMMRegister src) {
    3759   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    3760   emit_simd_arith(0xEB, dst, src, VEX_SIMD_66);
    3761 }
    3762
    3763 void Assembler::vpor(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
    3764   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
    3765   emit_vex_arith(0xEB, dst, nds, src, VEX_SIMD_66, vector256);
    3766 }
    3767
    3768 void Assembler::vpor(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
    3769   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
    3770   emit_vex_arith(0xEB, dst, nds, src, VEX_SIMD_66, vector256);
    3771 }
    3772
    3773 void Assembler::pxor(XMMRegister dst, XMMRegister src) {
    3774   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    3775   emit_simd_arith(0xEF, dst, src, VEX_SIMD_66);
    3776 }
    3777
    3778 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
    3779   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
    3780   emit_vex_arith(0xEF, dst, nds, src, VEX_SIMD_66, vector256);
    3781 }
    3782
    3783 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
    3784   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
    3785   emit_vex_arith(0xEF, dst, nds, src, VEX_SIMD_66, vector256);
    3786 }
    3787
    3788
    3789 void Assembler::vinsertf128h(XMMRegister dst, XMMRegister nds, XMMRegister src) {
    3790   assert(VM_Version::supports_avx(), "");
    3791   bool vector256 = true;
    3792   int encode = vex_prefix_and_encode(dst, nds, src, VEX_SIMD_66, vector256, VEX_OPCODE_0F_3A);
    3793   emit_int8(0x18);
    3794   emit_int8((unsigned char)(0xC0 | encode));
    3795   // 0x00 - insert into lower 128 bits
    3796   // 0x01 - insert into upper 128 bits
    3797   emit_int8(0x01);
    3798 }
    3799
    3800 void Assembler::vinsertf128h(XMMRegister dst, Address src) {
    3801   assert(VM_Version::supports_avx(), "");
    3802   InstructionMark im(this);
    3803   bool vector256 = true;
    3804   assert(dst != xnoreg, "sanity");
    3805   int dst_enc = dst->encoding();
    3806   // swap src<->dst for encoding
    3807   vex_prefix(src, dst_enc, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, false, vector256);
    3808   emit_int8(0x18);
    3809   emit_operand(dst, src);
    3810   // 0x01 - insert into upper 128 bits
    3811   emit_int8(0x01);
    3812 }
    3813
    3814 void Assembler::vextractf128h(Address dst, XMMRegister src) {
    3815   assert(VM_Version::supports_avx(), "");
    3816   InstructionMark im(this);
    3817   bool vector256 = true;
    3818   assert(src != xnoreg, "sanity");
    3819   int src_enc = src->encoding();
    3820   vex_prefix(dst, 0, src_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, false, vector256);
    3821   emit_int8(0x19);
    3822   emit_operand(src, dst);
    3823   // 0x01 - extract from upper 128 bits
    3824   emit_int8(0x01);
    3825 }
    3826
    3827 void Assembler::vinserti128h(XMMRegister dst, XMMRegister nds, XMMRegister src) {
    3828   assert(VM_Version::supports_avx2(), "");
    3829   bool vector256 = true;
    3830   int encode = vex_prefix_and_encode(dst, nds, src, VEX_SIMD_66, vector256, VEX_OPCODE_0F_3A);
    3831   emit_int8(0x38);
    3832   emit_int8((unsigned char)(0xC0 | encode));
    3833   // 0x00 - insert into lower 128 bits
    3834   // 0x01 - insert into upper 128 bits
    3835   emit_int8(0x01);
    3836 }
    3837
    3838 void Assembler::vinserti128h(XMMRegister dst, Address src) {
    3839   assert(VM_Version::supports_avx2(), "");
    3840   InstructionMark im(this);
    3841   bool vector256 = true;
    3842   assert(dst != xnoreg, "sanity");
    3843   int dst_enc = dst->encoding();
    3844   // swap src<->dst for encoding
    3845   vex_prefix(src, dst_enc, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, false, vector256);
    3846   emit_int8(0x38);
    3847   emit_operand(dst, src);
    3848   // 0x01 - insert into upper 128 bits
    3849   emit_int8(0x01);
    3850 }
    3851
    3852 void Assembler::vextracti128h(Address dst, XMMRegister src) {
    3853   assert(VM_Version::supports_avx2(), "");
    3854   InstructionMark im(this);
    3855   bool vector256 = true;
    3856   assert(src != xnoreg, "sanity");
    3857   int src_enc = src->encoding();
    3858   vex_prefix(dst, 0, src_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, false, vector256);
    3859   emit_int8(0x39);
    3860   emit_operand(src, dst);
    3861   // 0x01 - extract from upper 128 bits
    3862   emit_int8(0x01);
    3863 }
    3864
    3865 // duplicate 4-bytes integer data from src into 8 locations in dest
    3866 void Assembler::vpbroadcastd(XMMRegister dst, XMMRegister src) {
    3867   assert(VM_Version::supports_avx2(), "");
    3868   bool vector256 = true;
    3869   int encode = vex_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, vector256, VEX_OPCODE_0F_38);
    3870   emit_int8(0x58);
    3871   emit_int8((unsigned char)(0xC0 | encode));
    3872 }
    3873
    3874 // Carry-Less Multiplication Quadword
    3875 void Assembler::pclmulqdq(XMMRegister dst, XMMRegister src, int mask) {
    3876   assert(VM_Version::supports_clmul(), "");
    3877   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A);
    3878   emit_int8(0x44);
    3879   emit_int8((unsigned char)(0xC0 | encode));
    3880   emit_int8((unsigned char)mask);
    3881 }
    3882
    3883 // Carry-Less Multiplication Quadword
    3884 void Assembler::vpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask) {
    3885   assert(VM_Version::supports_avx() && VM_Version::supports_clmul(), "");
    3886   bool vector256 = false;
    3887   int encode = vex_prefix_and_encode(dst, nds, src, VEX_SIMD_66, vector256, VEX_OPCODE_0F_3A);
    3888   emit_int8(0x44);
    3889   emit_int8((unsigned char)(0xC0 | encode));
    3890   emit_int8((unsigned char)mask);
    3891 }
    3892
    3893 void Assembler::vzeroupper() {
    3894   assert(VM_Version::supports_avx(), "");
    3895   (void)vex_prefix_and_encode(xmm0, xmm0, xmm0, VEX_SIMD_NONE);
    3896   emit_int8(0x77);
    3897 }
    3898
    3899
    3900 #ifndef _LP64
    3901 // 32bit only pieces of the assembler
    3902
    3903 void Assembler::cmp_literal32(Register src1, int32_t imm32, RelocationHolder const& rspec) {
    3904   // NO PREFIX AS NEVER 64BIT
    3905   InstructionMark im(this);
    3906   emit_int8((unsigned char)0x81);
    3907   emit_int8((unsigned char)(0xF8 | src1->encoding()));
    3908   emit_data(imm32, rspec, 0);
    3909 }
    3910
    3911 void Assembler::cmp_literal32(Address src1, int32_t imm32, RelocationHolder const& rspec) {
    3912   // NO PREFIX AS NEVER 64BIT (not even 32bit versions of 64bit regs
    3913   InstructionMark im(this);
    3914   emit_int8((unsigned char)0x81);
    3915   emit_operand(rdi, src1);
    3916   emit_data(imm32, rspec, 0);
    3917 }
    3918
    3919 // The 64-bit (32bit platform) cmpxchg compares the value at adr with the contents of rdx:rax,
    3920 // and stores rcx:rbx into adr if so; otherwise, the value at adr is loaded
    3921 // into rdx:rax.  The ZF is set if the compared values were equal, and cleared otherwise.
    3922 void Assembler::cmpxchg8(Address adr) {
    3923   InstructionMark im(this);
    3924   emit_int8(0x0F);
    3925   emit_int8((unsigned char)0xC7);
    3926   emit_operand(rcx, adr);
    3927 }
    3928
    3929 void Assembler::decl(Register dst) {
    3930   // Don't use it directly. Use MacroAssembler::decrementl() instead.
    3931  emit_int8(0x48 | dst->encoding());
    3932 }
    3933
    3934 #endif // _LP64
    3935
    3936 // 64bit typically doesn't use the x87 but needs to for the trig funcs
    3937
    3938 void Assembler::fabs() {
    3939   emit_int8((unsigned char)0xD9);
    3940   emit_int8((unsigned char)0xE1);
    3941 }
    3942
    3943 void Assembler::fadd(int i) {
    3944   emit_farith(0xD8, 0xC0, i);
    3945 }
    3946
    3947 void Assembler::fadd_d(Address src) {
    3948   InstructionMark im(this);
    3949   emit_int8((unsigned char)0xDC);
    3950   emit_operand32(rax, src);
    3951 }
    3952
    3953 void Assembler::fadd_s(Address src) {
    3954   InstructionMark im(this);
    3955   emit_int8((unsigned char)0xD8);
    3956   emit_operand32(rax, src);
    3957 }
    3958
    3959 void Assembler::fadda(int i) {
    3960   emit_farith(0xDC, 0xC0, i);
    3961 }
    3962
    3963 void Assembler::faddp(int i) {
    3964   emit_farith(0xDE, 0xC0, i);
    3965 }
    3966
    3967 void Assembler::fchs() {
    3968   emit_int8((unsigned char)0xD9);
    3969   emit_int8((unsigned char)0xE0);
    3970 }
    3971
    3972 void Assembler::fcom(int i) {
    3973   emit_farith(0xD8, 0xD0, i);
    3974 }
    3975
    3976 void Assembler::fcomp(int i) {
    3977   emit_farith(0xD8, 0xD8, i);
    3978 }
    3979
    3980 void Assembler::fcomp_d(Address src) {
    3981   InstructionMark im(this);
    3982   emit_int8((unsigned char)0xDC);
    3983   emit_operand32(rbx, src);
    3984 }
    3985
    3986 void Assembler::fcomp_s(Address src) {
    3987   InstructionMark im(this);
    3988   emit_int8((unsigned char)0xD8);
    3989   emit_operand32(rbx, src);
    3990 }
    3991
    3992 void Assembler::fcompp() {
    3993   emit_int8((unsigned char)0xDE);
    3994   emit_int8((unsigned char)0xD9);
    3995 }
    3996
    3997 void Assembler::fcos() {
    3998   emit_int8((unsigned char)0xD9);
    3999   emit_int8((unsigned char)0xFF);
    4000 }
    4001
    4002 void Assembler::fdecstp() {
    4003   emit_int8((unsigned char)0xD9);
    4004   emit_int8((unsigned char)0xF6);
    4005 }
    4006
    4007 void Assembler::fdiv(int i) {
    4008   emit_farith(0xD8, 0xF0, i);
    4009 }
    4010
    4011 void Assembler::fdiv_d(Address src) {
    4012   InstructionMark im(this);
    4013   emit_int8((unsigned char)0xDC);
    4014   emit_operand32(rsi, src);
    4015 }
    4016
    4017 void Assembler::fdiv_s(Address src) {
    4018   InstructionMark im(this);
    4019   emit_int8((unsigned char)0xD8);
    4020   emit_operand32(rsi, src);
    4021 }
    4022
    4023 void Assembler::fdiva(int i) {
    4024   emit_farith(0xDC, 0xF8, i);
    4025 }
    4026
    4027 // Note: The Intel manual (Pentium Processor User's Manual, Vol.3, 1994)
    4028 //       is erroneous for some of the floating-point instructions below.
    4029
    4030 void Assembler::fdivp(int i) {
    4031   emit_farith(0xDE, 0xF8, i);                    // ST(0) <- ST(0) / ST(1) and pop (Intel manual wrong)
    4032 }
    4033
    4034 void Assembler::fdivr(int i) {
    4035   emit_farith(0xD8, 0xF8, i);
    4036 }
    4037
    4038 void Assembler::fdivr_d(Address src) {
    4039   InstructionMark im(this);
    4040   emit_int8((unsigned char)0xDC);
    4041   emit_operand32(rdi, src);
    4042 }
    4043
    4044 void Assembler::fdivr_s(Address src) {
    4045   InstructionMark im(this);
    4046   emit_int8((unsigned char)0xD8);
    4047   emit_operand32(rdi, src);
    4048 }
    4049
    4050 void Assembler::fdivra(int i) {
    4051   emit_farith(0xDC, 0xF0, i);
    4052 }
    4053
    4054 void Assembler::fdivrp(int i) {
    4055   emit_farith(0xDE, 0xF0, i);                    // ST(0) <- ST(1) / ST(0) and pop (Intel manual wrong)
    4056 }
    4057
    4058 void Assembler::ffree(int i) {
    4059   emit_farith(0xDD, 0xC0, i);
    4060 }
    4061
    4062 void Assembler::fild_d(Address adr) {
    4063   InstructionMark im(this);
    4064   emit_int8((unsigned char)0xDF);
    4065   emit_operand32(rbp, adr);
    4066 }
    4067
    4068 void Assembler::fild_s(Address adr) {
    4069   InstructionMark im(this);
    4070   emit_int8((unsigned char)0xDB);
    4071   emit_operand32(rax, adr);
    4072 }
    4073
    4074 void Assembler::fincstp() {
    4075   emit_int8((unsigned char)0xD9);
    4076   emit_int8((unsigned char)0xF7);
    4077 }
    4078
    4079 void Assembler::finit() {
    4080   emit_int8((unsigned char)0x9B);
    4081   emit_int8((unsigned char)0xDB);
    4082   emit_int8((unsigned char)0xE3);
    4083 }
    4084
    4085 void Assembler::fist_s(Address adr) {
    4086   InstructionMark im(this);
    4087   emit_int8((unsigned char)0xDB);
    4088   emit_operand32(rdx, adr);
    4089 }
    4090
    4091 void Assembler::fistp_d(Address adr) {
    4092   InstructionMark im(this);
    4093   emit_int8((unsigned char)0xDF);
    4094   emit_operand32(rdi, adr);
    4095 }
    4096
    4097 void Assembler::fistp_s(Address adr) {
    4098   InstructionMark im(this);
    4099   emit_int8((unsigned char)0xDB);
    4100   emit_operand32(rbx, adr);
    4101 }
    4102
    4103 void Assembler::fld1() {
    4104   emit_int8((unsigned char)0xD9);
    4105   emit_int8((unsigned char)0xE8);
    4106 }
    4107
    4108 void Assembler::fld_d(Address adr) {
    4109   InstructionMark im(this);
    4110   emit_int8((unsigned char)0xDD);
    4111   emit_operand32(rax, adr);
    4112 }
    4113
    4114 void Assembler::fld_s(Address adr) {
    4115   InstructionMark im(this);
    4116   emit_int8((unsigned char)0xD9);
    4117   emit_operand32(rax, adr);
    4118 }
    4119
    4120
    4121 void Assembler::fld_s(int index) {
    4122   emit_farith(0xD9, 0xC0, index);
    4123 }
    4124
    4125 void Assembler::fld_x(Address adr) {
    4126   InstructionMark im(this);
    4127   emit_int8((unsigned char)0xDB);
    4128   emit_operand32(rbp, adr);
    4129 }
    4130
    4131 void Assembler::fldcw(Address src) {
    4132   InstructionMark im(this);
    4133   emit_int8((unsigned char)0xD9);
    4134   emit_operand32(rbp, src);
    4135 }
    4136
    4137 void Assembler::fldenv(Address src) {
    4138   InstructionMark im(this);
    4139   emit_int8((unsigned char)0xD9);
    4140   emit_operand32(rsp, src);
    4141 }
    4142
    4143 void Assembler::fldlg2() {
    4144   emit_int8((unsigned char)0xD9);
    4145   emit_int8((unsigned char)0xEC);
    4146 }
    4147
    4148 void Assembler::fldln2() {
    4149   emit_int8((unsigned char)0xD9);
    4150   emit_int8((unsigned char)0xED);
    4151 }
    4152
    4153 void Assembler::fldz() {
    4154   emit_int8((unsigned char)0xD9);
    4155   emit_int8((unsigned char)0xEE);
    4156 }
    4157
    4158 void Assembler::flog() {
    4159   fldln2();
    4160   fxch();
    4161   fyl2x();
    4162 }
    4163
    4164 void Assembler::flog10() {
    4165   fldlg2();
    4166   fxch();
    4167   fyl2x();
    4168 }
    4169
    4170 void Assembler::fmul(int i) {
    4171   emit_farith(0xD8, 0xC8, i);
    4172 }
    4173
    4174 void Assembler::fmul_d(Address src) {
    4175   InstructionMark im(this);
    4176   emit_int8((unsigned char)0xDC);
    4177   emit_operand32(rcx, src);
    4178 }
    4179
    4180 void Assembler::fmul_s(Address src) {
    4181   InstructionMark im(this);
    4182   emit_int8((unsigned char)0xD8);
    4183   emit_operand32(rcx, src);
    4184 }
    4185
    4186 void Assembler::fmula(int i) {
    4187   emit_farith(0xDC, 0xC8, i);
    4188 }
    4189
    4190 void Assembler::fmulp(int i) {
    4191   emit_farith(0xDE, 0xC8, i);
    4192 }
    4193
    4194 void Assembler::fnsave(Address dst) {
    4195   InstructionMark im(this);
    4196   emit_int8((unsigned char)0xDD);
    4197   emit_operand32(rsi, dst);
    4198 }
    4199
    4200 void Assembler::fnstcw(Address src) {
    4201   InstructionMark im(this);
    4202   emit_int8((unsigned char)0x9B);
    4203   emit_int8((unsigned char)0xD9);
    4204   emit_operand32(rdi, src);
    4205 }
    4206
    4207 void Assembler::fnstsw_ax() {
    4208   emit_int8((unsigned char)0xDF);
    4209   emit_int8((unsigned char)0xE0);
    4210 }
    4211
    4212 void Assembler::fprem() {
    4213   emit_int8((unsigned char)0xD9);
    4214   emit_int8((unsigned char)0xF8);
    4215 }
    4216
    4217 void Assembler::fprem1() {
    4218   emit_int8((unsigned char)0xD9);
    4219   emit_int8((unsigned char)0xF5);
    4220 }
    4221
    4222 void Assembler::frstor(Address src) {
    4223   InstructionMark im(this);
    4224   emit_int8((unsigned char)0xDD);
    4225   emit_operand32(rsp, src);
    4226 }
    4227
    4228 void Assembler::fsin() {
    4229   emit_int8((unsigned char)0xD9);
    4230   emit_int8((unsigned char)0xFE);
    4231 }
    4232
    4233 void Assembler::fsqrt() {
    4234   emit_int8((unsigned char)0xD9);
    4235   emit_int8((unsigned char)0xFA);
    4236 }
    4237
    4238 void Assembler::fst_d(Address adr) {
    4239   InstructionMark im(this);
    4240   emit_int8((unsigned char)0xDD);
    4241   emit_operand32(rdx, adr);
    4242 }
    4243
    4244 void Assembler::fst_s(Address adr) {
    4245   InstructionMark im(this);
    4246   emit_int8((unsigned char)0xD9);
    4247   emit_operand32(rdx, adr);
    4248 }
    4249
    4250 void Assembler::fstp_d(Address adr) {
    4251   InstructionMark im(this);
    4252   emit_int8((unsigned char)0xDD);
    4253   emit_operand32(rbx, adr);
    4254 }
    4255
    4256 void Assembler::fstp_d(int index) {
    4257   emit_farith(0xDD, 0xD8, index);
    4258 }
    4259
    4260 void Assembler::fstp_s(Address adr) {
    4261   InstructionMark im(this);
    4262   emit_int8((unsigned char)0xD9);
    4263   emit_operand32(rbx, adr);
    4264 }
    4265
    4266 void Assembler::fstp_x(Address adr) {
    4267   InstructionMark im(this);
    4268   emit_int8((unsigned char)0xDB);
    4269   emit_operand32(rdi, adr);
    4270 }
    4271
    4272 void Assembler::fsub(int i) {
    4273   emit_farith(0xD8, 0xE0, i);
    4274 }
    4275
    4276 void Assembler::fsub_d(Address src) {
    4277   InstructionMark im(this);
    4278   emit_int8((unsigned char)0xDC);
    4279   emit_operand32(rsp, src);
    4280 }
    4281
    4282 void Assembler::fsub_s(Address src) {
    4283   InstructionMark im(this);
    4284   emit_int8((unsigned char)0xD8);
    4285   emit_operand32(rsp, src);
    4286 }
    4287
    4288 void Assembler::fsuba(int i) {
    4289   emit_farith(0xDC, 0xE8, i);
    4290 }
    4291
    4292 void Assembler::fsubp(int i) {
    4293   emit_farith(0xDE, 0xE8, i);                    // ST(0) <- ST(0) - ST(1) and pop (Intel manual wrong)
    4294 }
    4295
    4296 void Assembler::fsubr(int i) {
    4297   emit_farith(0xD8, 0xE8, i);
    4298 }
    4299
    4300 void Assembler::fsubr_d(Address src) {
    4301   InstructionMark im(this);
    4302   emit_int8((unsigned char)0xDC);
    4303   emit_operand32(rbp, src);
    4304 }
    4305
    4306 void Assembler::fsubr_s(Address src) {
    4307   InstructionMark im(this);
    4308   emit_int8((unsigned char)0xD8);
    4309   emit_operand32(rbp, src);
    4310 }
    4311
    4312 void Assembler::fsubra(int i) {
    4313   emit_farith(0xDC, 0xE0, i);
    4314 }
    4315
    4316 void Assembler::fsubrp(int i) {
    4317   emit_farith(0xDE, 0xE0, i);                    // ST(0) <- ST(1) - ST(0) and pop (Intel manual wrong)
    4318 }
    4319
    4320 void Assembler::ftan() {
    4321   emit_int8((unsigned char)0xD9);
    4322   emit_int8((unsigned char)0xF2);
    4323   emit_int8((unsigned char)0xDD);
    4324   emit_int8((unsigned char)0xD8);
    4325 }
    4326
    4327 void Assembler::ftst() {
    4328   emit_int8((unsigned char)0xD9);
    4329   emit_int8((unsigned char)0xE4);
    4330 }
    4331
    4332 void Assembler::fucomi(int i) {
    4333   // make sure the instruction is supported (introduced for P6, together with cmov)
    4334   guarantee(VM_Version::supports_cmov(), "illegal instruction");
    4335   emit_farith(0xDB, 0xE8, i);
    4336 }
    4337
    4338 void Assembler::fucomip(int i) {
    4339   // make sure the instruction is supported (introduced for P6, together with cmov)
    4340   guarantee(VM_Version::supports_cmov(), "illegal instruction");
    4341   emit_farith(0xDF, 0xE8, i);
    4342 }
    4343
    4344 void Assembler::fwait() {
    4345   emit_int8((unsigned char)0x9B);
    4346 }
    4347
    4348 void Assembler::fxch(int i) {
    4349   emit_farith(0xD9, 0xC8, i);
    4350 }
    4351
    4352 void Assembler::fyl2x() {
    4353   emit_int8((unsigned char)0xD9);
    4354   emit_int8((unsigned char)0xF1);
    4355 }
    4356
    4357 void Assembler::frndint() {
    4358   emit_int8((unsigned char)0xD9);
    4359   emit_int8((unsigned char)0xFC);
    4360 }
    4361
    4362 void Assembler::f2xm1() {
    4363   emit_int8((unsigned char)0xD9);
    4364   emit_int8((unsigned char)0xF0);
    4365 }
    4366
    4367 void Assembler::fldl2e() {
    4368   emit_int8((unsigned char)0xD9);
    4369   emit_int8((unsigned char)0xEA);
    4370 }
    4371
    4372 // SSE SIMD prefix byte values corresponding to VexSimdPrefix encoding.
    4373 static int simd_pre[4] = { 0, 0x66, 0xF3, 0xF2 };
    4374 // SSE opcode second byte values (first is 0x0F) corresponding to VexOpcode encoding.
    4375 static int simd_opc[4] = { 0,    0, 0x38, 0x3A };
    4376
    4377 // Generate SSE legacy REX prefix and SIMD opcode based on VEX encoding.
    4378 void Assembler::rex_prefix(Address adr, XMMRegister xreg, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
    4379   if (pre > 0) {
    4380     emit_int8(simd_pre[pre]);
    4381   }
    4382   if (rex_w) {
    4383     prefixq(adr, xreg);
    4384   } else {
    4385     prefix(adr, xreg);
    4386   }
    4387   if (opc > 0) {
    4388     emit_int8(0x0F);
    4389     int opc2 = simd_opc[opc];
    4390     if (opc2 > 0) {
    4391       emit_int8(opc2);
    4392     }
    4393   }
    4394 }
    4395
    4396 int Assembler::rex_prefix_and_encode(int dst_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
    4397   if (pre > 0) {
    4398     emit_int8(simd_pre[pre]);
    4399   }
    4400   int encode = (rex_w) ? prefixq_and_encode(dst_enc, src_enc) :
    4401                           prefix_and_encode(dst_enc, src_enc);
    4402   if (opc > 0) {
    4403     emit_int8(0x0F);
    4404     int opc2 = simd_opc[opc];
    4405     if (opc2 > 0) {
    4406       emit_int8(opc2);
    4407     }
    4408   }
    4409   return encode;
    4410 }
    4411
    4412
    4413 void Assembler::vex_prefix(bool vex_r, bool vex_b, bool vex_x, bool vex_w, int nds_enc, VexSimdPrefix pre, VexOpcode opc, bool vector256) {
    4414   if (vex_b || vex_x || vex_w || (opc == VEX_OPCODE_0F_38) || (opc == VEX_OPCODE_0F_3A)) {
    4415     prefix(VEX_3bytes);
    4416
    4417     int byte1 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0);
    4418     byte1 = (~byte1) & 0xE0;
    4419     byte1 |= opc;
    4420     emit_int8(byte1);
    4421
    4422     int byte2 = ((~nds_enc) & 0xf) << 3;
    4423     byte2 |= (vex_w ? VEX_W : 0) | (vector256 ? 4 : 0) | pre;
    4424     emit_int8(byte2);
    4425   } else {
    4426     prefix(VEX_2bytes);
    4427
    4428     int byte1 = vex_r ? VEX_R : 0;
    4429     byte1 = (~byte1) & 0x80;
    4430     byte1 |= ((~nds_enc) & 0xf) << 3;
    4431     byte1 |= (vector256 ? 4 : 0) | pre;
    4432     emit_int8(byte1);
    4433   }
    4434 }
    4435
    4436 void Assembler::vex_prefix(Address adr, int nds_enc, int xreg_enc, VexSimdPrefix pre, VexOpcode opc, bool vex_w, bool vector256){
    4437   bool vex_r = (xreg_enc >= 8);
    4438   bool vex_b = adr.base_needs_rex();
    4439   bool vex_x = adr.index_needs_rex();
    4440   vex_prefix(vex_r, vex_b, vex_x, vex_w, nds_enc, pre, opc, vector256);
    4441 }
    4442
    4443 int Assembler::vex_prefix_and_encode(int dst_enc, int nds_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, bool vex_w, bool vector256) {
    4444   bool vex_r = (dst_enc >= 8);
    4445   bool vex_b = (src_enc >= 8);
    4446   bool vex_x = false;
    4447   vex_prefix(vex_r, vex_b, vex_x, vex_w, nds_enc, pre, opc, vector256);
    4448   return (((dst_enc & 7) << 3) | (src_enc & 7));
    4449 }
    4450
    4451
    4452 void Assembler::simd_prefix(XMMRegister xreg, XMMRegister nds, Address adr, VexSimdPrefix pre, VexOpcode opc, bool rex_w, bool vector256) {
    4453   if (UseAVX > 0) {
    4454     int xreg_enc = xreg->encoding();
    4455     int  nds_enc = nds->is_valid() ? nds->encoding() : 0;
    4456     vex_prefix(adr, nds_enc, xreg_enc, pre, opc, rex_w, vector256);
    4457   } else {
    4458     assert((nds == xreg) || (nds == xnoreg), "wrong sse encoding");
    4459     rex_prefix(adr, xreg, pre, opc, rex_w);
    4460   }
    4461 }
    4462
    4463 int Assembler::simd_prefix_and_encode(XMMRegister dst, XMMRegister nds, XMMRegister src, VexSimdPrefix pre, VexOpcode opc, bool rex_w, bool vector256) {
    4464   int dst_enc = dst->encoding();
    4465   int src_enc = src->encoding();
    4466   if (UseAVX > 0) {
    4467     int nds_enc = nds->is_valid() ? nds->encoding() : 0;
    4468     return vex_prefix_and_encode(dst_enc, nds_enc, src_enc, pre, opc, rex_w, vector256);
    4469   } else {
    4470     assert((nds == dst) || (nds == src) || (nds == xnoreg), "wrong sse encoding");
    4471     return rex_prefix_and_encode(dst_enc, src_enc, pre, opc, rex_w);
    4472   }
    4473 }
    4474
    4475 void Assembler::emit_simd_arith(int opcode, XMMRegister dst, Address src, VexSimdPrefix pre) {
    4476   InstructionMark im(this);
    4477   simd_prefix(dst, dst, src, pre);
    4478   emit_int8(opcode);
    4479   emit_operand(dst, src);
    4480 }
    4481
    4482 void Assembler::emit_simd_arith(int opcode, XMMRegister dst, XMMRegister src, VexSimdPrefix pre) {
    4483   int encode = simd_prefix_and_encode(dst, dst, src, pre);
    4484   emit_int8(opcode);
    4485   emit_int8((unsigned char)(0xC0 | encode));
    4486 }
    4487
    4488 // Versions with no second source register (non-destructive source).
    4489 void Assembler::emit_simd_arith_nonds(int opcode, XMMRegister dst, Address src, VexSimdPrefix pre) {
    4490   InstructionMark im(this);
    4491   simd_prefix(dst, xnoreg, src, pre);
    4492   emit_int8(opcode);
    4493   emit_operand(dst, src);
    4494 }
    4495
    4496 void Assembler::emit_simd_arith_nonds(int opcode, XMMRegister dst, XMMRegister src, VexSimdPrefix pre) {
    4497   int encode = simd_prefix_and_encode(dst, xnoreg, src, pre);
    4498   emit_int8(opcode);
    4499   emit_int8((unsigned char)(0xC0 | encode));
    4500 }
    4501
    4502 // 3-operands AVX instructions
    4503 void Assembler::emit_vex_arith(int opcode, XMMRegister dst, XMMRegister nds,
    4504                                Address src, VexSimdPrefix pre, bool vector256) {
    4505   InstructionMark im(this);
    4506   vex_prefix(dst, nds, src, pre, vector256);
    4507   emit_int8(opcode);
    4508   emit_operand(dst, src);
    4509 }
    4510
    4511 void Assembler::emit_vex_arith(int opcode, XMMRegister dst, XMMRegister nds,
    4512                                XMMRegister src, VexSimdPrefix pre, bool vector256) {
    4513   int encode = vex_prefix_and_encode(dst, nds, src, pre, vector256);
    4514   emit_int8(opcode);
    4515   emit_int8((unsigned char)(0xC0 | encode));
    4516 }
    4517
    4518 #ifndef _LP64
    4519
    4520 void Assembler::incl(Register dst) {
    4521   // Don't use it directly. Use MacroAssembler::incrementl() instead.
    4522   emit_int8(0x40 | dst->encoding());
    4523 }
    4524
    4525 void Assembler::lea(Register dst, Address src) {
    4526   leal(dst, src);
    4527 }
    4528
    4529 void Assembler::mov_literal32(Address dst, int32_t imm32,  RelocationHolder const& rspec) {
    4530   InstructionMark im(this);
    4531   emit_int8((unsigned char)0xC7);
    4532   emit_operand(rax, dst);
    4533   emit_data((int)imm32, rspec, 0);
    4534 }
    4535
    4536 void Assembler::mov_literal32(Register dst, int32_t imm32, RelocationHolder const& rspec) {
    4537   InstructionMark im(this);
    4538   int encode = prefix_and_encode(dst->encoding());
    4539   emit_int8((unsigned char)(0xB8 | encode));
    4540   emit_data((int)imm32, rspec, 0);
    4541 }
    4542
    4543 void Assembler::popa() { // 32bit
    4544   emit_int8(0x61);
    4545 }
    4546
    4547 void Assembler::push_literal32(int32_t imm32, RelocationHolder const& rspec) {
    4548   InstructionMark im(this);
    4549   emit_int8(0x68);
    4550   emit_data(imm32, rspec, 0);
    4551 }
    4552
    4553 void Assembler::pusha() { // 32bit
    4554   emit_int8(0x60);
    4555 }
    4556
    4557 void Assembler::set_byte_if_not_zero(Register dst) {
    4558   emit_int8(0x0F);
    4559   emit_int8((unsigned char)0x95);
    4560   emit_int8((unsigned char)(0xE0 | dst->encoding()));
    4561 }
    4562
    4563 void Assembler::shldl(Register dst, Register src) {
    4564   emit_int8(0x0F);
    4565   emit_int8((unsigned char)0xA5);
    4566   emit_int8((unsigned char)(0xC0 | src->encoding() << 3 | dst->encoding()));
    4567 }
    4568
    4569 void Assembler::shrdl(Register dst, Register src) {
    4570   emit_int8(0x0F);
    4571   emit_int8((unsigned char)0xAD);
    4572   emit_int8((unsigned char)(0xC0 | src->encoding() << 3 | dst->encoding()));
    4573 }
    4574
    4575 #else // LP64
    4576
    4577 void Assembler::set_byte_if_not_zero(Register dst) {
    4578   int enc = prefix_and_encode(dst->encoding(), true);
    4579   emit_int8(0x0F);
    4580   emit_int8((unsigned char)0x95);
    4581   emit_int8((unsigned char)(0xE0 | enc));
    4582 }
    4583
    4584 // 64bit only pieces of the assembler
    4585 // This should only be used by 64bit instructions that can use rip-relative
    4586 // it cannot be used by instructions that want an immediate value.
    4587
    4588 bool Assembler::reachable(AddressLiteral adr) {
    4589   int64_t disp;
    4590   // None will force a 64bit literal to the code stream. Likely a placeholder
    4591   // for something that will be patched later and we need to certain it will
    4592   // always be reachable.
    4593   if (adr.reloc() == relocInfo::none) {
    4594     return false;
    4595   }
    4596   if (adr.reloc() == relocInfo::internal_word_type) {
    4597     // This should be rip relative and easily reachable.
    4598     return true;
    4599   }
    4600   if (adr.reloc() == relocInfo::virtual_call_type ||
    4601       adr.reloc() == relocInfo::opt_virtual_call_type ||
    4602       adr.reloc() == relocInfo::static_call_type ||
    4603       adr.reloc() == relocInfo::static_stub_type ) {
    4604     // This should be rip relative within the code cache and easily
    4605     // reachable until we get huge code caches. (At which point
    4606     // ic code is going to have issues).
    4607     return true;
    4608   }
    4609   if (adr.reloc() != relocInfo::external_word_type &&
    4610       adr.reloc() != relocInfo::poll_return_type &&  // these are really external_word but need special
    4611       adr.reloc() != relocInfo::poll_type &&         // relocs to identify them
    4612       adr.reloc() != relocInfo::runtime_call_type ) {
    4613     return false;
    4614   }
    4615
    4616   // Stress the correction code
    4617   if (ForceUnreachable) {
    4618     // Must be runtimecall reloc, see if it is in the codecache
    4619     // Flipping stuff in the codecache to be unreachable causes issues
    4620     // with things like inline caches where the additional instructions
    4621     // are not handled.
    4622     if (CodeCache::find_blob(adr._target) == NULL) {
    4623       return false;
    4624     }
    4625   }
    4626   // For external_word_type/runtime_call_type if it is reachable from where we
    4627   // are now (possibly a temp buffer) and where we might end up
    4628   // anywhere in the codeCache then we are always reachable.
    4629   // This would have to change if we ever save/restore shared code
    4630   // to be more pessimistic.
    4631   disp = (int64_t)adr._target - ((int64_t)CodeCache::low_bound() + sizeof(int));
    4632   if (!is_simm32(disp)) return false;
    4633   disp = (int64_t)adr._target - ((int64_t)CodeCache::high_bound() + sizeof(int));
    4634   if (!is_simm32(disp)) return false;
    4635
    4636   disp = (int64_t)adr._target - ((int64_t)pc() + sizeof(int));
    4637
    4638   // Because rip relative is a disp + address_of_next_instruction and we
    4639   // don't know the value of address_of_next_instruction we apply a fudge factor
    4640   // to make sure we will be ok no matter the size of the instruction we get placed into.
    4641   // We don't have to fudge the checks above here because they are already worst case.
    4642
    4643   // 12 == override/rex byte, opcode byte, rm byte, sib byte, a 4-byte disp , 4-byte literal
    4644   // + 4 because better safe than sorry.
    4645   const int fudge = 12 + 4;
    4646   if (disp < 0) {
    4647     disp -= fudge;
    4648   } else {
    4649     disp += fudge;
    4650   }
    4651   return is_simm32(disp);
    4652 }
    4653
    4654 // Check if the polling page is not reachable from the code cache using rip-relative
    4655 // addressing.
    4656 bool Assembler::is_polling_page_far() {
    4657   intptr_t addr = (intptr_t)os::get_polling_page();
    4658   return ForceUnreachable ||
    4659          !is_simm32(addr - (intptr_t)CodeCache::low_bound()) ||
    4660          !is_simm32(addr - (intptr_t)CodeCache::high_bound());
    4661 }
    4662
    4663 void Assembler::emit_data64(jlong data,
    4664                             relocInfo::relocType rtype,
    4665                             int format) {
    4666   if (rtype == relocInfo::none) {
    4667     emit_int64(data);
    4668   } else {
    4669     emit_data64(data, Relocation::spec_simple(rtype), format);
    4670   }
    4671 }
    4672
    4673 void Assembler::emit_data64(jlong data,
    4674                             RelocationHolder const& rspec,
    4675                             int format) {
    4676   assert(imm_operand == 0, "default format must be immediate in this file");
    4677   assert(imm_operand == format, "must be immediate");
    4678   assert(inst_mark() != NULL, "must be inside InstructionMark");
    4679   // Do not use AbstractAssembler::relocate, which is not intended for
    4680   // embedded words.  Instead, relocate to the enclosing instruction.
    4681   code_section()->relocate(inst_mark(), rspec, format);
    4682 #ifdef ASSERT
    4683   check_relocation(rspec, format);
    4684 #endif
    4685   emit_int64(data);
    4686 }
    4687
    4688 int Assembler::prefix_and_encode(int reg_enc, bool byteinst) {
    4689   if (reg_enc >= 8) {
    4690     prefix(REX_B);
    4691     reg_enc -= 8;
    4692   } else if (byteinst && reg_enc >= 4) {
    4693     prefix(REX);
    4694   }
    4695   return reg_enc;
    4696 }
    4697
    4698 int Assembler::prefixq_and_encode(int reg_enc) {
    4699   if (reg_enc < 8) {
    4700     prefix(REX_W);
    4701   } else {
    4702     prefix(REX_WB);
    4703     reg_enc -= 8;
    4704   }
    4705   return reg_enc;
    4706 }
    4707
    4708 int Assembler::prefix_and_encode(int dst_enc, int src_enc, bool byteinst) {
    4709   if (dst_enc < 8) {
    4710     if (src_enc >= 8) {
    4711       prefix(REX_B);
    4712       src_enc -= 8;
    4713     } else if (byteinst && src_enc >= 4) {
    4714       prefix(REX);
    4715     }
    4716   } else {
    4717     if (src_enc < 8) {
    4718       prefix(REX_R);
    4719     } else {
    4720       prefix(REX_RB);
    4721       src_enc -= 8;
    4722     }
    4723     dst_enc -= 8;
    4724   }
    4725   return dst_enc << 3 | src_enc;
    4726 }
    4727
    4728 int Assembler::prefixq_and_encode(int dst_enc, int src_enc) {
    4729   if (dst_enc < 8) {
    4730     if (src_enc < 8) {
    4731       prefix(REX_W);
    4732     } else {
    4733       prefix(REX_WB);
    4734       src_enc -= 8;
    4735     }
    4736   } else {
    4737     if (src_enc < 8) {
    4738       prefix(REX_WR);
    4739     } else {
    4740       prefix(REX_WRB);
    4741       src_enc -= 8;
    4742     }
    4743     dst_enc -= 8;
    4744   }
    4745   return dst_enc << 3 | src_enc;
    4746 }
    4747
    4748 void Assembler::prefix(Register reg) {
    4749   if (reg->encoding() >= 8) {
    4750     prefix(REX_B);
    4751   }
    4752 }
    4753
    4754 void Assembler::prefix(Address adr) {
    4755   if (adr.base_needs_rex()) {
    4756     if (adr.index_needs_rex()) {
    4757       prefix(REX_XB);
    4758     } else {
    4759       prefix(REX_B);
    4760     }
    4761   } else {
    4762     if (adr.index_needs_rex()) {
    4763       prefix(REX_X);
    4764     }
    4765   }
    4766 }
    4767
    4768 void Assembler::prefixq(Address adr) {
    4769   if (adr.base_needs_rex()) {
    4770     if (adr.index_needs_rex()) {
    4771       prefix(REX_WXB);
    4772     } else {
    4773       prefix(REX_WB);
    4774     }
    4775   } else {
    4776     if (adr.index_needs_rex()) {
    4777       prefix(REX_WX);
    4778     } else {
    4779       prefix(REX_W);
    4780     }
    4781   }
    4782 }
    4783
    4784
    4785 void Assembler::prefix(Address adr, Register reg, bool byteinst) {
    4786   if (reg->encoding() < 8) {
    4787     if (adr.base_needs_rex()) {
    4788       if (adr.index_needs_rex()) {
    4789         prefix(REX_XB);
    4790       } else {
    4791         prefix(REX_B);
    4792       }
    4793     } else {
    4794       if (adr.index_needs_rex()) {
    4795         prefix(REX_X);
    4796       } else if (byteinst && reg->encoding() >= 4 ) {
    4797         prefix(REX);
    4798       }
    4799     }
    4800   } else {
    4801     if (adr.base_needs_rex()) {
    4802       if (adr.index_needs_rex()) {
    4803         prefix(REX_RXB);
    4804       } else {
    4805         prefix(REX_RB);
    4806       }
    4807     } else {
    4808       if (adr.index_needs_rex()) {
    4809         prefix(REX_RX);
    4810       } else {
    4811         prefix(REX_R);
    4812       }
    4813     }
    4814   }
    4815 }
    4816
    4817 void Assembler::prefixq(Address adr, Register src) {
    4818   if (src->encoding() < 8) {
    4819     if (adr.base_needs_rex()) {
    4820       if (adr.index_needs_rex()) {
    4821         prefix(REX_WXB);
    4822       } else {
    4823         prefix(REX_WB);
    4824       }
    4825     } else {
    4826       if (adr.index_needs_rex()) {
    4827         prefix(REX_WX);
    4828       } else {
    4829         prefix(REX_W);
    4830       }
    4831     }
    4832   } else {
    4833     if (adr.base_needs_rex()) {
    4834       if (adr.index_needs_rex()) {
    4835         prefix(REX_WRXB);
    4836       } else {
    4837         prefix(REX_WRB);
    4838       }
    4839     } else {
    4840       if (adr.index_needs_rex()) {
    4841         prefix(REX_WRX);
    4842       } else {
    4843         prefix(REX_WR);
    4844       }
    4845     }
    4846   }
    4847 }
    4848
    4849 void Assembler::prefix(Address adr, XMMRegister reg) {
    4850   if (reg->encoding() < 8) {
    4851     if (adr.base_needs_rex()) {
    4852       if (adr.index_needs_rex()) {
    4853         prefix(REX_XB);
    4854       } else {
    4855         prefix(REX_B);
    4856       }
    4857     } else {
    4858       if (adr.index_needs_rex()) {
    4859         prefix(REX_X);
    4860       }
    4861     }
    4862   } else {
    4863     if (adr.base_needs_rex()) {
    4864       if (adr.index_needs_rex()) {
    4865         prefix(REX_RXB);
    4866       } else {
    4867         prefix(REX_RB);
    4868       }
    4869     } else {
    4870       if (adr.index_needs_rex()) {
    4871         prefix(REX_RX);
    4872       } else {
    4873         prefix(REX_R);
    4874       }
    4875     }
    4876   }
    4877 }
    4878
    4879 void Assembler::prefixq(Address adr, XMMRegister src) {
    4880   if (src->encoding() < 8) {
    4881     if (adr.base_needs_rex()) {
    4882       if (adr.index_needs_rex()) {
    4883         prefix(REX_WXB);
    4884       } else {
    4885         prefix(REX_WB);
    4886       }
    4887     } else {
    4888       if (adr.index_needs_rex()) {
    4889         prefix(REX_WX);
    4890       } else {
    4891         prefix(REX_W);
    4892       }
    4893     }
    4894   } else {
    4895     if (adr.base_needs_rex()) {
    4896       if (adr.index_needs_rex()) {
    4897         prefix(REX_WRXB);
    4898       } else {
    4899         prefix(REX_WRB);
    4900       }
    4901     } else {
    4902       if (adr.index_needs_rex()) {
    4903         prefix(REX_WRX);
    4904       } else {
    4905         prefix(REX_WR);
    4906       }
    4907     }
    4908   }
    4909 }
    4910
    4911 void Assembler::adcq(Register dst, int32_t imm32) {
    4912   (void) prefixq_and_encode(dst->encoding());
    4913   emit_arith(0x81, 0xD0, dst, imm32);
    4914 }
    4915
    4916 void Assembler::adcq(Register dst, Address src) {
    4917   InstructionMark im(this);
    4918   prefixq(src, dst);
    4919   emit_int8(0x13);
    4920   emit_operand(dst, src);
    4921 }
    4922
    4923 void Assembler::adcq(Register dst, Register src) {
    4924   (void) prefixq_and_encode(dst->encoding(), src->encoding());
    4925   emit_arith(0x13, 0xC0, dst, src);
    4926 }
    4927
    4928 void Assembler::addq(Address dst, int32_t imm32) {
    4929   InstructionMark im(this);
    4930   prefixq(dst);
    4931   emit_arith_operand(0x81, rax, dst,imm32);
    4932 }
    4933
    4934 void Assembler::addq(Address dst, Register src) {
    4935   InstructionMark im(this);
    4936   prefixq(dst, src);
    4937   emit_int8(0x01);
    4938   emit_operand(src, dst);
    4939 }
    4940
    4941 void Assembler::addq(Register dst, int32_t imm32) {
    4942   (void) prefixq_and_encode(dst->encoding());
    4943   emit_arith(0x81, 0xC0, dst, imm32);
    4944 }
    4945
    4946 void Assembler::addq(Register dst, Address src) {
    4947   InstructionMark im(this);
    4948   prefixq(src, dst);
    4949   emit_int8(0x03);
    4950   emit_operand(dst, src);
    4951 }
    4952
    4953 void Assembler::addq(Register dst, Register src) {
    4954   (void) prefixq_and_encode(dst->encoding(), src->encoding());
    4955   emit_arith(0x03, 0xC0, dst, src);
    4956 }
    4957
    4958 void Assembler::adcxq(Register dst, Register src) {
    4959   //assert(VM_Version::supports_adx(), "adx instructions not supported");
    4960   emit_int8((unsigned char)0x66);
    4961   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
    4962   emit_int8(0x0F);
    4963   emit_int8(0x38);
    4964   emit_int8((unsigned char)0xF6);
    4965   emit_int8((unsigned char)(0xC0 | encode));
    4966 }
    4967
    4968 void Assembler::adoxq(Register dst, Register src) {
    4969   //assert(VM_Version::supports_adx(), "adx instructions not supported");
    4970   emit_int8((unsigned char)0xF3);
    4971   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
    4972   emit_int8(0x0F);
    4973   emit_int8(0x38);
    4974   emit_int8((unsigned char)0xF6);
    4975   emit_int8((unsigned char)(0xC0 | encode));
    4976 }
    4977
    4978 void Assembler::andq(Address dst, int32_t imm32) {
    4979   InstructionMark im(this);
    4980   prefixq(dst);
    4981   emit_int8((unsigned char)0x81);
    4982   emit_operand(rsp, dst, 4);
    4983   emit_int32(imm32);
    4984 }
    4985
    4986 void Assembler::andq(Register dst, int32_t imm32) {
    4987   (void) prefixq_and_encode(dst->encoding());
    4988   emit_arith(0x81, 0xE0, dst, imm32);
    4989 }
    4990
    4991 void Assembler::andq(Register dst, Address src) {
    4992   InstructionMark im(this);
    4993   prefixq(src, dst);
    4994   emit_int8(0x23);
    4995   emit_operand(dst, src);
    4996 }
    4997
    4998 void Assembler::andq(Register dst, Register src) {
    4999   (void) prefixq_and_encode(dst->encoding(), src->encoding());
    5000   emit_arith(0x23, 0xC0, dst, src);
    5001 }
    5002
    5003 void Assembler::andnq(Register dst, Register src1, Register src2) {
    5004   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
    5005   int encode = vex_prefix_0F38_and_encode_q(dst, src1, src2);
    5006   emit_int8((unsigned char)0xF2);
    5007   emit_int8((unsigned char)(0xC0 | encode));
    5008 }
    5009
    5010 void Assembler::andnq(Register dst, Register src1, Address src2) {
    5011   InstructionMark im(this);
    5012   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
    5013   vex_prefix_0F38_q(dst, src1, src2);
    5014   emit_int8((unsigned char)0xF2);
    5015   emit_operand(dst, src2);
    5016 }
    5017
    5018 void Assembler::bsfq(Register dst, Register src) {
    5019   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
    5020   emit_int8(0x0F);
    5021   emit_int8((unsigned char)0xBC);
    5022   emit_int8((unsigned char)(0xC0 | encode));
    5023 }
    5024
    5025 void Assembler::bsrq(Register dst, Register src) {
    5026   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
    5027   emit_int8(0x0F);
    5028   emit_int8((unsigned char)0xBD);
    5029   emit_int8((unsigned char)(0xC0 | encode));
    5030 }
    5031
    5032 void Assembler::bswapq(Register reg) {
    5033   int encode = prefixq_and_encode(reg->encoding());
    5034   emit_int8(0x0F);
    5035   emit_int8((unsigned char)(0xC8 | encode));
    5036 }
    5037
    5038 void Assembler::blsiq(Register dst, Register src) {
    5039   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
    5040   int encode = vex_prefix_0F38_and_encode_q(rbx, dst, src);
    5041   emit_int8((unsigned char)0xF3);
    5042   emit_int8((unsigned char)(0xC0 | encode));
    5043 }
    5044
    5045 void Assembler::blsiq(Register dst, Address src) {
    5046   InstructionMark im(this);
    5047   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
    5048   vex_prefix_0F38_q(rbx, dst, src);
    5049   emit_int8((unsigned char)0xF3);
    5050   emit_operand(rbx, src);
    5051 }
    5052
    5053 void Assembler::blsmskq(Register dst, Register src) {
    5054   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
    5055   int encode = vex_prefix_0F38_and_encode_q(rdx, dst, src);
    5056   emit_int8((unsigned char)0xF3);
    5057   emit_int8((unsigned char)(0xC0 | encode));
    5058 }
    5059
    5060 void Assembler::blsmskq(Register dst, Address src) {
    5061   InstructionMark im(this);
    5062   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
    5063   vex_prefix_0F38_q(rdx, dst, src);
    5064   emit_int8((unsigned char)0xF3);
    5065   emit_operand(rdx, src);
    5066 }
    5067
    5068 void Assembler::blsrq(Register dst, Register src) {
    5069   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
    5070   int encode = vex_prefix_0F38_and_encode_q(rcx, dst, src);
    5071   emit_int8((unsigned char)0xF3);
    5072   emit_int8((unsigned char)(0xC0 | encode));
    5073 }
    5074
    5075 void Assembler::blsrq(Register dst, Address src) {
    5076   InstructionMark im(this);
    5077   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
    5078   vex_prefix_0F38_q(rcx, dst, src);
    5079   emit_int8((unsigned char)0xF3);
    5080   emit_operand(rcx, src);
    5081 }
    5082
    5083 void Assembler::cdqq() {
    5084   prefix(REX_W);
    5085   emit_int8((unsigned char)0x99);
    5086 }
    5087
    5088 void Assembler::clflush(Address adr) {
    5089   prefix(adr);
    5090   emit_int8(0x0F);
    5091   emit_int8((unsigned char)0xAE);
    5092   emit_operand(rdi, adr);
    5093 }
    5094
    5095 void Assembler::cmovq(Condition cc, Register dst, Register src) {
    5096   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
    5097   emit_int8(0x0F);
    5098   emit_int8(0x40 | cc);
    5099   emit_int8((unsigned char)(0xC0 | encode));
    5100 }
    5101
    5102 void Assembler::cmovq(Condition cc, Register dst, Address src) {
    5103   InstructionMark im(this);
    5104   prefixq(src, dst);
    5105   emit_int8(0x0F);
    5106   emit_int8(0x40 | cc);
    5107   emit_operand(dst, src);
    5108 }
    5109
    5110 void Assembler::cmpq(Address dst, int32_t imm32) {
    5111   InstructionMark im(this);
    5112   prefixq(dst);
    5113   emit_int8((unsigned char)0x81);
    5114   emit_operand(rdi, dst, 4);
    5115   emit_int32(imm32);
    5116 }
    5117
    5118 void Assembler::cmpq(Register dst, int32_t imm32) {
    5119   (void) prefixq_and_encode(dst->encoding());
    5120   emit_arith(0x81, 0xF8, dst, imm32);
    5121 }
    5122
    5123 void Assembler::cmpq(Address dst, Register src) {
    5124   InstructionMark im(this);
    5125   prefixq(dst, src);
    5126   emit_int8(0x3B);
    5127   emit_operand(src, dst);
    5128 }
    5129
    5130 void Assembler::cmpq(Register dst, Register src) {
    5131   (void) prefixq_and_encode(dst->encoding(), src->encoding());
    5132   emit_arith(0x3B, 0xC0, dst, src);
    5133 }
    5134
    5135 void Assembler::cmpq(Register dst, Address  src) {
    5136   InstructionMark im(this);
    5137   prefixq(src, dst);
    5138   emit_int8(0x3B);
    5139   emit_operand(dst, src);
    5140 }
    5141
    5142 void Assembler::cmpxchgq(Register reg, Address adr) {
    5143   InstructionMark im(this);
    5144   prefixq(adr, reg);
    5145   emit_int8(0x0F);
    5146   emit_int8((unsigned char)0xB1);
    5147   emit_operand(reg, adr);
    5148 }
    5149
    5150 void Assembler::cvtsi2sdq(XMMRegister dst, Register src) {
    5151   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    5152   int encode = simd_prefix_and_encode_q(dst, dst, src, VEX_SIMD_F2);
    5153   emit_int8(0x2A);
    5154   emit_int8((unsigned char)(0xC0 | encode));
    5155 }
    5156
    5157 void Assembler::cvtsi2sdq(XMMRegister dst, Address src) {
    5158   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    5159   InstructionMark im(this);
    5160   simd_prefix_q(dst, dst, src, VEX_SIMD_F2);
    5161   emit_int8(0x2A);
    5162   emit_operand(dst, src);
    5163 }
    5164
    5165 void Assembler::cvtsi2ssq(XMMRegister dst, Register src) {
    5166   NOT_LP64(assert(VM_Version::supports_sse(), ""));
    5167   int encode = simd_prefix_and_encode_q(dst, dst, src, VEX_SIMD_F3);
    5168   emit_int8(0x2A);
    5169   emit_int8((unsigned char)(0xC0 | encode));
    5170 }
    5171
    5172 void Assembler::cvtsi2ssq(XMMRegister dst, Address src) {
    5173   NOT_LP64(assert(VM_Version::supports_sse(), ""));
    5174   InstructionMark im(this);
    5175   simd_prefix_q(dst, dst, src, VEX_SIMD_F3);
    5176   emit_int8(0x2A);
    5177   emit_operand(dst, src);
    5178 }
    5179
    5180 void Assembler::cvttsd2siq(Register dst, XMMRegister src) {
    5181   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    5182   int encode = simd_prefix_and_encode_q(dst, src, VEX_SIMD_F2);
    5183   emit_int8(0x2C);
    5184   emit_int8((unsigned char)(0xC0 | encode));
    5185 }
    5186
    5187 void Assembler::cvttss2siq(Register dst, XMMRegister src) {
    5188   NOT_LP64(assert(VM_Version::supports_sse(), ""));
    5189   int encode = simd_prefix_and_encode_q(dst, src, VEX_SIMD_F3);
    5190   emit_int8(0x2C);
    5191   emit_int8((unsigned char)(0xC0 | encode));
    5192 }
    5193
    5194 void Assembler::decl(Register dst) {
    5195   // Don't use it directly. Use MacroAssembler::decrementl() instead.
    5196   // Use two-byte form (one-byte form is a REX prefix in 64-bit mode)
    5197   int encode = prefix_and_encode(dst->encoding());
    5198   emit_int8((unsigned char)0xFF);
    5199   emit_int8((unsigned char)(0xC8 | encode));
    5200 }
    5201
    5202 void Assembler::decq(Register dst) {
    5203   // Don't use it directly. Use MacroAssembler::decrementq() instead.
    5204   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
    5205   int encode = prefixq_and_encode(dst->encoding());
    5206   emit_int8((unsigned char)0xFF);
    5207   emit_int8(0xC8 | encode);
    5208 }
    5209
    5210 void Assembler::decq(Address dst) {
    5211   // Don't use it directly. Use MacroAssembler::decrementq() instead.
    5212   InstructionMark im(this);
    5213   prefixq(dst);
    5214   emit_int8((unsigned char)0xFF);
    5215   emit_operand(rcx, dst);
    5216 }
    5217
    5218 void Assembler::fxrstor(Address src) {
    5219   prefixq(src);
    5220   emit_int8(0x0F);
    5221   emit_int8((unsigned char)0xAE);
    5222   emit_operand(as_Register(1), src);
    5223 }
    5224
    5225 void Assembler::fxsave(Address dst) {
    5226   prefixq(dst);
    5227   emit_int8(0x0F);
    5228   emit_int8((unsigned char)0xAE);
    5229   emit_operand(as_Register(0), dst);
    5230 }
    5231
    5232 void Assembler::idivq(Register src) {
    5233   int encode = prefixq_and_encode(src->encoding());
    5234   emit_int8((unsigned char)0xF7);
    5235   emit_int8((unsigned char)(0xF8 | encode));
    5236 }
    5237
    5238 void Assembler::imulq(Register dst, Register src) {
    5239   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
    5240   emit_int8(0x0F);
    5241   emit_int8((unsigned char)0xAF);
    5242   emit_int8((unsigned char)(0xC0 | encode));
    5243 }
    5244
    5245 void Assembler::imulq(Register dst, Register src, int value) {
    5246   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
    5247   if (is8bit(value)) {
    5248     emit_int8(0x6B);
    5249     emit_int8((unsigned char)(0xC0 | encode));
    5250     emit_int8(value & 0xFF);
    5251   } else {
    5252     emit_int8(0x69);
    5253     emit_int8((unsigned char)(0xC0 | encode));
    5254     emit_int32(value);
    5255   }
    5256 }
    5257
    5258 void Assembler::imulq(Register dst, Address src) {
    5259   InstructionMark im(this);
    5260   prefixq(src, dst);
    5261   emit_int8(0x0F);
    5262   emit_int8((unsigned char) 0xAF);
    5263   emit_operand(dst, src);
    5264 }
    5265
    5266 void Assembler::incl(Register dst) {
    5267   // Don't use it directly. Use MacroAssembler::incrementl() instead.
    5268   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
    5269   int encode = prefix_and_encode(dst->encoding());
    5270   emit_int8((unsigned char)0xFF);
    5271   emit_int8((unsigned char)(0xC0 | encode));
    5272 }
    5273
    5274 void Assembler::incq(Register dst) {
    5275   // Don't use it directly. Use MacroAssembler::incrementq() instead.
    5276   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
    5277   int encode = prefixq_and_encode(dst->encoding());
    5278   emit_int8((unsigned char)0xFF);
    5279   emit_int8((unsigned char)(0xC0 | encode));
    5280 }
    5281
    5282 void Assembler::incq(Address dst) {
    5283   // Don't use it directly. Use MacroAssembler::incrementq() instead.
    5284   InstructionMark im(this);
    5285   prefixq(dst);
    5286   emit_int8((unsigned char)0xFF);
    5287   emit_operand(rax, dst);
    5288 }
    5289
    5290 void Assembler::lea(Register dst, Address src) {
    5291   leaq(dst, src);
    5292 }
    5293
    5294 void Assembler::leaq(Register dst, Address src) {
    5295   InstructionMark im(this);
    5296   prefixq(src, dst);
    5297   emit_int8((unsigned char)0x8D);
    5298   emit_operand(dst, src);
    5299 }
    5300
    5301 void Assembler::mov64(Register dst, int64_t imm64) {
    5302   InstructionMark im(this);
    5303   int encode = prefixq_and_encode(dst->encoding());
    5304   emit_int8((unsigned char)(0xB8 | encode));
    5305   emit_int64(imm64);
    5306 }
    5307
    5308 void Assembler::mov_literal64(Register dst, intptr_t imm64, RelocationHolder const& rspec) {
    5309   InstructionMark im(this);
    5310   int encode = prefixq_and_encode(dst->encoding());
    5311   emit_int8(0xB8 | encode);
    5312   emit_data64(imm64, rspec);
    5313 }
    5314
    5315 void Assembler::mov_narrow_oop(Register dst, int32_t imm32, RelocationHolder const& rspec) {
    5316   InstructionMark im(this);
    5317   int encode = prefix_and_encode(dst->encoding());
    5318   emit_int8((unsigned char)(0xB8 | encode));
    5319   emit_data((int)imm32, rspec, narrow_oop_operand);
    5320 }
    5321
    5322 void Assembler::mov_narrow_oop(Address dst, int32_t imm32,  RelocationHolder const& rspec) {
    5323   InstructionMark im(this);
    5324   prefix(dst);
    5325   emit_int8((unsigned char)0xC7);
    5326   emit_operand(rax, dst, 4);
    5327   emit_data((int)imm32, rspec, narrow_oop_operand);
    5328 }
    5329
    5330 void Assembler::cmp_narrow_oop(Register src1, int32_t imm32, RelocationHolder const& rspec) {
    5331   InstructionMark im(this);
    5332   int encode = prefix_and_encode(src1->encoding());
    5333   emit_int8((unsigned char)0x81);
    5334   emit_int8((unsigned char)(0xF8 | encode));
    5335   emit_data((int)imm32, rspec, narrow_oop_operand);
    5336 }
    5337
    5338 void Assembler::cmp_narrow_oop(Address src1, int32_t imm32, RelocationHolder const& rspec) {
    5339   InstructionMark im(this);
    5340   prefix(src1);
    5341   emit_int8((unsigned char)0x81);
    5342   emit_operand(rax, src1, 4);
    5343   emit_data((int)imm32, rspec, narrow_oop_operand);
    5344 }
    5345
    5346 void Assembler::lzcntq(Register dst, Register src) {
    5347   assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
    5348   emit_int8((unsigned char)0xF3);
    5349   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
    5350   emit_int8(0x0F);
    5351   emit_int8((unsigned char)0xBD);
    5352   emit_int8((unsigned char)(0xC0 | encode));
    5353 }
    5354
    5355 void Assembler::movdq(XMMRegister dst, Register src) {
    5356   // table D-1 says MMX/SSE2
    5357   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    5358   int encode = simd_prefix_and_encode_q(dst, src, VEX_SIMD_66);
    5359   emit_int8(0x6E);
    5360   emit_int8((unsigned char)(0xC0 | encode));
    5361 }
    5362
    5363 void Assembler::movdq(Register dst, XMMRegister src) {
    5364   // table D-1 says MMX/SSE2
    5365   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
    5366   // swap src/dst to get correct prefix
    5367   int encode = simd_prefix_and_encode_q(src, dst, VEX_SIMD_66);
    5368   emit_int8(0x7E);
    5369   emit_int8((unsigned char)(0xC0 | encode));
    5370 }
    5371
    5372 void Assembler::movq(Register dst, Register src) {
    5373   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
    5374   emit_int8((unsigned char)0x8B);
    5375   emit_int8((unsigned char)(0xC0 | encode));
    5376 }
    5377
    5378 void Assembler::movq(Register dst, Address src) {
    5379   InstructionMark im(this);
    5380   prefixq(src, dst);
    5381   emit_int8((unsigned char)0x8B);
    5382   emit_operand(dst, src);
    5383 }
    5384
    5385 void Assembler::movq(Address dst, Register src) {
    5386   InstructionMark im(this);
    5387   prefixq(dst, src);
    5388   emit_int8((unsigned char)0x89);
    5389   emit_operand(src, dst);
    5390 }
    5391
    5392 void Assembler::movsbq(Register dst, Address src) {
    5393   InstructionMark im(this);
    5394   prefixq(src, dst);
    5395   emit_int8(0x0F);
    5396   emit_int8((unsigned char)0xBE);
    5397   emit_operand(dst, src);
    5398 }
    5399
    5400 void Assembler::movsbq(Register dst, Register src) {
    5401   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
    5402   emit_int8(0x0F);
    5403   emit_int8((unsigned char)0xBE);
    5404   emit_int8((unsigned char)(0xC0 | encode));
    5405 }
    5406
    5407 void Assembler::movslq(Register dst, int32_t imm32) {
    5408   // dbx shows movslq(rcx, 3) as movq     $0x0000000049000000,(%rbx)
    5409   // and movslq(r8, 3); as movl     $0x0000000048000000,(%rbx)
    5410   // as a result we shouldn't use until tested at runtime...
    5411   ShouldNotReachHere();
    5412   InstructionMark im(this);
    5413   int encode = prefixq_and_encode(dst->encoding());
    5414   emit_int8((unsigned char)(0xC7 | encode));
    5415   emit_int32(imm32);
    5416 }
    5417
    5418 void Assembler::movslq(Address dst, int32_t imm32) {
    5419   assert(is_simm32(imm32), "lost bits");
    5420   InstructionMark im(this);
    5421   prefixq(dst);
    5422   emit_int8((unsigned char)0xC7);
    5423   emit_operand(rax, dst, 4);
    5424   emit_int32(imm32);
    5425 }
    5426
    5427 void Assembler::movslq(Register dst, Address src) {
    5428   InstructionMark im(this);
    5429   prefixq(src, dst);
    5430   emit_int8(0x63);
    5431   emit_operand(dst, src);
    5432 }
    5433
    5434 void Assembler::movslq(Register dst, Register src) {
    5435   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
    5436   emit_int8(0x63);
    5437   emit_int8((unsigned char)(0xC0 | encode));
    5438 }
    5439
    5440 void Assembler::movswq(Register dst, Address src) {
    5441   InstructionMark im(this);
    5442   prefixq(src, dst);
    5443   emit_int8(0x0F);
    5444   emit_int8((unsigned char)0xBF);
    5445   emit_operand(dst, src);
    5446 }
    5447
    5448 void Assembler::movswq(Register dst, Register src) {
    5449   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
    5450   emit_int8((unsigned char)0x0F);
    5451   emit_int8((unsigned char)0xBF);
    5452   emit_int8((unsigned char)(0xC0 | encode));
    5453 }
    5454
    5455 void Assembler::movzbq(Register dst, Address src) {
    5456   InstructionMark im(this);
    5457   prefixq(src, dst);
    5458   emit_int8((unsigned char)0x0F);
    5459   emit_int8((unsigned char)0xB6);
    5460   emit_operand(dst, src);
    5461 }
    5462
    5463 void Assembler::movzbq(Register dst, Register src) {
    5464   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
    5465   emit_int8(0x0F);
    5466   emit_int8((unsigned char)0xB6);
    5467   emit_int8(0xC0 | encode);
    5468 }
    5469
    5470 void Assembler::movzwq(Register dst, Address src) {
    5471   InstructionMark im(this);
    5472   prefixq(src, dst);
    5473   emit_int8((unsigned char)0x0F);
    5474   emit_int8((unsigned char)0xB7);
    5475   emit_operand(dst, src);
    5476 }
    5477
    5478 void Assembler::movzwq(Register dst, Register src) {
    5479   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
    5480   emit_int8((unsigned char)0x0F);
    5481   emit_int8((unsigned char)0xB7);
    5482   emit_int8((unsigned char)(0xC0 | encode));
    5483 }
    5484
    5485 void Assembler::mulq(Address src) {
    5486   InstructionMark im(this);
    5487   prefixq(src);
    5488   emit_int8((unsigned char)0xF7);
    5489   emit_operand(rsp, src);
    5490 }
    5491
    5492 void Assembler::mulq(Register src) {
    5493   int encode = prefixq_and_encode(src->encoding());
    5494   emit_int8((unsigned char)0xF7);
    5495   emit_int8((unsigned char)(0xE0 | encode));
    5496 }
    5497
    5498 void Assembler::mulxq(Register dst1, Register dst2, Register src) {
    5499   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
    5500   int encode = vex_prefix_and_encode(dst1->encoding(), dst2->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, true, false);
    5501   emit_int8((unsigned char)0xF6);
    5502   emit_int8((unsigned char)(0xC0 | encode));
    5503 }
    5504
    5505 void Assembler::negq(Register dst) {
    5506   int encode = prefixq_and_encode(dst->encoding());
    5507   emit_int8((unsigned char)0xF7);
    5508   emit_int8((unsigned char)(0xD8 | encode));
    5509 }
    5510
    5511 void Assembler::notq(Register dst) {
    5512   int encode = prefixq_and_encode(dst->encoding());
    5513   emit_int8((unsigned char)0xF7);
    5514   emit_int8((unsigned char)(0xD0 | encode));
    5515 }
    5516
    5517 void Assembler::orq(Address dst, int32_t imm32) {
    5518   InstructionMark im(this);
    5519   prefixq(dst);
    5520   emit_int8((unsigned char)0x81);
    5521   emit_operand(rcx, dst, 4);
    5522   emit_int32(imm32);
    5523 }
    5524
    5525 void Assembler::orq(Register dst, int32_t imm32) {
    5526   (void) prefixq_and_encode(dst->encoding());
    5527   emit_arith(0x81, 0xC8, dst, imm32);
    5528 }
    5529
    5530 void Assembler::orq(Register dst, Address src) {
    5531   InstructionMark im(this);
    5532   prefixq(src, dst);
    5533   emit_int8(0x0B);
    5534   emit_operand(dst, src);
    5535 }
    5536
    5537 void Assembler::orq(Register dst, Register src) {
    5538   (void) prefixq_and_encode(dst->encoding(), src->encoding());
    5539   emit_arith(0x0B, 0xC0, dst, src);
    5540 }
    5541
    5542 void Assembler::popa() { // 64bit
    5543   movq(r15, Address(rsp, 0));
    5544   movq(r14, Address(rsp, wordSize));
    5545   movq(r13, Address(rsp, 2 * wordSize));
    5546   movq(r12, Address(rsp, 3 * wordSize));
    5547   movq(r11, Address(rsp, 4 * wordSize));
    5548   movq(r10, Address(rsp, 5 * wordSize));
    5549   movq(r9,  Address(rsp, 6 * wordSize));
    5550   movq(r8,  Address(rsp, 7 * wordSize));
    5551   movq(rdi, Address(rsp, 8 * wordSize));
    5552   movq(rsi, Address(rsp, 9 * wordSize));
    5553   movq(rbp, Address(rsp, 10 * wordSize));
    5554   // skip rsp
    5555   movq(rbx, Address(rsp, 12 * wordSize));
    5556   movq(rdx, Address(rsp, 13 * wordSize));
    5557   movq(rcx, Address(rsp, 14 * wordSize));
    5558   movq(rax, Address(rsp, 15 * wordSize));
    5559
    5560   addq(rsp, 16 * wordSize);
    5561 }
    5562
    5563 void Assembler::popcntq(Register dst, Address src) {
    5564   assert(VM_Version::supports_popcnt(), "must support");
    5565   InstructionMark im(this);
    5566   emit_int8((unsigned char)0xF3);
    5567   prefixq(src, dst);
    5568   emit_int8((unsigned char)0x0F);
    5569   emit_int8((unsigned char)0xB8);
    5570   emit_operand(dst, src);
    5571 }
    5572
    5573 void Assembler::popcntq(Register dst, Register src) {
    5574   assert(VM_Version::supports_popcnt(), "must support");
    5575   emit_int8((unsigned char)0xF3);
    5576   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
    5577   emit_int8((unsigned char)0x0F);
    5578   emit_int8((unsigned char)0xB8);
    5579   emit_int8((unsigned char)(0xC0 | encode));
    5580 }
    5581
    5582 void Assembler::popq(Address dst) {
    5583   InstructionMark im(this);
    5584   prefixq(dst);
    5585   emit_int8((unsigned char)0x8F);
    5586   emit_operand(rax, dst);
    5587 }
    5588
    5589 void Assembler::pusha() { // 64bit
    5590   // we have to store original rsp.  ABI says that 128 bytes
    5591   // below rsp are local scratch.
    5592   movq(Address(rsp, -5 * wordSize), rsp);
    5593
    5594   subq(rsp, 16 * wordSize);
    5595
    5596   movq(Address(rsp, 15 * wordSize), rax);
    5597   movq(Address(rsp, 14 * wordSize), rcx);
    5598   movq(Address(rsp, 13 * wordSize), rdx);
    5599   movq(Address(rsp, 12 * wordSize), rbx);
    5600   // skip rsp
    5601   movq(Address(rsp, 10 * wordSize), rbp);
    5602   movq(Address(rsp, 9 * wordSize), rsi);
    5603   movq(Address(rsp, 8 * wordSize), rdi);
    5604   movq(Address(rsp, 7 * wordSize), r8);
    5605   movq(Address(rsp, 6 * wordSize), r9);
    5606   movq(Address(rsp, 5 * wordSize), r10);
    5607   movq(Address(rsp, 4 * wordSize), r11);
    5608   movq(Address(rsp, 3 * wordSize), r12);
    5609   movq(Address(rsp, 2 * wordSize), r13);
    5610   movq(Address(rsp, wordSize), r14);
    5611   movq(Address(rsp, 0), r15);
    5612 }
    5613
    5614 void Assembler::pushq(Address src) {
    5615   InstructionMark im(this);
    5616   prefixq(src);
    5617   emit_int8((unsigned char)0xFF);
    5618   emit_operand(rsi, src);
    5619 }
    5620
    5621 void Assembler::rclq(Register dst, int imm8) {
    5622   assert(isShiftCount(imm8 >> 1), "illegal shift count");
    5623   int encode = prefixq_and_encode(dst->encoding());
    5624   if (imm8 == 1) {
    5625     emit_int8((unsigned char)0xD1);
    5626     emit_int8((unsigned char)(0xD0 | encode));
    5627   } else {
    5628     emit_int8((unsigned char)0xC1);
    5629     emit_int8((unsigned char)(0xD0 | encode));
    5630     emit_int8(imm8);
    5631   }
    5632 }
    5633
    5634 void Assembler::rcrq(Register dst, int imm8) {
    5635   assert(isShiftCount(imm8 >> 1), "illegal shift count");
    5636   int encode = prefixq_and_encode(dst->encoding());
    5637   if (imm8 == 1) {
    5638     emit_int8((unsigned char)0xD1);
    5639     emit_int8((unsigned char)(0xD8 | encode));
    5640   } else {
    5641     emit_int8((unsigned char)0xC1);
    5642     emit_int8((unsigned char)(0xD8 | encode));
    5643     emit_int8(imm8);
    5644   }
    5645 }
    5646
    5647 void Assembler::rorq(Register dst, int imm8) {
    5648   assert(isShiftCount(imm8 >> 1), "illegal shift count");
    5649   int encode = prefixq_and_encode(dst->encoding());
    5650   if (imm8 == 1) {
    5651     emit_int8((unsigned char)0xD1);
    5652     emit_int8((unsigned char)(0xC8 | encode));
    5653   } else {
    5654     emit_int8((unsigned char)0xC1);
    5655     emit_int8((unsigned char)(0xc8 | encode));
    5656     emit_int8(imm8);
    5657   }
    5658 }
    5659
    5660 void Assembler::rorxq(Register dst, Register src, int imm8) {
    5661   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
    5662   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, true, false);
    5663   emit_int8((unsigned char)0xF0);
    5664   emit_int8((unsigned char)(0xC0 | encode));
    5665   emit_int8(imm8);
    5666 }
    5667
    5668 void Assembler::sarq(Register dst, int imm8) {
    5669   assert(isShiftCount(imm8 >> 1), "illegal shift count");
    5670   int encode = prefixq_and_encode(dst->encoding());
    5671   if (imm8 == 1) {
    5672     emit_int8((unsigned char)0xD1);
    5673     emit_int8((unsigned char)(0xF8 | encode));
    5674   } else {
    5675     emit_int8((unsigned char)0xC1);
    5676     emit_int8((unsigned char)(0xF8 | encode));
    5677     emit_int8(imm8);
    5678   }
    5679 }
    5680
    5681 void Assembler::sarq(Register dst) {
    5682   int encode = prefixq_and_encode(dst->encoding());
    5683   emit_int8((unsigned char)0xD3);
    5684   emit_int8((unsigned char)(0xF8 | encode));
    5685 }
    5686
    5687 void Assembler::sbbq(Address dst, int32_t imm32) {
    5688   InstructionMark im(this);
    5689   prefixq(dst);
    5690   emit_arith_operand(0x81, rbx, dst, imm32);
    5691 }
    5692
    5693 void Assembler::sbbq(Register dst, int32_t imm32) {
    5694   (void) prefixq_and_encode(dst->encoding());
    5695   emit_arith(0x81, 0xD8, dst, imm32);
    5696 }
    5697
    5698 void Assembler::sbbq(Register dst, Address src) {
    5699   InstructionMark im(this);
    5700   prefixq(src, dst);
    5701   emit_int8(0x1B);
    5702   emit_operand(dst, src);
    5703 }
    5704
    5705 void Assembler::sbbq(Register dst, Register src) {
    5706   (void) prefixq_and_encode(dst->encoding(), src->encoding());
    5707   emit_arith(0x1B, 0xC0, dst, src);
    5708 }
    5709
    5710 void Assembler::shlq(Register dst, int imm8) {
    5711   assert(isShiftCount(imm8 >> 1), "illegal shift count");
    5712   int encode = prefixq_and_encode(dst->encoding());
    5713   if (imm8 == 1) {
    5714     emit_int8((unsigned char)0xD1);
    5715     emit_int8((unsigned char)(0xE0 | encode));
    5716   } else {
    5717     emit_int8((unsigned char)0xC1);
    5718     emit_int8((unsigned char)(0xE0 | encode));
    5719     emit_int8(imm8);
    5720   }
    5721 }
    5722
    5723 void Assembler::shlq(Register dst) {
    5724   int encode = prefixq_and_encode(dst->encoding());
    5725   emit_int8((unsigned char)0xD3);
    5726   emit_int8((unsigned char)(0xE0 | encode));
    5727 }
    5728
    5729 void Assembler::shrq(Register dst, int imm8) {
    5730   assert(isShiftCount(imm8 >> 1), "illegal shift count");
    5731   int encode = prefixq_and_encode(dst->encoding());
    5732   emit_int8((unsigned char)0xC1);
    5733   emit_int8((unsigned char)(0xE8 | encode));
    5734   emit_int8(imm8);
    5735 }
    5736
    5737 void Assembler::shrq(Register dst) {
    5738   int encode = prefixq_and_encode(dst->encoding());
    5739   emit_int8((unsigned char)0xD3);
    5740   emit_int8(0xE8 | encode);
    5741 }
    5742
    5743 void Assembler::subq(Address dst, int32_t imm32) {
    5744   InstructionMark im(this);
    5745   prefixq(dst);
    5746   emit_arith_operand(0x81, rbp, dst, imm32);
    5747 }
    5748
    5749 void Assembler::subq(Address dst, Register src) {
    5750   InstructionMark im(this);
    5751   prefixq(dst, src);
    5752   emit_int8(0x29);
    5753   emit_operand(src, dst);
    5754 }
    5755
    5756 void Assembler::subq(Register dst, int32_t imm32) {
    5757   (void) prefixq_and_encode(dst->encoding());
    5758   emit_arith(0x81, 0xE8, dst, imm32);
    5759 }
    5760
    5761 // Force generation of a 4 byte immediate value even if it fits into 8bit
    5762 void Assembler::subq_imm32(Register dst, int32_t imm32) {
    5763   (void) prefixq_and_encode(dst->encoding());
    5764   emit_arith_imm32(0x81, 0xE8, dst, imm32);
    5765 }
    5766
    5767 void Assembler::subq(Register dst, Address src) {
    5768   InstructionMark im(this);
    5769   prefixq(src, dst);
    5770   emit_int8(0x2B);
    5771   emit_operand(dst, src);
    5772 }
    5773
    5774 void Assembler::subq(Register dst, Register src) {
    5775   (void) prefixq_and_encode(dst->encoding(), src->encoding());
    5776   emit_arith(0x2B, 0xC0, dst, src);
    5777 }
    5778
    5779 void Assembler::testq(Register dst, int32_t imm32) {
    5780   // not using emit_arith because test
    5781   // doesn't support sign-extension of
    5782   // 8bit operands
    5783   int encode = dst->encoding();
    5784   if (encode == 0) {
    5785     prefix(REX_W);
    5786     emit_int8((unsigned char)0xA9);
    5787   } else {
    5788     encode = prefixq_and_encode(encode);
    5789     emit_int8((unsigned char)0xF7);
    5790     emit_int8((unsigned char)(0xC0 | encode));
    5791   }
    5792   emit_int32(imm32);
    5793 }
    5794
    5795 void Assembler::testq(Register dst, Register src) {
    5796   (void) prefixq_and_encode(dst->encoding(), src->encoding());
    5797   emit_arith(0x85, 0xC0, dst, src);
    5798 }
    5799
    5800 void Assembler::xaddq(Address dst, Register src) {
    5801   InstructionMark im(this);
    5802   prefixq(dst, src);
    5803   emit_int8(0x0F);
    5804   emit_int8((unsigned char)0xC1);
    5805   emit_operand(src, dst);
    5806 }
    5807
    5808 void Assembler::xchgq(Register dst, Address src) {
    5809   InstructionMark im(this);
    5810   prefixq(src, dst);
    5811   emit_int8((unsigned char)0x87);
    5812   emit_operand(dst, src);
    5813 }
    5814
    5815 void Assembler::xchgq(Register dst, Register src) {
    5816   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
    5817   emit_int8((unsigned char)0x87);
    5818   emit_int8((unsigned char)(0xc0 | encode));
    5819 }
    5820
    5821 void Assembler::xorq(Register dst, Register src) {
    5822   (void) prefixq_and_encode(dst->encoding(), src->encoding());
    5823   emit_arith(0x33, 0xC0, dst, src);
    5824 }
    5825
    5826 void Assembler::xorq(Register dst, Address src) {
    5827   InstructionMark im(this);
    5828   prefixq(src, dst);
    5829   emit_int8(0x33);
    5830   emit_operand(dst, src);
    5831 }
    5832
    5833 #endif // !LP64