Help ensure we get the smallest encoding for commutative instructions

src/coreclr/jit/codegenxarch.cpp

@@ -990,6 +990,27 @@ void CodeGen::genCodeForBinary(GenTreeOp* treeNode)
         // all have RMW semantics if VEX support is not available
 
         bool isRMW = !m_compiler->canUseVexEncoding();
+
+        if (!isRMW && !treeNode->OperIs(GT_DIV, GT_SUB))
+        {
+            assert(treeNode->OperIs(GT_ADD, GT_MUL));
+
+            if (!emitter::IsExtendedReg(op1reg) && emitter::IsExtendedReg(op2reg))
+            {
+                // We have a VEX encoded commutative instruction in which
+                // case we want to try to put the non-extended register as
+                // op2 since this may allow the 2-byte VEX prefix to be used.
+                //
+                // This is not handled by the general path because the scalar
+                // instructions copy the upper bits from op1 and so it is
+                // normally not safe. It is safe here because we know we're
+                // emitting the instruction for a non-SIMD path and so the
+                // upper bits are irrelevant.
+
+                std::swap(op1, op2);
+                std::swap(op1reg, op2reg);
+            }
+        }
         inst_RV_RV_TT(ins, emitTypeSize(treeNode), targetReg, op1reg, op2, isRMW, INS_OPTS_NONE);
 
         genProduceReg(treeNode);

src/coreclr/jit/emitxarch.cpp

@@ -669,6 +669,18 @@ bool emitter::IsThreeOperandAVXInstruction(instruction ins) const
     return (flags & INS_Flags_Is3OperandInstructionMask) != 0;
 }
 
+// Returns true if the AVX instruction has op1/op2 being commutative
+bool emitter::IsAvxCommutative(instruction ins) const
+{
+    if (!UseVEXEncoding())
+    {
+        return false;
+    }
+
+    insFlags flags = CodeGenInterface::instInfo[ins];
+    return (flags & INS_Flags_IsAvxCommutative) != 0;
+}
+
 //------------------------------------------------------------------------
 // HasRegularWideForm: Many x86/x64 instructions follow a regular encoding scheme where the
 // byte-sized version of an instruction has the lowest bit of the opcode cleared
@@ -2717,7 +2729,7 @@ regNumber AbsRegNumber(regNumber reg)
 // Since XMM registers overlap with YMM registers, this routine
 // can also be used to know whether a YMM register if the
 // instruction in question is AVX.
-bool IsExtendedReg(regNumber reg)
+bool emitter::IsExtendedReg(regNumber reg)
 {
 #ifdef TARGET_AMD64
     return ((reg >= REG_R8) && (reg <= REG_R31)) || ((reg >= REG_XMM8) && (reg <= REG_XMM31));
@@ -2749,7 +2761,7 @@ bool emitter::IsExtendedGPReg(regNumber reg) const
 }
 
 // Returns true if using this register, for the given EA_SIZE(attr), will require a REX.* prefix
-bool IsExtendedReg(regNumber reg, emitAttr attr)
+bool emitter::IsExtendedReg(regNumber reg, emitAttr attr)
 {
 #ifdef TARGET_AMD64
     // Not a register, so doesn't need a prefix
@@ -10024,6 +10036,16 @@ void emitter::emitIns_SIMD_R_R_R(
 {
     if (UseSimdEncoding())
     {
+        if (IsAvxCommutative(ins) && (instOptions == INS_OPTS_NONE))
+        {
+            if (!IsExtendedReg(op1Reg) && IsExtendedReg(op2Reg))
+            {
+                // We have a VEX encoded commutative instruction in which
+                // case we want to try to put the non-extended register as
+                // op2 since this may allow the 2-byte VEX prefix to be used.
+                std::swap(op1Reg, op2Reg);
+            }
+        }
         emitIns_R_R_R(ins, attr, targetReg, op1Reg, op2Reg, instOptions);
     }
     else

src/coreclr/jit/emitxarch.h

@@ -254,7 +254,9 @@ bool hasRex2Prefix(code_t code)
 #endif
 }
 
-bool IsExtendedGPReg(regNumber reg) const;
+static bool IsExtendedReg(regNumber reg);
+static bool IsExtendedReg(regNumber reg, emitAttr attr);
+bool        IsExtendedGPReg(regNumber reg) const;
 
 //------------------------------------------------------------------------
 // HasKMaskRegisterDest: Temporary check to identify instructions that can
@@ -765,6 +767,7 @@ void SetContainsCallNeedingVzeroupper(bool value)
 bool        IsDstDstSrcAVXInstruction(instruction ins) const;
 bool        IsDstSrcSrcAVXInstruction(instruction ins) const;
 bool        IsThreeOperandAVXInstruction(instruction ins) const;
+bool        IsAvxCommutative(instruction ins) const;
 static bool HasRegularWideForm(instruction ins);
 static bool HasRegularWideImmediateForm(instruction ins);
 static bool DoesWriteZeroFlag(instruction ins);

src/coreclr/jit/instr.h

@@ -188,6 +188,10 @@ enum insFlags : uint64_t
     INS_Flags_IsDstSrcSrcAVXInstruction = 1ULL << 27,
     INS_Flags_Is3OperandInstructionMask = (INS_Flags_IsDstDstSrcAVXInstruction | INS_Flags_IsDstSrcSrcAVXInstruction),
 
+    // The instruction is commutative for op1/op2 and so can have
+    // these operands swapped if it will result in a smaller encoding.
+    INS_Flags_IsAvxCommutative = 1ULL << 28,
+
     // w and s bits
     INS_FLAGS_Has_Wbit = 1ULL << 29,
     INS_FLAGS_Has_Sbit = 1ULL << 30,