Index: lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp =================================================================== --- lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp (revision 0) +++ lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp (revision 0) @@ -0,0 +1,243 @@ +//===-- LegalizeVectorOps.cpp - Implement SelectionDAG::LegalizeVectors ---===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file implements the SelectionDAG::LegalizeVectors method. +// +//===----------------------------------------------------------------------===// + +#include "llvm/CodeGen/SelectionDAG.h" +#include "llvm/Target/TargetLowering.h" +using namespace llvm; + +namespace { +class VectorLegalizer { + SelectionDAG& DAG; + TargetLowering& TLI; + SDValue UnrollVectorOp(SDValue Op); + SDValue PromoteVectorOp(SDValue Op); + + public: + bool Run(); + VectorLegalizer(SelectionDAG& dag) : + DAG(dag), TLI(dag.getTargetLoweringInfo()) {} +}; + +bool VectorLegalizer::Run() { + bool Changed = false; + + // The vector legalizer is a relatively simple process because it doesn't + // need to legalize everything: it just needs to catch vector operations + // which might expand to libcalls. + DAG.AssignTopologicalOrder(); + for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(), + E = prior(DAG.allnodes_end()); I != next(E); ++I) { + bool HasVectorValue = false; + for (SDNode::value_iterator J = I->value_begin(); J != I->value_end(); ++J) + HasVectorValue |= J->isVector(); + if (!HasVectorValue) continue; + SDNode* Result = I; + switch (I->getOpcode()) { + default: + assert(I->getOpcode() > ISD::BUILTIN_OP_END && "Unexpected node!"); + break; + case ISD::UNDEF: + case ISD::FORMAL_ARGUMENTS: + case ISD::CALL: + case ISD::MERGE_VALUES: + case ISD::RET: + case ISD::VAARG: + case ISD::Register: + case ISD::INTRINSIC_WO_CHAIN: + case ISD::INTRINSIC_W_CHAIN: + case ISD::INTRINSIC_VOID: + case ISD::CopyToReg: + case ISD::CopyFromReg: + case ISD::AssertSext: + case ISD::AssertZext: + // Node cannot be illegal if types are legal + break; + case ISD::BUILD_VECTOR: + case ISD::INSERT_VECTOR_ELT: + case ISD::EXTRACT_VECTOR_ELT: + case ISD::CONCAT_VECTORS: + case ISD::EXTRACT_SUBVECTOR: + case ISD::VECTOR_SHUFFLE: + case ISD::SCALAR_TO_VECTOR: + case ISD::BIT_CONVERT: + case ISD::LOAD: + case ISD::STORE: + // These are intentionally not handled here; the point of this is to + // eliminate illegal operations that could potentially generate libcalls, + // not to completely legalize vectors. Also, leaving these around + // exposes them to both custom legalization and later DAGCombines. + case ISD::ZERO_EXTEND: + case ISD::ANY_EXTEND: + case ISD::TRUNCATE: + // These don't need handling: they can't generate libcalls. + // FIXME: But it might be nice for optimization to handle them. + break; + case ISD::SIGN_EXTEND: + case ISD::VSETCC: + // FIXME: Needs handling; could concievably generate a libcall, although + // it would be unusual + break; + case ISD::ADD: + case ISD::SUB: + case ISD::MUL: + case ISD::SDIV: + case ISD::UDIV: + case ISD::SREM: + case ISD::UREM: + case ISD::FADD: + case ISD::FSUB: + case ISD::FMUL: + case ISD::FDIV: + case ISD::FREM: + case ISD::AND: + case ISD::OR: + case ISD::XOR: + case ISD::SHL: + case ISD::SRA: + case ISD::SRL: + case ISD::ROTL: + case ISD::ROTR: + case ISD::CTTZ: + case ISD::CTLZ: + case ISD::CTPOP: + case ISD::SELECT: + case ISD::SELECT_CC: + case ISD::SINT_TO_FP: + case ISD::UINT_TO_FP: + case ISD::FP_TO_SINT: + case ISD::FP_TO_UINT: + case ISD::FNEG: + case ISD::FABS: + case ISD::FSQRT: + case ISD::FSIN: + case ISD::FCOS: + case ISD::FPOWI: + case ISD::FPOW: + case ISD::FLOG: + case ISD::FLOG2: + case ISD::FLOG10: + case ISD::FEXP: + case ISD::FEXP2: + case ISD::FCEIL: + case ISD::FTRUNC: + case ISD::FRINT: + case ISD::FNEARBYINT: + case ISD::FFLOOR: + switch (TLI.getOperationAction(I->getOpcode(), I->getValueType(0))) { + case TargetLowering::Promote: + Result = PromoteVectorOp(SDValue(Result, 0)).getNode(); + break; + case TargetLowering::Legal: break; + case TargetLowering::Custom: { + SDValue Tmp1 = TLI.LowerOperation(SDValue(Result, 0), DAG); + if (Tmp1.getNode()) { + // FIXME: Should the returned value be recursively checked? + Result = Tmp1.getNode(); + break; + } + // FALL THROUGH + } + case TargetLowering::Expand: + // FIXME: Handle some special cases: FNEG->FSUB + Result = UnrollVectorOp(SDValue(Result, 0)).getNode(); + break; + } + break; + } + if (&*I != Result) { + Changed = true; + DAG.ReplaceAllUsesWith(I, Result); + } + } + + // Remove dead nodes now. + DAG.RemoveDeadNodes(); + + return Changed; +} + +SDValue VectorLegalizer::PromoteVectorOp(SDValue Op) { + MVT VT = Op.getValueType(); + assert(Op.getNode()->getNumValues() == 1 && + "Can't promote a vector with multiple results!"); + MVT NVT = TLI.getTypeToPromoteTo(Op.getOpcode(), VT); + DebugLoc dl = Op.getDebugLoc(); + SmallVector Operands(Op.getNumOperands()); + + for (unsigned j = 0; j != Op.getNumOperands(); ++j) { + if (Op.getOperand(j).getValueType().isVector()) + Operands[j] = DAG.getNode(ISD::BIT_CONVERT, dl, NVT, Op.getOperand(j)); + else + Operands[j] = Op.getOperand(j); + } + + Op = DAG.getNode(Op.getOpcode(), dl, NVT, &Operands[0], Operands.size()); + + return DAG.getNode(ISD::BIT_CONVERT, dl, VT, Op); +} + +/// UnrollVectorOp - We know that the given vector has a legal type, however +/// the operation it performs is not legal, and the target has requested that +/// the operation be expanded. "Unroll" the vector, splitting out the scalars +/// and operating on each element individually. +SDValue VectorLegalizer::UnrollVectorOp(SDValue Op) { + MVT VT = Op.getValueType(); + assert(Op.getNode()->getNumValues() == 1 && + "Can't unroll a vector with multiple results!"); + unsigned NE = VT.getVectorNumElements(); + MVT EltVT = VT.getVectorElementType(); + DebugLoc dl = Op.getDebugLoc(); + + SmallVector Scalars; + SmallVector Operands(Op.getNumOperands()); + for (unsigned i = 0; i != NE; ++i) { + for (unsigned j = 0; j != Op.getNumOperands(); ++j) { + SDValue Operand = Op.getOperand(j); + MVT OperandVT = Operand.getValueType(); + if (OperandVT.isVector()) { + // A vector operand; extract a single element. + MVT OperandEltVT = OperandVT.getVectorElementType(); + Operands[j] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, + OperandEltVT, + Operand, + DAG.getConstant(i, MVT::i32)); + } else { + // A scalar operand; just use it as is. + Operands[j] = Operand; + } + } + + switch (Op.getOpcode()) { + default: + Scalars.push_back(DAG.getNode(Op.getOpcode(), dl, EltVT, + &Operands[0], Operands.size())); + break; + case ISD::SHL: + case ISD::SRA: + case ISD::SRL: + case ISD::ROTL: + case ISD::ROTR: + Scalars.push_back(DAG.getNode(Op.getOpcode(), dl, EltVT, Operands[0], + DAG.getShiftAmountOperand(Operands[1]))); + break; + } + } + + return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &Scalars[0], Scalars.size()); +} + +} + +bool SelectionDAG::LegalizeVectors() { + return VectorLegalizer(*this).Run(); +} Index: lib/CodeGen/SelectionDAG/LegalizeDAG.cpp =================================================================== --- lib/CodeGen/SelectionDAG/LegalizeDAG.cpp (revision 72159) +++ lib/CodeGen/SelectionDAG/LegalizeDAG.cpp (working copy) @@ -495,7 +495,7 @@ // types here except for target constants (the type legalizer does not touch // those) or for build vector used as a mask for a vector shuffle. assert((TypesNeedLegalizing || getTypeAction(VT) == Legal || - IsLegalizingCallArgs || Op.getOpcode() == ISD::TargetConstant) && + Op.getOpcode() == ISD::TargetConstant) && "Illegal type introduced after type legalization?"); switch (getTypeAction(VT)) { default: assert(0 && "Bad type action!"); Index: lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp =================================================================== --- lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp (revision 72159) +++ lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp (working copy) @@ -611,6 +611,36 @@ DOUT << "Optimized type-legalized selection DAG:\n"; DEBUG(CurDAG->dump()); } + + if (TimePassesIsEnabled) { + NamedRegionTimer T("Vector Legalization", GroupName); + Changed = CurDAG->LegalizeVectors(); + } else { + Changed = CurDAG->LegalizeVectors(); + } + + if (Changed) { + if (TimePassesIsEnabled) { + NamedRegionTimer T("Type Legalization 2", GroupName); + Changed = CurDAG->LegalizeTypes(); + } else { + Changed = CurDAG->LegalizeTypes(); + } + + if (ViewDAGCombineLT) + CurDAG->viewGraph("dag-combine-lv input for " + BlockName); + + // Run the DAG combiner in post-type-legalize mode. + if (TimePassesIsEnabled) { + NamedRegionTimer T("DAG Combining after legalize vectors", GroupName); + CurDAG->Combine(NoIllegalOperations, *AA, OptLevel); + } else { + CurDAG->Combine(NoIllegalOperations, *AA, OptLevel); + } + + DOUT << "Optimized vector-legalized selection DAG:\n"; + DEBUG(CurDAG->dump()); + } } if (ViewLegalizeDAGs) CurDAG->viewGraph("legalize input for " + BlockName); Index: include/llvm/CodeGen/SelectionDAG.h =================================================================== --- include/llvm/CodeGen/SelectionDAG.h (revision 72159) +++ include/llvm/CodeGen/SelectionDAG.h (working copy) @@ -220,6 +220,13 @@ /// the graph. void Legalize(bool TypesNeedLegalizing, CodeGenOpt::Level OptLevel); + /// LegalizeVectors - This transforms the SelectionDAG into a SelectionDAG + /// that only uses vector operations supported by the target. + /// + /// Note that this is an involved process that may invalidate pointers into + /// the graph. + bool LegalizeVectors(); + /// RemoveDeadNodes - This method deletes all unreachable nodes in the /// SelectionDAG. void RemoveDeadNodes();