/*========================== begin_copyright_notice ============================

Copyright (C) 2017-2026 Intel Corporation

SPDX-License-Identifier: MIT

============================= end_copyright_notice ===========================*/

#include "Compiler/Optimizer/OpenCLPasses/ReplaceUnsupportedIntrinsics/ReplaceUnsupportedIntrinsics.hpp"
#include "Compiler/CodeGenContextWrapper.hpp"
#include "Compiler/CodeGenPublic.h"
#include "Compiler/IGCPassSupport.h"
#include "GenISAIntrinsics/GenIntrinsics.h"
#include "common/igc_regkeys.hpp"

#include "common/LLVMWarningsPush.hpp"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/IntrinsicInst.h"
#include <llvm/ADT/APFloat.h>
#include <llvm/ADT/APInt.h>
#include <llvm/ADT/FloatingPointMode.h>
#include <llvm/IR/Constant.h>
#include <llvm/IR/Constants.h>
#include <llvm/IR/InstVisitor.h>
#include "common/LLVMWarningsPop.hpp"
#include "llvmWrapper/IR/DerivedTypes.h"
#include "llvmWrapper/IR/Intrinsics.h"
#include "llvmWrapper/IR/Instructions.h"
#include "llvmWrapper/IR/IRBuilder.h"
#include "llvmWrapper/Support/Alignment.h"
#include "llvmWrapper/Support/TypeSize.h"

#include <map>
#include "Probe/Assertion.h"

using namespace llvm;
using namespace IGC;
using IGCLLVM::getAlign;

namespace {
/// ReplaceIntrinsics pass lowers calls to unsupported intrinsics functions.
// Two llvm instrinsics are replaced llvm.memcpy and llvm.memset. Both appear in SPIR spec.
class ReplaceUnsupportedIntrinsics : public llvm::FunctionPass, public llvm::InstVisitor<ReplaceUnsupportedIntrinsics> {
public:
  typedef void (ReplaceUnsupportedIntrinsics::*MemFuncPtr_t)(IntrinsicInst *);
  static char ID;

  ReplaceUnsupportedIntrinsics();

  ~ReplaceUnsupportedIntrinsics() {}

  virtual llvm::StringRef getPassName() const override { return "ReplaceUnsupportedIntrinsics"; }

  virtual bool runOnFunction(llvm::Function &F) override;

  virtual void getAnalysisUsage(llvm::AnalysisUsage &AU) const override { AU.addRequired<CodeGenContextWrapper>(); }

  void visitIntrinsicInst(llvm::IntrinsicInst &I);

private:
  CodeGenContext *m_Ctx = nullptr;
  std::vector<llvm::IntrinsicInst *> m_instsToReplace;

  /// Helper
  ///
  // if the value comes from a bitcast, return the source, otherwise return itself
  Value *SkipBitCast(Value *v) {
    if (BitCastInst *bc = dyn_cast<BitCastInst>(v)) {
      // Don't skip if this is a pointer cast and the addrspace changed
      if (v->getType()->isPointerTy() && bc->getOperand(0)->getType()->isPointerTy() &&
          v->getType()->getPointerAddressSpace() != bc->getOperand(0)->getType()->getPointerAddressSpace()) {
        return v;
      }
      v = bc->getOperand(0);
    }
    return v;
  }

  // Get the largest of power-of-2 value that is <= C AND that can divide C.
  uint32_t getLargestPowerOfTwo(uint32_t C) {
    // If C == 0 (shouldn't happen), return a big one.
    return (C == 0) ? 4096 : (C & (~C + 1));
  }

  MemCpyInst *MemMoveToMemCpy(MemMoveInst *MM);
  Instruction *insertReverseLoop(BasicBlock *Loc, BasicBlock *Post, Value *Length, StringRef BBName);
  Instruction *insertLoop(Instruction *Loc, Value *Length, StringRef BBName);
  Value *replicateScalar(Value *ScalarVal, Type *Ty, Instruction *InsertBefore);
  void generalGroupI8Stream(LLVMContext &C, uint32_t NumI8, uint32_t Align, uint32_t &NumI32, Type **Vecs, uint32_t &L,
                            uint32_t BaseTypeSize);
  // support function for replaceCountTheLeadingZeros
  Value *evaluateCtlzUpto32bit(IGCLLVM::IRBuilder<> *Builder, Value *inVal, Type *singleElementType,
                               Value *canBePoison);
  Value *evaluateCtlz64bit(IGCLLVM::IRBuilder<> *Builder, Value *inVal, Type *singleElementType, Value *canBePoison);

  /// replace member function
  void replaceMemcpy(IntrinsicInst *I);
  void replaceMemset(IntrinsicInst *I);
  void replaceMemMove(IntrinsicInst *I);
  void replaceExpect(IntrinsicInst *I);
  void replaceFunnelShift(IntrinsicInst *I);
  void replaceLRound(IntrinsicInst *I);
  void replaceLRint(IntrinsicInst *I);
  void replaceCountTheLeadingZeros(IntrinsicInst *I);
  void replaceCountTheTrailingZeros(IntrinsicInst *I);
  void replaceMinMax(IntrinsicInst *I);
  void replaceI1MinMax(IntrinsicInst *I);
  void replaceI64MinMax(IntrinsicInst *I);
  void replaceHalvesDivsSqrts(IntrinsicInst *I);
#if LLVM_VERSION_MAJOR >= 15
  void replaceIsFpClass(IntrinsicInst *I);
#endif

  static const std::map<Intrinsic::ID, MemFuncPtr_t> m_intrinsicToFunc;
};
} // namespace

// Register pass to igc-opt
#define PASS_FLAG "igc-replace-unsupported-intrinsics"
#define PASS_DESCRIPTION "Replace calls to instrinsics which are not supported by the codegen"
#define PASS_CFG_ONLY false
#define PASS_ANALYSIS false
IGC_INITIALIZE_PASS_BEGIN(ReplaceUnsupportedIntrinsics, PASS_FLAG, PASS_DESCRIPTION, PASS_CFG_ONLY, PASS_ANALYSIS)
IGC_INITIALIZE_PASS_DEPENDENCY(CodeGenContextWrapper)
IGC_INITIALIZE_PASS_END(ReplaceUnsupportedIntrinsics, PASS_FLAG, PASS_DESCRIPTION, PASS_CFG_ONLY, PASS_ANALYSIS)

char ReplaceUnsupportedIntrinsics::ID = 0;

const std::map<Intrinsic::ID, ReplaceUnsupportedIntrinsics::MemFuncPtr_t>
    ReplaceUnsupportedIntrinsics::m_intrinsicToFunc = {
        // clang-format off
    { Intrinsic::fshl,                          &ReplaceUnsupportedIntrinsics::replaceFunnelShift },
    { Intrinsic::fshr,                          &ReplaceUnsupportedIntrinsics::replaceFunnelShift },
    { Intrinsic::memcpy,                        &ReplaceUnsupportedIntrinsics::replaceMemcpy },
    { Intrinsic::memset,                        &ReplaceUnsupportedIntrinsics::replaceMemset },
    { Intrinsic::memmove,                       &ReplaceUnsupportedIntrinsics::replaceMemMove },
    { Intrinsic::expect,                        &ReplaceUnsupportedIntrinsics::replaceExpect },
    { Intrinsic::lround,                        &ReplaceUnsupportedIntrinsics::replaceLRound },
    { Intrinsic::llround,                       &ReplaceUnsupportedIntrinsics::replaceLRound },
    { Intrinsic::lrint,                         &ReplaceUnsupportedIntrinsics::replaceLRint },
    { Intrinsic::llrint,                        &ReplaceUnsupportedIntrinsics::replaceLRint },
    { Intrinsic::ctlz,                          &ReplaceUnsupportedIntrinsics::replaceCountTheLeadingZeros },
    { Intrinsic::cttz,                          &ReplaceUnsupportedIntrinsics::replaceCountTheTrailingZeros },
    { Intrinsic::smax,                          &ReplaceUnsupportedIntrinsics::replaceMinMax },
    { Intrinsic::smin,                          &ReplaceUnsupportedIntrinsics::replaceMinMax },
    { Intrinsic::umax,                          &ReplaceUnsupportedIntrinsics::replaceMinMax },
    { Intrinsic::umin,                          &ReplaceUnsupportedIntrinsics::replaceMinMax },
    { Intrinsic::experimental_constrained_fdiv, &ReplaceUnsupportedIntrinsics::replaceHalvesDivsSqrts},
    { Intrinsic::experimental_constrained_sqrt, &ReplaceUnsupportedIntrinsics::replaceHalvesDivsSqrts},
#if LLVM_VERSION_MAJOR >= 15
    { Intrinsic::is_fpclass,                    &ReplaceUnsupportedIntrinsics::replaceIsFpClass },
#endif
        // clang-format on
};

ReplaceUnsupportedIntrinsics::ReplaceUnsupportedIntrinsics() : FunctionPass(ID) {
  initializeReplaceUnsupportedIntrinsicsPass(*PassRegistry::getPassRegistry());
}

MemCpyInst *ReplaceUnsupportedIntrinsics::MemMoveToMemCpy(MemMoveInst *MM) {
  SmallVector<Value *, 5> args;
  for (unsigned i = 0; i < IGCLLVM::getNumArgOperands(MM); i++)
    args.push_back(MM->getArgOperand(i));

  auto *Dst = MM->getRawDest();
  auto *Src = MM->getRawSource();
  auto *Size = MM->getLength();

  Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
  auto *M = MM->getParent()->getParent()->getParent();
  auto TheFn = IGCLLVM::getOrInsertDeclaration(M, Intrinsic::memcpy, Tys);

  return cast<MemCpyInst>(MemCpyInst::Create(TheFn, args));
}

// insertReverseLoop - Insert an empty loop at the end of BB 'Loc'.
// The loop's induction variable iterates from 'Length'-1 to 0.
// The return value is the value of the induction variable in the loop's body.
Instruction *ReplaceUnsupportedIntrinsics::insertReverseLoop(BasicBlock *Loc, BasicBlock *Post, Value *Length,
                                                             StringRef BBName) {
  DebugLoc DL = Loc->getTerminator()->getDebugLoc();
  Function *F = Loc->getParent();
  LLVMContext &C = F->getContext();
  IntegerType *LengthType = cast<IntegerType>(Length->getType());
  // Create an alloca for storing the loop's induction variable
  Value *pIV = new AllocaInst(LengthType, 0, "pIV", &(*F->getEntryBlock().begin()));
  // Split the BB at the location of the call
  BasicBlock *Pre = Loc;
  // Create a new BB for the loop Body
  BasicBlock *Body = BasicBlock::Create(C, Twine(BBName) + ".body", F, Post);
  ConstantInt *Zero = ConstantInt::get(LengthType, 0);
  ConstantInt *One = ConstantInt::get(LengthType, 1);
  {
    // Remove the unconditional 'br' instruction which will be replaced by a conditional 'br'
    Pre->getTerminator()->eraseFromParent();
    IGCLLVM::IRBuilder<> B(Pre);
    B.SetCurrentDebugLocation(DL);
    // Init the IV
    auto *Init = B.CreateSub(Length, One);
    B.CreateStore(Init, pIV);
    Value *IsContinue = B.CreateICmpSGE(Init, Zero);
    B.CreateCondBr(IsContinue, Body, Post);
  }
  // The induction variable's value
  Instruction *IV;
  {
    // Loop body's Basic Block
    IGCLLVM::IRBuilder<> B(Body);
    B.SetCurrentDebugLocation(DL);
    IV = B.CreateLoad(cast<llvm::AllocaInst>(pIV)->getAllocatedType(), pIV, "IV");
    // User of function will add more instructions at this point ...
    // Decrement the IV and check for end of loop
    Value *Dec = B.CreateSub(IV, One);
    B.CreateStore(Dec, pIV);
    Value *IsContinue = B.CreateICmpSGE(Dec, Zero);
    B.CreateCondBr(IsContinue, Body, Post);
  }
  return IV;
}

// insertLoop - Insert an empty loop before instruction 'Loc'.
// The loop's induction variable iterates from 0 to 'Length'-1.
// The return value is the value of the induction variable in the loop's body.
Instruction *ReplaceUnsupportedIntrinsics::insertLoop(Instruction *Loc, Value *Length, StringRef BBName) {
  DebugLoc DL = Loc->getDebugLoc();
  Function *F = Loc->getParent()->getParent();
  LLVMContext &C = F->getContext();
  IntegerType *LengthType = cast<IntegerType>(Length->getType());
  // Create an alloca for storing the loop's induction variable
  Value *pIV = new AllocaInst(LengthType, 0, "pIV", &(*F->getEntryBlock().begin()));
  // Split the BB at the location of the call
  BasicBlock *Pre = Loc->getParent();
  BasicBlock *Post = Pre->splitBasicBlock(BasicBlock::iterator(Loc), Twine(BBName) + ".post");
  // Create a new BB for the loop Body
  BasicBlock *Body = BasicBlock::Create(C, Twine(BBName) + ".body", F, Post);
  {
    // Remove the unconditional 'br' instruction which will be replaced by a conditional 'br'
    Pre->getTerminator()->eraseFromParent();
    IGCLLVM::IRBuilder<> B(Pre);
    B.SetCurrentDebugLocation(DL);
    ConstantInt *Zero = ConstantInt::get(LengthType, 0);
    // Init the IV
    B.CreateStore(Zero, pIV);
    Value *IsContinue = B.CreateICmpULT(Zero, Length);
    B.CreateCondBr(IsContinue, Body, Post);
  }
  // The induction variable's value
  Instruction *IV;
  {
    // Loop body's Basic Block
    IGCLLVM::IRBuilder<> B(Body);
    B.SetCurrentDebugLocation(DL);
    IV = B.CreateLoad(cast<llvm::AllocaInst>(pIV)->getAllocatedType(), pIV, "IV");
    // User of function will add more instructions at this point ...
    // Increment the IV and check for end of loop
    Value *Inc = B.CreateAdd(IV, ConstantInt::get(LengthType, 1));
    B.CreateStore(Inc, pIV);
    Value *IsContinue = B.CreateICmpULT(Inc, Length);
    B.CreateCondBr(IsContinue, Body, Post);
  }
  return IV;
}

Value *ReplaceUnsupportedIntrinsics::replicateScalar(Value *ScalarVal, Type *Ty, Instruction *InsertBefore) {
  IGCLLVM::FixedVectorType *VTy = dyn_cast<IGCLLVM::FixedVectorType>(Ty);
  Type *ETy = VTy ? VTy->getElementType() : Ty;
  uint32_t sBits = (unsigned int)ScalarVal->getType()->getPrimitiveSizeInBits();
  uint32_t nBits = (unsigned int)ETy->getPrimitiveSizeInBits();
  IGC_ASSERT(sBits);
  IGC_ASSERT_MESSAGE((nBits % sBits) == 0, "Type mismatch in replicateScalar!");
  IGC_ASSERT_MESSAGE(nBits <= 64, "Type mismatch in replicateScalar!");
  uint32_t ratio = nBits / sBits;

  IGCLLVM::IRBuilder<> Builder(InsertBefore);
  Value *NewVal;
  if (ratio > 1) {
    if (ConstantInt *CI = dyn_cast<ConstantInt>(ScalarVal)) {
      uint64_t s = CI->getZExtValue();
      uint64_t n = s;
      for (unsigned i = 1; i < ratio; ++i) {
        n = (n << sBits) | s;
      }
      NewVal = ConstantInt::get(ETy, n);
    } else {
      Value *nScalarVal = Builder.CreateZExt(ScalarVal, ETy);
      NewVal = nScalarVal;
      for (unsigned i = 1; i < ratio; ++i) {
        NewVal = Builder.CreateShl(NewVal, sBits);
        NewVal = Builder.CreateAdd(NewVal, nScalarVal);
      }
    }
  } else {
    NewVal = ScalarVal;
  }

  Value *Res;
  if (VTy) {
    Res = UndefValue::get(VTy);
    Type *TyI32 = Type::getInt32Ty(ScalarVal->getContext());
    for (unsigned i = 0; i < VTy->getNumElements(); ++i) {
      Value *Idx = ConstantInt::get(TyI32, i);
      Res = Builder.CreateInsertElement(Res, NewVal, Idx);
    }
  } else {
    Res = NewVal;
  }
  return Res;
}

// A help functions to generate vector load or stores for efficient
// memory operations.
// However, if size of base type is to kept the generated vectors will be different
// <8xi32>  for size of the base type = 32
// <16xi16> for size of the base type = 16
//
// generalGroupI8Stream() groups a stream of i8 into a stream of <8xi32> or <16xi16> as
// much as possible. Then for the remaining i8's ( < 32), group them
// into vectors of element type i32(i16) and/or i8. This results in at most
// the following 5 vectors and/or scalars:
//    <4xi32>, <3xi32> or <2xi32>, i32, <2xi8>, i8 or
//    <8xi16>, <4xi16>, <2xi16>, i16, i8
// Note that we will not generate <3xi8> (see also the code for details).
// For example, given 127 i8's, we can
// have:
//   <8xi32>, <8xi32>, <8xi32>, <4xi32>, <3xi32>, <2xi8>, i8
//
// The grouping result are kept in Vecs, L (actual length of Vecs),
// and NumI32 (the number of <8xi32>, ie. the number of Vecs[0]. For all
// the other vectors/scalars, ie Vecs[1 : L-1], the number is always 1).
// For the above case, they are:
//    Vecs[0] = <8xi32>
//    Vecs[1] = <4xi32>
//    Vecs[2] = <3xi32>
//    Vecs[3] = <2xi8>
//    Vecs[4] = i8
//    L = 5;
//    NumI32 = 3;
//
// We may generate <3xi32>, but not <3xi8> as <3xi32> can be loaded
// or stored by a single send instruction, where <3xi8> cannot (even
// <3xi8> can be splitted later in VectorProcessing, but it's better
// not generate <3xi8> vector in the first place).
//
// The same example with given 127 i8's but with keeping size of base
// type of initial vector as 16 we can have:
//   <16xi16>, <16xi16>, <16xi16>, <8xi16>, <4xi16>, <2xi16>, i16, i8
//
// The grouping result are kept in Vecs, L (actual length of Vecs),
// and NumI32 (the number of <16xi16>, ie. the number of Vecs[0]. For all
// the other vectors/scalars, ie Vecs[1 : L-1], the number is always 1).
// For the above case, they are:
//    Vecs[0] = <16xi16>
//    Vecs[1] = <8xi16>
//    Vecs[2] = <4xi16>
//    Vecs[3] = <2xi16>
//    Vecs[4] = i16
//    Vecs[5] = i8
//    L = 6;
//    NumI32 = 3;
//
// Note that Vecs[] should be allocated by callers with enough space
// to hold all vectors (6 should be enough; 1 for <8xi32>(<16xi16>),
// 5 for the others).
// We want from <4x<2xhalf>> [with size of the base type 16(half)]
// generate <8xi16> not <4xi32>
// Default BaseTypeSize=32 means that we don't concern about keeping
// size of the base type
void ReplaceUnsupportedIntrinsics::generalGroupI8Stream(LLVMContext &C, uint32_t NumI8, uint32_t Align,
                                                        uint32_t &VectorsNum, Type **Vecs, uint32_t &L,
                                                        uint32_t BaseTypeSize = 32) {
  VectorsNum = NumI8 / 32; // size of <8xi32> = 32. count of <8xi32> or <16xi16>
  uint32_t RemI8 = NumI8 % 32;
  uint32_t BaseTypeSizeInBytes = BaseTypeSize / 8;
  uint32_t CntI = RemI8 / BaseTypeSizeInBytes;  // the number of i32(0..7) or i16(0..15)
  uint32_t CntI8 = RemI8 % BaseTypeSizeInBytes; // remaining number of i8(0-3) - for base_type_size = 32 or
                                                //                     i8(0-1) - for base_type_size = 16

  // To process all cases (3 for i32 and 4 for i16: it depends of how much CntI do we have)
  uint32_t Power = 256 / BaseTypeSize; // i32: (256 / 32) = 0b1000 = 8
                                       // i16: (256 / 16) = 0b10000 = 16

  Type *BaseType = Type::getIntNTy(C, BaseTypeSize);
  Type *TyI8 = Type::getInt8Ty(C);

  uint32_t n = 0;
  Vecs[n++] = IGCLLVM::FixedVectorType::get(BaseType, Power);

  while ((Power >>= 1) > 1) {
    if (CntI >= Power) {
      Vecs[n++] = IGCLLVM::FixedVectorType::get(BaseType, Power);
      CntI -= Power;
    }
    if (CntI == 3 && BaseTypeSize == 32 &&
        Align >= 4) // special case for <8xi32> not to generate <3xi8> but to generate <3xi32>
    {
      Vecs[n++] = IGCLLVM::FixedVectorType::get(BaseType, 3);
      CntI = 0;
      break;
    }
  }
  if (CntI >= 1) {
    Vecs[n++] = BaseType;
    CntI -= Power; // Assume that pow should be 1 to generate i32(i16) and not <1xi32>(<1xi16>)
  }
  IGC_ASSERT_MESSAGE(CntI == 0, "Did not handle all types of base_type");

  Power = BaseTypeSize / 4; // i32: 32 / 8 = 4
                            // i16: 16 / 8 = 2
  while ((Power >>= 1) > 1) {
    if (CntI8 >= Power) {
      Vecs[n++] = IGCLLVM::FixedVectorType::get(TyI8, Power);
      CntI8 -= Power;
    }
  }
  if (CntI8 >= 1) {
    Vecs[n++] = TyI8;
    CntI8 -= Power; // Assume that pow should be 1 to generate i8 not <1xi8>
  }
  IGC_ASSERT_MESSAGE(CntI8 == 0, "Did not handle all types of I8");

  L = n;
}

void ReplaceUnsupportedIntrinsics::replaceMemcpy(IntrinsicInst *I) {
  // The idea is to convert
  //
  //   memcpy (i8* Dst, i8* Src, len)
  //
  // into a vector load and store for cases where "len" is
  // constant. If "len" isn't constant,  just use i8 copy as
  // this should not happen with OCL code (all memcpy is
  // generated by the compiler for cases such as structure
  // assignment, etc.)
  //
  // If len is constant, it will be transferred to
  //
  //   lenv8 = len / 32 (<8xi32>);
  //   len_rem = len % 32;
  //
  //   // main loop
  //   dstV8 = bitcast Dst, <8xi32>*
  //   srcV8 = bitcast Src, <8xi32>*
  //   for(i=0; i < lenv8; ++i)
  //     dstV8[i] = srcV8[i];
  //
  //   // epilog, process remaining elements
  //   for(i=0; i < len_rem; ++i)
  //     Dst[lenv8*32 + i] = Src[lenv8*32 + i];
  //
  //   Note that the above epilog loop is optimized away with
  //   as much as possible <nxi32> and <mxi8> loads and stores
  //   or if we want to keep size of the base type
  //   (for 16bit there will be <nxi16> and <mxi8>)
  //
  // Selecting 8 as vector length or 16 in case of i16 is due to
  // that A64 messages can load eight i32 or sixteen i16 per SIMD channel.
  // A32 will have 2 loads/stores for each vector, which is still efficient.
  // Unaligned vector will be handled correctly and effciently later
  // in vector load and store emit.
  MemCpyInst *MC = cast<MemCpyInst>(I);
  Value *Dst = MC->getRawDest();
  Value *Src = MC->getRawSource();
  Value *LPCount = MC->getLength();
  uint32_t Align = MC->getDestAlign().valueOrOne().value();
  Align = Align != 0 ? Align : 1;
  uint32_t SrcAlign = MC->getSourceAlign().valueOrOne().value();
  SrcAlign = SrcAlign != 0 ? SrcAlign : 1;
  const bool IsVolatile = MC->isVolatile();
  const uint32_t SrcAS = MC->getSourceAddressSpace();
  const uint32_t DstAS = MC->getDestAddressSpace();

  LLVMContext &C = MC->getContext();
  Type *TySrcPtrI8 = Type::getInt8PtrTy(C, SrcAS);
  Type *TyDstPtrI8 = Type::getInt8PtrTy(C, DstAS);

  IGCLLVM::IRBuilder<> Builder(MC);

  // BaseSize == 32 if we want to handle algorithm in general way
  // or different value if want to keep size of base type to further optimizations
  PointerType *ptrTy = cast<PointerType>(Dst->stripPointerCasts()->getType());
  uint32_t BaseSize = 0;
  Type *RawDstType = IGCLLVM::isPointerTy(ptrTy)
                         ? Builder.getInt8Ty()
                         : IGCLLVM::getNonOpaquePtrEltTy(ptrTy); // Legacy code: getNonOpaquePtrEltTy
  if (Type *BaseType = GetBaseType(RawDstType))
    BaseSize = BaseType->getScalarSizeInBits();

  if (BaseSize != 16)
    // size 32 is equal to size of i32, so general algorithm will be applied
    BaseSize = 32;

  ConstantInt *CI = dyn_cast<ConstantInt>(LPCount);
  if (CI) {
    uint32_t Count = (uint32_t)CI->getZExtValue();

    Type *VecTys[8];
    uint32_t Len, NewCount;
    generalGroupI8Stream(C, Count, Align, NewCount, VecTys, Len, BaseSize);

    Value *NewSrc, *NewDst, *vDst, *vSrc;
    uint32_t BOfst = 0; // Byte offset

    // First, insert main loop before MC.
    // Note that if NewCount is small, we may directly generate ld/st
    // without generating the loop.
    if (NewCount > 0) {
      vSrc = Builder.CreateBitCast(SkipBitCast(Src), PointerType::get(VecTys[0], SrcAS), "memcpy_vsrc");
      vDst = Builder.CreateBitCast(SkipBitCast(Dst), PointerType::get(VecTys[0], DstAS), "memcpy_vdst");

      // getPrimitiveSizeInBits() should be enough, no need to
      // use DataLayout to get target-dependent size.
      uint32_t SZ = (unsigned int)(VecTys[0]->getPrimitiveSizeInBits() / 8);

      // To set alignment correctly
      uint32_t adjust_align = getLargestPowerOfTwo(SZ);
      Align = adjust_align < Align ? adjust_align : Align;
      SrcAlign = adjust_align < SrcAlign ? adjust_align : SrcAlign;

      // If NewCount is less than the threshold, don't generate loop.
      if (NewCount < IGC_GET_FLAG_VALUE(MemCpyLoweringUnrollThreshold)) {
        for (unsigned i = 0; i < NewCount; ++i) {
          Value *tSrc = Builder.CreateConstGEP1_32(VecTys[0], vSrc, i);
          Value *tDst = Builder.CreateConstGEP1_32(VecTys[0], vDst, i);
          LoadInst *L = Builder.CreateAlignedLoad(VecTys[0], tSrc, getAlign(SrcAlign), IsVolatile);
          (void)Builder.CreateAlignedStore(L, tDst, getAlign(Align), IsVolatile);
        }
      } else {
        Value *NewLPCount = ConstantInt::get(LPCount->getType(), NewCount);
        Instruction *IV = insertLoop(MC, NewLPCount, "memcpy");
        {
          IGCLLVM::IRBuilder<> B(&(*++BasicBlock::iterator(IV)));
          Value *tSrc = B.CreateGEP(VecTys[0], vSrc, IV);
          Value *tDst = B.CreateGEP(VecTys[0], vDst, IV);
          LoadInst *L = B.CreateAlignedLoad(VecTys[0], tSrc, getAlign(SrcAlign), IsVolatile);
          (void)B.CreateAlignedStore(L, tDst, getAlign(Align), IsVolatile);
        }
      }

      BOfst = NewCount * SZ;
    }

    // Second, generate epilog code before MC.
    // Note that as MC has been moved to a different BB by
    //   inserting the main loop! Reset it to MC.
    Builder.SetInsertPoint(MC);
    if (Len > 1) {
      Src = Builder.CreateBitCast(SkipBitCast(Src), TySrcPtrI8, "memcpy_src");
      Dst = Builder.CreateBitCast(SkipBitCast(Dst), TyDstPtrI8, "memcpy_dst");
    }
    for (unsigned i = 1; i < Len; ++i) {
      uint32_t SZ = (unsigned int)VecTys[i]->getPrimitiveSizeInBits() / 8;
      uint32_t adjust_align = getLargestPowerOfTwo(SZ);
      Align = adjust_align < Align ? adjust_align : Align;
      SrcAlign = adjust_align < SrcAlign ? adjust_align : SrcAlign;
      NewSrc = BOfst > 0 ? Builder.CreateConstGEP1_32(Builder.getInt8Ty(), Src, BOfst) : Src;
      NewDst = BOfst > 0 ? Builder.CreateConstGEP1_32(Builder.getInt8Ty(), Dst, BOfst) : Dst;
      vSrc = Builder.CreateBitCast(SkipBitCast(NewSrc), PointerType::get(VecTys[i], SrcAS), "memcpy_rem");
      vDst = Builder.CreateBitCast(SkipBitCast(NewDst), PointerType::get(VecTys[i], DstAS), "memcpy_rem");
      LoadInst *L = Builder.CreateAlignedLoad(VecTys[i], vSrc, getAlign(SrcAlign), IsVolatile);
      (void)Builder.CreateAlignedStore(L, vDst, getAlign(Align), IsVolatile);
      BOfst += SZ;
    }
  } else {
    Src = Builder.CreateBitCast(SkipBitCast(Src), TySrcPtrI8, "memcpy_src");
    Dst = Builder.CreateBitCast(SkipBitCast(Dst), TyDstPtrI8, "memcpy_dst");
    // Fall back to i8 copy
    Instruction *IV = insertLoop(MC, LPCount, "memcpy");
    {
      IGCLLVM::IRBuilder<> B(&(*++BasicBlock::iterator(IV)));
      Value *tSrc = B.CreateGEP(Builder.getInt8Ty(), Src, IV);
      Value *tDst = B.CreateGEP(Builder.getInt8Ty(), Dst, IV);
      LoadInst *L = B.CreateAlignedLoad(Builder.getInt8Ty(), tSrc, getAlign(SrcAlign), IsVolatile);
      (void)B.CreateAlignedStore(L, tDst, getAlign(Align), IsVolatile);
    }
  }
  MC->eraseFromParent();
}

void ReplaceUnsupportedIntrinsics::replaceMemMove(IntrinsicInst *I) {
  MemMoveInst *MM = cast<MemMoveInst>(I);
  Value *Dst = MM->getRawDest();
  Value *Src = MM->getRawSource();
  Value *LPCount = MM->getLength();
  uint32_t Align = MM->getDestAlign().valueOrOne().value();
  if (Align == 0)
    Align = 1;
  const bool IsVolatile = MM->isVolatile();
  const uint32_t SrcAS = MM->getSourceAddressSpace();
  const uint32_t DstAS = MM->getDestAddressSpace();

  // If non-generic address spaces mismatch, they can't alias
  // and we can do a memcpy.

  if (SrcAS < ADDRESS_SPACE_NUM_ADDRESSES && DstAS < ADDRESS_SPACE_NUM_ADDRESSES && SrcAS != ADDRESS_SPACE_GENERIC &&
      DstAS != ADDRESS_SPACE_GENERIC && SrcAS != DstAS) {
    auto *MemCpy = MemMoveToMemCpy(MM);
    MemCpy->insertBefore(MM);
    replaceMemcpy(MemCpy);
    MM->eraseFromParent();
    return;
  }

  LLVMContext &C = MM->getContext();
  Type *TySrcPtrI8 = Type::getInt8PtrTy(C, SrcAS);
  Type *TyDstPtrI8 = Type::getInt8PtrTy(C, DstAS);

  auto *F = MM->getParent()->getParent();

  IGCLLVM::IRBuilder<> B(MM);

  auto *i8Src = B.CreateBitCast(SkipBitCast(Src), TySrcPtrI8, "memcpy_src");
  auto *i8Dst = B.CreateBitCast(SkipBitCast(Dst), TyDstPtrI8, "memcpy_dst");

  // Setup control flow to do:
  // if (Src < Dst)
  //   reverse copy data
  // else
  //   normal copy (such as memcpy())

  // Src < Dst
  Value *pCmp = nullptr;
  {
    auto *cmpCastSrc = (DstAS == ADDRESS_SPACE_GENERIC) ? B.CreateAddrSpaceCast(i8Src, TyDstPtrI8) : i8Src;
    auto *cmpCastDst = (SrcAS == ADDRESS_SPACE_GENERIC) ? B.CreateAddrSpaceCast(i8Dst, TySrcPtrI8) : i8Dst;

    pCmp = B.CreateICmpULT(cmpCastSrc, cmpCastDst);
  }

  auto *Pre = MM->getParent();
  auto *Post = Pre->splitBasicBlock(MM, "memmove.post");

  Pre->getTerminator()->eraseFromParent();

  auto *BBTrue = BasicBlock::Create(C, "memmove.true", F, Post);
  auto *BBFalse = BasicBlock::Create(C, "memmove.false", F, Post);

  B.SetInsertPoint(Pre);
  B.CreateCondBr(pCmp, BBTrue, BBFalse);

  B.SetInsertPoint(BBTrue);
  B.CreateBr(Post);
  B.SetInsertPoint(BBFalse);
  B.CreateBr(Post);

  auto *CI = dyn_cast<ConstantInt>(LPCount);
  if (CI) {
    uint32_t Count = (uint32_t)CI->getZExtValue();

    // noop
    if (Count == 0) {
      MM->eraseFromParent();
      return;
    }

    Type *VecTys[8];
    uint32_t Len, NewCount;
    generalGroupI8Stream(C, Count, Align, NewCount, VecTys, Len);

    // for true block (Src < Dst), do a reverse copy.
    {
      B.SetInsertPoint(BBTrue->getTerminator());

      // calculate byte offsets so we can walk backwards through them
      SmallVector<uint, 8> byteOffsets{0};

      {
        uint32_t SZ = (unsigned int)(VecTys[0]->getPrimitiveSizeInBits() / 8);
        uint32_t BOfst = NewCount * SZ;

        for (unsigned i = 1; i < Len; i++) {
          byteOffsets.push_back(BOfst);
          uint32_t SZ = (unsigned int)(VecTys[i]->getPrimitiveSizeInBits() / 8);
          BOfst += SZ;
        }
      }

      // emit the smaller than <8 x i32> stores
      for (unsigned i = Len - 1; i >= 1; i--) {
        uint offset = byteOffsets[i];
        uint32_t newAlign = getLargestPowerOfTwo(Align + offset);
        auto *tSrc = B.CreateConstGEP1_32(B.getInt8Ty(), i8Src, offset);
        auto *tDst = B.CreateConstGEP1_32(B.getInt8Ty(), i8Dst, offset);

        auto *vSrc = B.CreateBitCast(SkipBitCast(tSrc), PointerType::get(VecTys[i], SrcAS), "memcpy_rem");
        auto *vDst = B.CreateBitCast(SkipBitCast(tDst), PointerType::get(VecTys[i], DstAS), "memcpy_rem");
        LoadInst *L = B.CreateAlignedLoad(VecTys[i], vSrc, getAlign(newAlign), IsVolatile);
        (void)B.CreateAlignedStore(L, vDst, getAlign(newAlign), IsVolatile);
      }

      // now emit the <8 x i32> stores
      auto *vSrc = B.CreateBitCast(SkipBitCast(Src), PointerType::get(VecTys[0], SrcAS), "memcpy_vsrc");
      auto *vDst = B.CreateBitCast(SkipBitCast(Dst), PointerType::get(VecTys[0], DstAS), "memcpy_vdst");
      // If NewCount is less than the threshold, don't generate loop.
      uint32_t SZ = (unsigned int)(VecTys[0]->getPrimitiveSizeInBits() / 8);
      uint32_t newAlign = getLargestPowerOfTwo(Align + SZ);
      if (NewCount < IGC_GET_FLAG_VALUE(MemCpyLoweringUnrollThreshold)) {
        for (unsigned i = 0; i < NewCount; i++) {
          unsigned idx = NewCount - 1 - i;
          auto *tSrc = B.CreateConstGEP1_32(VecTys[0], vSrc, idx);
          auto *tDst = B.CreateConstGEP1_32(VecTys[0], vDst, idx);
          LoadInst *L = B.CreateAlignedLoad(VecTys[0], tSrc, getAlign(newAlign), IsVolatile);
          (void)B.CreateAlignedStore(L, tDst, getAlign(newAlign), IsVolatile);
        }
      } else {
        auto *NewLPCount = ConstantInt::get(LPCount->getType(), NewCount);
        Instruction *IV = insertReverseLoop(BBTrue, Post, NewLPCount, "memmmove");
        {
          IGCLLVM::IRBuilder<> B(&(*++BasicBlock::iterator(IV)));
          Value *tSrc = B.CreateGEP(VecTys[0], vSrc, IV);
          Value *tDst = B.CreateGEP(VecTys[0], vDst, IV);
          LoadInst *L = B.CreateAlignedLoad(VecTys[0], tSrc, getAlign(newAlign), IsVolatile);
          (void)B.CreateAlignedStore(L, tDst, getAlign(newAlign), IsVolatile);
        }
      }
    }

    // for false block (Src >= Dst), just a plain memcpy.
    {
      auto *MemCpy = MemMoveToMemCpy(MM);
      MemCpy->insertBefore(BBFalse->getTerminator());
      replaceMemcpy(MemCpy);
    }
  } else {
    // (Src < Dst)
    {
      B.SetInsertPoint(BBTrue->getTerminator());
      // Fall back to i8 copy
      Instruction *IV = insertReverseLoop(BBTrue, Post, LPCount, "memmove");
      {
        IGCLLVM::IRBuilder<> B(&(*++BasicBlock::iterator(IV)));
        Value *tSrc = B.CreateGEP(B.getInt8Ty(), i8Src, IV);
        Value *tDst = B.CreateGEP(B.getInt8Ty(), i8Dst, IV);
        LoadInst *L = B.CreateAlignedLoad(B.getInt8Ty(), tSrc, getAlign(1), IsVolatile);
        (void)B.CreateAlignedStore(L, tDst, getAlign(1), IsVolatile);
      }
    }

    // for false block (Src >= Dst), just a plain memcpy.
    {
      auto *MemCpy = MemMoveToMemCpy(MM);
      MemCpy->insertBefore(BBFalse->getTerminator());
      replaceMemcpy(MemCpy);
    }
  }

  MM->eraseFromParent();
}

void ReplaceUnsupportedIntrinsics::replaceMemset(IntrinsicInst *I) {
  // Same idea as replaceMemcpy (see comment of replaceMemcpy).
  MemSetInst *MS = cast<MemSetInst>(I);
  Value *Dst = MS->getRawDest();
  Value *Src = MS->getValue();
  Value *LPCount = MS->getLength();
  uint32_t Align = MS->getDestAlign().valueOrOne().value();
  const bool IsVolatile = MS->isVolatile();
  const uint32_t AS = MS->getDestAddressSpace();

  LLVMContext &C = MS->getContext();
  Type *TyPtrI8 = Type::getInt8PtrTy(C, AS);

  IGCLLVM::IRBuilder<> Builder(MS);

  // BaseSize == 32 if we want to handle algorithm in general way
  // or different value if want to keep size of base type to further optimizations
  uint32_t BaseSize = 0;
  PointerType *ptrTy = cast<PointerType>(Dst->stripPointerCasts()->getType());

  // For typed pointers we can get a type from more complex type
  // like e.g. struct by using GetBaseType(), but for opaque pointers
  // we also need to be able to deduce this type, so we can get this
  // from investigating instructions and then using GetBaseType().
  Type *RawDstType = Builder.getInt8Ty();
  if (IGCLLVM::isPointerTy(ptrTy)) {
    if (auto *alloca = dyn_cast<AllocaInst>(Dst))
      RawDstType = alloca->getAllocatedType();
    else if (auto *gep = dyn_cast<GetElementPtrInst>(Dst))
      RawDstType = gep->getResultElementType();
    else if (auto *gv = dyn_cast<GlobalVariable>(Dst))
      RawDstType = gv->getValueType();
  } else {
    RawDstType = IGCLLVM::getNonOpaquePtrEltTy(ptrTy);
  }

  if (Type *BaseType = GetBaseType(RawDstType))
    BaseSize = BaseType->getScalarSizeInBits();

  if (BaseSize != 16)
    // size 32 is equal to size of i32, so general algorithm will be applied
    BaseSize = 32;

  ConstantInt *CI = dyn_cast<ConstantInt>(LPCount);
  if (CI) {
    uint32_t Count = (uint32_t)CI->getZExtValue();

    Type *VecTys[8];
    uint32_t Len, NewCount;
    generalGroupI8Stream(C, Count, Align, NewCount, VecTys, Len, BaseSize);

    Value *NewDst, *vDst, *vSrc;
    uint32_t BOfst = 0; // Byte offset

    // First, insert main loop before MC.
    if (NewCount > 0) {
      PointerType *PTy = PointerType::get(VecTys[0], AS);
      vSrc = replicateScalar(Src, VecTys[0], MS);
      vDst = Builder.CreateBitCast(SkipBitCast(Dst), PTy, "memset_vdst");

      // getPrimitiveSizeInBits() should be enough, no need to
      // use DataLayout to get target-dependent size.
      uint32_t SZ = (unsigned int)(VecTys[0]->getPrimitiveSizeInBits() / 8);

      // To set alignment correctly
      uint32_t adjust_align = getLargestPowerOfTwo(SZ);
      Align = adjust_align < Align ? adjust_align : Align;

      // If NewCount is less than the threshold, don't generate loop.
      if (NewCount < IGC_GET_FLAG_VALUE(MemCpyLoweringUnrollThreshold)) {
        for (unsigned i = 0; i < NewCount; ++i) {
          Value *tDst = Builder.CreateConstGEP1_32(VecTys[0], vDst, i);
          (void)Builder.CreateAlignedStore(vSrc, tDst, getAlign(Align), IsVolatile);
        }
      } else {
        Value *NewLPCount = ConstantInt::get(LPCount->getType(), NewCount);
        Instruction *IV = insertLoop(MS, NewLPCount, "memset");
        {
          IGCLLVM::IRBuilder<> B(&(*++BasicBlock::iterator(IV)));
          Value *tDst = B.CreateGEP(VecTys[0], vDst, IV);
          (void)B.CreateAlignedStore(vSrc, tDst, getAlign(Align), IsVolatile);
        }
      }

      // Set offset for the remaining elements
      BOfst = NewCount * SZ;
    }

    // Second, generate epilog code before MS.
    // Note that as MC has been moved to a different BB by
    //   inserting the main loop! Reset it to MS.
    Builder.SetInsertPoint(MS);
    if (Len > 1) {
      Dst = Builder.CreateBitCast(SkipBitCast(Dst), TyPtrI8, "memset_dst");
    }
    for (unsigned i = 1; i < Len; ++i) {
      uint32_t SZ = (unsigned int)VecTys[i]->getPrimitiveSizeInBits() / 8;
      uint32_t adjust_align = getLargestPowerOfTwo(SZ);
      Align = adjust_align < Align ? adjust_align : Align;
      PointerType *PTy = PointerType::get(VecTys[i], AS);
      NewDst = BOfst > 0 ? Builder.CreateConstGEP1_32(Builder.getInt8Ty(), Dst, BOfst) : Dst;
      vSrc = replicateScalar(Src, VecTys[i], MS);
      vDst = Builder.CreateBitCast(SkipBitCast(NewDst), PTy, "memset_rem");
      (void)Builder.CreateAlignedStore(vSrc, vDst, getAlign(Align), IsVolatile);
      BOfst += SZ;
    }
  } else {
    Dst = Builder.CreateBitCast(SkipBitCast(Dst), TyPtrI8, "memset_dst");
    // Fall back to i8 copy
    Instruction *IV = insertLoop(MS, LPCount, "memset");
    {
      IGCLLVM::IRBuilder<> B(&(*++BasicBlock::iterator(IV)));
      Value *tDst = B.CreateGEP(Builder.getInt8Ty(), Dst, IV);
      (void)B.CreateAlignedStore(Src, tDst, getAlign(Align), IsVolatile);
    }
  }
  MS->eraseFromParent();
}

void ReplaceUnsupportedIntrinsics::replaceExpect(IntrinsicInst *MS) {
  MS->replaceAllUsesWith(MS->getOperand(0));
  MS->eraseFromParent();
}

/*
  Replaces llvm.fshl.* and llvm.fshr.* funnel shift intrinsics.
  E.g. for fshl we would produce a following sequence:
  %r = call i8 @llvm.fshl.i8(i8 %a, i8 %b, i8 %c) =>
  %modRes = and i8 %c, 7         // (urem i8 %c, 8 ) get the modulo of shift value
  %subRes = sub i8 8, %modRes    // subtract from the type's number of bits
  %shlRes = shl i8 %a, %modRes   // shift the bits according to instruction spec
  %shrRes = lshr i8 %b, %subRes
  %r = or i8 %shlRes, %shrRes    // compose the final result
*/
void ReplaceUnsupportedIntrinsics::replaceFunnelShift(IntrinsicInst *I) {
  IGC_ASSERT(I->getIntrinsicID() == Intrinsic::fshl || I->getIntrinsicID() == Intrinsic::fshr);
  IGCLLVM::IRBuilder<> Builder(I);
  unsigned sizeInBits = I->getArgOperand(0)->getType()->getScalarSizeInBits();

  // Don't replace rotate
  if (I->getArgOperand(0) == I->getArgOperand(1) && !I->getType()->isVectorTy() &&
      m_Ctx->platform.supportRotateInstruction()) {
    if (m_Ctx->platform.supportQWRotateInstructions() && sizeInBits == 64) {
      return;
    }
    if (sizeInBits == 16 || sizeInBits == 32) {
      return;
    }
  }

  IGC_ASSERT(isPowerOf2_32(sizeInBits));
  Value *numBits = Builder.getIntN(sizeInBits, sizeInBits);
  Value *mask = Builder.getIntN(sizeInBits, sizeInBits - 1);
  if (auto IVT = dyn_cast<IGCLLVM::FixedVectorType>(I->getType())) {
    numBits =
        ConstantVector::getSplat(IGCLLVM::getElementCount((uint32_t)IVT->getNumElements()), cast<Constant>(numBits));
    mask = ConstantVector::getSplat(IGCLLVM::getElementCount((uint32_t)IVT->getNumElements()), cast<Constant>(mask));
  }
  auto shiftModulo = Builder.CreateAnd(I->getArgOperand(2), mask);
  Value *isZeroShift = Builder.CreateICmpEQ(shiftModulo, Constant::getNullValue(shiftModulo->getType()));
  auto negativeShift = Builder.CreateSub(numBits, shiftModulo);
  if (I->getIntrinsicID() == Intrinsic::fshr) {
    std::swap(shiftModulo, negativeShift);
  }
  auto upperShifted = Builder.CreateShl(I->getArgOperand(0), shiftModulo);
  auto lowerShifted = Builder.CreateLShr(I->getArgOperand(1), negativeShift);
  Value *shiftedResult = Builder.CreateOr(upperShifted, lowerShifted);

  bool isFshl = (I->getIntrinsicID() == Intrinsic::fshl);
  Value *zeroShiftResult = isFshl ? I->getArgOperand(0) : I->getArgOperand(1);
  Value *result = Builder.CreateSelect(isZeroShift, zeroShiftResult, shiftedResult);

  I->replaceAllUsesWith(result);
  I->eraseFromParent();
}

/*
  Replaces llvm.lround.* and llvm.llround.* intrinsics.
  The llvm.lround.* intrinsics return the operand rounded to
  the nearest integer with ties away from zero.
  @llvm.lround.i32.f32(float f) => (i32)(f + (f >= 0.0 ? 0.5 : -0.5))
  E.g. for lround we would produce a following sequence:
  %r = call i32 @llvm.lround.i32.f32(float f)
  =>
  %cmp = fcmp oge float %f, 0.000000e+00
  %cond = select i1 %cmp, double 5.000000e-01, double -5.000000e-01
  %df = fpext float %f to double
  %add = fadd double %df, %cond
  %conv = fptosi double %add to i32
*/
void ReplaceUnsupportedIntrinsics::replaceLRound(IntrinsicInst *I) {
  IGC_ASSERT(I->getIntrinsicID() == Intrinsic::lround || I->getIntrinsicID() == Intrinsic::llround);
  Value *inVal = I->getArgOperand(0);
  Type *dstType = I->getType();
  [[maybe_unused]] Type *srcType = inVal->getType();
  IGC_ASSERT(!(srcType->isVectorTy() || dstType->isVectorTy()));
  IGC_ASSERT(srcType->isFloatTy() || srcType->isDoubleTy());
  IGC_ASSERT(dstType->isIntegerTy());
  IGCLLVM::IRBuilder<> Builder(I);
  Value *rounded = Builder.CreateUnaryIntrinsic(Intrinsic::round, inVal);
  Value *conv = Builder.CreateFPToSI(rounded, dstType);
  I->replaceAllUsesWith(conv);
  I->eraseFromParent();
}

/*
  Replaces llvm.lrint.* and llvm.llrint.* intrinsics.
  Both simply return the operand rounded to the nearest integer.
  @llvm.lrint.i32.f32(float f) => (i32)(f)
  Example:
  %r = call i32 @llvm.lrint.i32.f32(float %f)
  =>
  %f.lrint = fptosi float %f.fpext to i32
*/
void ReplaceUnsupportedIntrinsics::replaceLRint(IntrinsicInst *I) {
  auto IntrID = I->getIntrinsicID();
  IGC_ASSERT(IntrID == Intrinsic::lrint || IntrID == Intrinsic::llrint);
  Value *inVal = I->getArgOperand(0);
  Type *dstType = I->getType();
  [[maybe_unused]] Type *srcType = inVal->getType();
  IGC_ASSERT(!(srcType->isVectorTy() || dstType->isVectorTy()));
  IGC_ASSERT(srcType->isFloatTy() || srcType->isDoubleTy());
  IGC_ASSERT(dstType->isIntegerTy());
  IGCLLVM::IRBuilder<> Builder(I);
  StringRef inValName = inVal->getName();
  const char *suffix = IntrID == Intrinsic::lrint ? ".lrint" : ".llrint";
  Value *FPToInt = Builder.CreateFPToSI(inVal, dstType, inValName + suffix);
  I->replaceAllUsesWith(FPToInt);
  I->eraseFromParent();
}

void ReplaceUnsupportedIntrinsics::replaceMinMax(IntrinsicInst *I) {
  if (I->getType()->isIntegerTy(64))
    replaceI64MinMax(I);

  if (I->getType()->isIntegerTy(1))
    replaceI1MinMax(I);
}

#if LLVM_VERSION_MAJOR >= 15
// Expand to individual tests combined with ORs.
// Returns the result but without replacing the input instruction.
// Based on implementations in llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp and LLVM-SPIRV-Translator
Value *lowerIsFpClass(IntrinsicInst *Intrinsic) {
  IGC_ASSERT(Intrinsic->getIntrinsicID() == Intrinsic::is_fpclass);

  Value *InputValue = Intrinsic->getArgOperand(0);
  Type *InputType = InputValue->getType();
  Type *ResultType = Intrinsic->getType();

  FPClassTest ClassTest = static_cast<FPClassTest>(cast<ConstantInt>(Intrinsic->getArgOperand(1))->getZExtValue());

  // Handle degenerate cases
  if (ClassTest == 0)
    return llvm::Constant::getNullValue(ResultType);
  if (ClassTest == fcAllFlags)
    return llvm::Constant::getAllOnesValue(ResultType);

  const uint32_t BitWidth = InputType->getScalarSizeInBits();
  Type *IntType = IntegerType::getIntNTy(Intrinsic->getContext(), BitWidth);
  if (InputType->isVectorTy())
    IntType = FixedVectorType::get(IntType, cast<FixedVectorType>(InputType)->getNumElements());

  const llvm::fltSemantics &FloatSemantics = InputType->getScalarType()->getFltSemantics();
  const APInt InfinityBits = APFloat::getInf(FloatSemantics).bitcastToAPInt();
  const APInt MantissaMask = APFloat::getLargest(FloatSemantics).bitcastToAPInt() & ~InfinityBits;
  const APInt SignBitMask = APInt::getSignMask(BitWidth);
  const APInt AbsoluteMask = APInt::getSignedMaxValue(BitWidth);

  Value *InfinityInt = llvm::Constant::getIntegerValue(IntType, InfinityBits);
  Value *InfinityFloat = llvm::ConstantFP::get(InputType, APFloat::getInf(FloatSemantics));
  Value *ZeroInt = ConstantInt::getNullValue(IntType);

  Value *InputAsInt = nullptr;
  Value *AbsoluteFloat = nullptr;
  Value *AbsoluteInt = nullptr;
  Value *IsNegative = nullptr;
  Value *IsPositive = nullptr;

  IGCLLVM::IRBuilder<> Builder(Intrinsic);

  auto GetInputAsInt = [&]() -> Value * {
    if (!InputAsInt)
      InputAsInt = Builder.CreateBitCast(InputValue, IntType);
    return InputAsInt;
  };

  auto GetAbsoluteInt = [&]() -> Value * {
    if (!AbsoluteInt) {
      AbsoluteInt = Builder.CreateAnd(GetInputAsInt(), AbsoluteMask);
    }
    return AbsoluteInt;
  };

  auto GetAbsoluteValue = [&]() -> Value * {
    if (!AbsoluteFloat) {
      AbsoluteFloat = Builder.CreateBitCast(GetAbsoluteInt(), InputType);
    }
    return AbsoluteFloat;
  };

  auto ApplySignTest = [&](Value *Test, bool TestNegative) -> Value * {
    if (!IsNegative) {
      Value *SignBits = Builder.CreateAnd(GetInputAsInt(), SignBitMask);
      IsNegative = Builder.CreateICmpNE(SignBits, ZeroInt);
    }
    if (TestNegative)
      return Builder.CreateAnd(IsNegative, Test);

    if (!IsPositive)
      IsPositive = Builder.CreateNot(IsNegative);
    return Builder.CreateAnd(IsPositive, Test);
  };

  Value *Result = nullptr;
  auto AccumulateResult = [&](Value *Test) { Result = Result ? Builder.CreateOr(Result, Test) : Test; };

  auto HandleSignedTest = [&](Value *Test, FPClassTest NegFlag, FPClassTest PosFlag) {
    bool TestNegative = ClassTest & NegFlag;
    bool TestPositive = ClassTest & PosFlag;

    if (TestNegative && TestPositive) {
      AccumulateResult(Test);
    } else {
      AccumulateResult(ApplySignTest(Test, TestNegative));
    }
  };

  if (ClassTest & fcNan) {
    // isnan(V) ==> abs(V) > int(inf)
    // isquiet(V) ==> abs(V) >= (unsigned(Inf) | quiet_bit)
    bool TestQNaN = ClassTest & fcQNan;
    bool TestSNaN = ClassTest & fcSNan;

    Value *IsNaN = Builder.CreateICmpUGT(GetAbsoluteInt(), InfinityInt);

    if (TestQNaN && TestSNaN) {
      AccumulateResult(IsNaN);
    } else {
      APInt QNaNBit = APInt::getOneBitSet(BitWidth, MantissaMask.getActiveBits() - 1);
      Value *QNaNThreshold = llvm::Constant::getIntegerValue(IntType, InfinityBits | QNaNBit);
      Value *IsQNaN = Builder.CreateICmpUGE(GetAbsoluteInt(), QNaNThreshold);

      if (TestQNaN) {
        AccumulateResult(IsQNaN);
      } else {
        // isSNaN(V) => isnan(V) && !isQNaN(V)
        Value *IsSNaN = Builder.CreateAnd(IsNaN, Builder.CreateNot(IsQNaN));
        AccumulateResult(IsSNaN);
      }
    }
  }

  if (ClassTest & fcInf) {
    Value *IsInfinity = Builder.CreateFCmpOEQ(GetAbsoluteValue(), InfinityFloat);
    HandleSignedTest(IsInfinity, fcNegInf, fcPosInf);
  }

  if (ClassTest & fcNormal) {
    // isnormal(V) ==> (0 < exp < max_exp) ==> (unsigned(exp-1) < (max_exp-1))
    APInt ExponentLSB = InfinityBits & ~(InfinityBits.shl(1));
    APInt ExponentLimit = InfinityBits - ExponentLSB;
    Value *ExponentMinus1 = Builder.CreateSub(GetAbsoluteInt(), ConstantInt::get(IntType, ExponentLSB));
    Value *IsNormal = Builder.CreateICmpULT(ExponentMinus1, ConstantInt::get(IntType, ExponentLimit));
    HandleSignedTest(IsNormal, fcNegNormal, fcPosNormal);
  }

  if (ClassTest & fcSubnormal) {
    // issubnormal(V) ==> unsigned(abs(V) - 1) < (all mantissa bits set)
    Value *AbsMinus1 = Builder.CreateSub(GetAbsoluteInt(), ConstantInt::get(IntType, APInt(BitWidth, 1)));
    Value *IsSubnormal = Builder.CreateICmpULT(AbsMinus1, Constant::getIntegerValue(IntType, MantissaMask));
    HandleSignedTest(IsSubnormal, fcNegSubnormal, fcPosSubnormal);
  }

  if (ClassTest & fcZero) {
    Value *IsZero = Builder.CreateICmpEQ(GetAbsoluteInt(), ZeroInt);
    HandleSignedTest(IsZero, fcNegZero, fcPosZero);
  }

  return Result;
}

void ReplaceUnsupportedIntrinsics::replaceIsFpClass(IntrinsicInst *I) {
  Value *Res = lowerIsFpClass(I);
  I->replaceAllUsesWith(Res);
  I->eraseFromParent();
}
#endif

/*
    Replaces i64 calls to llvm.smax, llvm.smin, llvm.umax, llvm.umin to
    icmp + select instructionc that can be emulated.
*/
void ReplaceUnsupportedIntrinsics::replaceI64MinMax(IntrinsicInst *I) {
  const SmallDenseMap<Intrinsic::ID, CmpInst::Predicate, 4> CmpPredMap{{Intrinsic::smax, CmpInst::Predicate::ICMP_SGT},
                                                                       {Intrinsic::smin, CmpInst::Predicate::ICMP_SLT},
                                                                       {Intrinsic::umax, CmpInst::Predicate::ICMP_UGT},
                                                                       {Intrinsic::umin, CmpInst::Predicate::ICMP_ULT}};

  IGCLLVM::IRBuilder<> Builder(I);

  Value *LHS = I->getArgOperand(0), *RHS = I->getArgOperand(1);
  auto Cmp = cast<Instruction>(Builder.CreateICmp(CmpPredMap.lookup(I->getIntrinsicID()), LHS, RHS));
  I->replaceAllUsesWith(Builder.CreateSelect(Cmp, LHS, RHS));
}

void ReplaceUnsupportedIntrinsics::replaceI1MinMax(IntrinsicInst *I) {
  IGCLLVM::IRBuilder<> Builder(I);

  Value *LHS = I->getArgOperand(0), *RHS = I->getArgOperand(1);

  auto IntrId = I->getIntrinsicID();

  if (IntrId == Intrinsic::smax || IntrId == Intrinsic::umax)
    I->replaceAllUsesWith(Builder.CreateOr(LHS, RHS));
  else // Intrinsic::smin || Intrinsic::umin
    I->replaceAllUsesWith(Builder.CreateAnd(LHS, RHS));
}

/*
  Replaces half-precision llvm.experimental.constrained.fdiv and
  llvm.experimental.constrained.sqrt intrinsics by promoting operands to float,
  performing the operation in float with RNE rounding via GenISA intrinsics,
  and converting the result back to half with the original rounding mode
  using GenISA_ftof_rt{e,z,p,n}.

  E.g. for fdiv:
  %r = call half @llvm.experimental.constrained.fdiv.f16(half %a, half %b,
           metadata !"round.upward", metadata !"fpexcept.strict")
  =>
  %a.ext = fpext half %a to float
  %b.ext = fpext half %b to float
  %div = call float @llvm.genx.GenISA.IEEE.Divide.rm(float %a.ext, float %b.ext, i32 0)
  %r = call half @llvm.genx.GenISA.ftof.rtp.f16.f32(float %div)

  E.g. for sqrt:
  %r = call half @llvm.experimental.constrained.sqrt.f16(half %a,
           metadata !"round.upward", metadata !"fpexcept.strict")
  =>
  %a.ext = fpext half %a to float
  %sqrt = call float @llvm.genx.GenISA.IEEE.Sqrt.rm(float %a.ext, i32 0)
  %r = call half @llvm.genx.GenISA.ftof.rtp.f16.f32(float %sqrt)
*/
void ReplaceUnsupportedIntrinsics::replaceHalvesDivsSqrts(IntrinsicInst *I) {
  auto *CFP = cast<ConstrainedFPIntrinsic>(I);
  Type *OrigTy = I->getType();

  // Only replace half-precision operations.
  if (!OrigTy->getScalarType()->isHalfTy())
    return;

  Intrinsic::ID IID = I->getIntrinsicID();
  IGC_ASSERT(IID == Intrinsic::experimental_constrained_fdiv || IID == Intrinsic::experimental_constrained_sqrt);

  Type *FloatTy = Type::getFloatTy(I->getContext());
  Module *M = I->getModule();

  // Map LLVM constrained rounding mode to GenISA ftof intrinsic.
  auto OrigRounding = CFP->getRoundingMode();
  GenISAIntrinsic::ID FtofID = GenISAIntrinsic::GenISA_ftof_rte; // default RNE
  if (OrigRounding) {
    switch (*OrigRounding) {
    case RoundingMode::NearestTiesToEven:
      FtofID = GenISAIntrinsic::GenISA_ftof_rte;
      break;
    case RoundingMode::TowardZero:
      FtofID = GenISAIntrinsic::GenISA_ftof_rtz;
      break;
    case RoundingMode::TowardPositive:
      FtofID = GenISAIntrinsic::GenISA_ftof_rtp;
      break;
    case RoundingMode::TowardNegative:
      FtofID = GenISAIntrinsic::GenISA_ftof_rtn;
      break;
    default:
      FtofID = GenISAIntrinsic::GenISA_ftof_rte;
      break;
    }
  }

  // RNE rounding mode value for GenISA_IEEE_Divide_rm / GenISA_IEEE_Sqrt_rm.
  Value *RNE =
      ConstantInt::get(Type::getInt32Ty(I->getContext()), static_cast<uint32_t>(ERoundingMode::ROUND_TO_NEAREST_EVEN));

  IGCLLVM::IRBuilder<> Builder(I);
  Value *ComputeResult = nullptr;

  if (IID == Intrinsic::experimental_constrained_fdiv) {
    Value *LHS = Builder.CreateFPExt(I->getArgOperand(0), FloatTy, "fdiv.lhs.ext");
    Value *RHS = Builder.CreateFPExt(I->getArgOperand(1), FloatTy, "fdiv.rhs.ext");

    Function *DivFn = GenISAIntrinsic::getDeclaration(M, GenISAIntrinsic::GenISA_IEEE_Divide_rm, {FloatTy});
    ComputeResult = Builder.CreateCall(DivFn, {LHS, RHS, RNE}, "div_rm");
  } else {
    Value *Op = Builder.CreateFPExt(I->getArgOperand(0), FloatTy, "fsqrt.op.ext");

    Function *SqrtFn = GenISAIntrinsic::getDeclaration(M, GenISAIntrinsic::GenISA_IEEE_Sqrt_rm, {FloatTy});
    ComputeResult = Builder.CreateCall(SqrtFn, {Op, RNE}, "sqrt_rm");
  }

  // Truncate float result back to half with the original rounding mode.
  Type *OverloadTypes[] = {OrigTy, FloatTy};
  Function *FtofFn = GenISAIntrinsic::getDeclaration(M, FtofID, OverloadTypes);
  Value *Result = Builder.CreateCall(FtofFn, {ComputeResult}, "ftof_rm");
  cast<Instruction>(Result)->setDebugLoc(I->getDebugLoc());

  I->replaceAllUsesWith(Result);
  I->eraseFromParent();
}

/*
  Replaces llvm.ctlz.* intrinsics (count the leading zeros)
  to llvm.ctlz.i32 because we support llvm.ctlz intrinsic
  only with source type i32.

  E.g.
  %1 = call <2 x i8> @llvm.ctlz.v2i8(<2 x i8> %0, i1 false)
  ret <2 x i8> %1
  =>
  %1 = extractelement <2 x i8> %0, i64 0
  %2 = zext i8 %1 to i32
  %3 = call i32 @llvm.ctlz.i32(i32 %2, i1 false)
  %4 = trunc i32 %3 to i8
  %5 = add nsw i8 %4, -24
  %6 = insertelement <2 x i8> undef, i8 %5, i32 0
  %7 = extractelement <2 x i8> %0, i64 1
  %8 = zext i8 %7 to i32
  %9 = call i32 @llvm.ctlz.i32(i32 %8, i1 false)
  %10 = trunc i32 %9 to i8
  %11 = add nsw i8 %10, -24
  %12 = insertelement <2 x i8> %6, i8 %11, i32 1
  %13 = call <2 x i8> @llvm.ctlz.v2i8(<2 x i8> %0, i1 false)
  ret <2 x i8> %12
*/
void ReplaceUnsupportedIntrinsics::replaceCountTheLeadingZeros(IntrinsicInst *I) {
  IGC_ASSERT(I->getIntrinsicID() == Intrinsic::ctlz);

  Type *oldIntrinsicDstType = I->getType();
  Type *singleElementType = oldIntrinsicDstType;
  uint32_t numOfElements = 1;
  bool isVector = oldIntrinsicDstType->isVectorTy();

  if (isVector) {
    auto oldIntrinsicDstTypeFVT = cast<IGCLLVM::FixedVectorType>(oldIntrinsicDstType);
    numOfElements = (uint32_t)oldIntrinsicDstTypeFVT->getNumElements();
    singleElementType = oldIntrinsicDstTypeFVT->getElementType();
  }

  int singleElementSizeInBits = singleElementType->getScalarSizeInBits();

  IGC_ASSERT_MESSAGE(singleElementSizeInBits == 8 || singleElementSizeInBits == 16 || singleElementSizeInBits == 32 ||
                         singleElementSizeInBits == 64,
                     "Currently for Intrinsic::ctlz we support source bit size: 8,16,32,64");

  // noting to replace, early return
  if (!isVector && singleElementSizeInBits == 32)
    return;

  bool bitSizeLowerThan32 = singleElementSizeInBits < 32;
  bool bitSizeEqual64 = singleElementSizeInBits == 64;

  IGCLLVM::IRBuilder<> Builder(I);

  Value *inputVal = I->getArgOperand(0);
  Value *canBePoison = I->getArgOperand(1);
  Value *outputVal = llvm::UndefValue::get(oldIntrinsicDstType); // Will be overwritten in scalar case.
  Value *retVal = inputVal;

  for (uint32_t i = 0; i < numOfElements; i++) {
    if (isVector)
      retVal = Builder.CreateExtractElement(inputVal, i);

    if (bitSizeLowerThan32)
      retVal = evaluateCtlzUpto32bit(&Builder, retVal, singleElementType, canBePoison);
    else if (bitSizeEqual64)
      retVal = evaluateCtlz64bit(&Builder, retVal, singleElementType, canBePoison);

    if (singleElementSizeInBits == 32)
      retVal = Builder.CreateIntrinsic(Intrinsic::ctlz, {Builder.getInt32Ty()}, {retVal, canBePoison});

    if (isVector)
      outputVal = Builder.CreateInsertElement(outputVal, retVal, Builder.getInt32(i));
    else // for scalar type
      outputVal = retVal;
  }
  I->replaceAllUsesWith(outputVal);
}

Value *ReplaceUnsupportedIntrinsics::evaluateCtlzUpto32bit(IGCLLVM::IRBuilder<> *Builder, Value *inVal,
                                                           Type *singleElementType, Value *canBePoison) {
  int sizeInBits = singleElementType->getScalarSizeInBits();
  Value *retVal = Builder->CreateZExt(inVal, Builder->getInt32Ty());
  retVal = Builder->CreateIntrinsic(Intrinsic::ctlz, {Builder->getInt32Ty()}, {retVal, canBePoison});
  retVal = Builder->CreateTrunc(retVal, singleElementType);
  auto constInt = Builder->getIntN(sizeInBits, sizeInBits - 32);
  retVal = Builder->CreateNSWAdd(retVal, constInt);
  return retVal;
}

Value *ReplaceUnsupportedIntrinsics::evaluateCtlz64bit(IGCLLVM::IRBuilder<> *Builder, Value *inVal,
                                                       Type *singleElementType, Value *canBePoison) {
  Value *lowBits = Builder->CreateTrunc(inVal, Builder->getInt32Ty());
  lowBits = Builder->CreateIntrinsic(Intrinsic::ctlz, {Builder->getInt32Ty()}, {lowBits, canBePoison});

  Value *hiBits = Builder->CreateLShr(inVal, 32);
  hiBits = Builder->CreateTrunc(hiBits, Builder->getInt32Ty());
  hiBits = Builder->CreateIntrinsic(Intrinsic::ctlz, {Builder->getInt32Ty()}, {hiBits, canBePoison});

  auto maxValueIn32BitsPlusOne = Builder->getInt64((uint64_t)(0xffffffff) + 1); // maxValueIn32Bits + 1
  Value *cmp = Builder->CreateICmp(CmpInst::Predicate::ICMP_ULT, inVal, maxValueIn32BitsPlusOne);

  auto constInt = Builder->getInt32(32);
  lowBits = Builder->CreateAdd(lowBits, constInt);

  Value *retVal = Builder->CreateSelect(cmp, lowBits, hiBits);
  retVal = Builder->CreateZExt(retVal, singleElementType);
  return retVal;
}

// Gen HW has no native count-trailing-zeros. Lower llvm.cttz to the vISA
// primitive ISA_FBL (exposed as GenISA_firstbitLo), which returns the
// position of the lowest set bit for non-zero inputs and 0xFFFFFFFF for
// zero. The LLVM LangRef defines llvm.cttz as:
//   declare iN @llvm.cttz.iN(iN <src>, i1 <is_zero_poison>)
// When is_zero_poison == false, llvm.cttz(0) must return the source
// bit-width; firstbitLo's 0xFFFFFFFF result is wrong for that case, so a
// select against the zero input is emitted to substitute the bit-width.
// When is_zero_poison == true, firstbitLo's result on zero is poison
// anyway and the select is omitted. All supported widths (i8/i16/i32/i64)
// are routed through firstbitLo on i32, with i64 split into hi/lo halves.
void ReplaceUnsupportedIntrinsics::replaceCountTheTrailingZeros(IntrinsicInst *I) {
  IGC_ASSERT(I->getIntrinsicID() == Intrinsic::cttz);

  Type *DstType = I->getType();
  Type *ElementType = DstType;
  uint32_t NumElements = 1;
  bool IsVector = DstType->isVectorTy();

  if (IsVector) {
    auto *FVT = cast<IGCLLVM::FixedVectorType>(DstType);
    NumElements = (uint32_t)FVT->getNumElements();
    ElementType = FVT->getElementType();
  }

  int Bits = ElementType->getScalarSizeInBits();
  IGC_ASSERT_MESSAGE(Bits == 8 || Bits == 16 || Bits == 32 || Bits == 64,
                     "Currently for Intrinsic::cttz we support source bit size: 8,16,32,64");

  IGCLLVM::IRBuilder<> Builder(I);
  Module *M = I->getModule();
  Function *FirstbitLo = GenISAIntrinsic::getDeclaration(M, GenISAIntrinsic::GenISA_firstbitLo);

  Value *InputVal = I->getArgOperand(0);
  bool IsZeroPoison = cast<ConstantInt>(I->getArgOperand(1))->isOne();

  Value *OutputVal = llvm::PoisonValue::get(DstType);

  for (uint32_t Idx = 0; Idx < NumElements; ++Idx) {
    Value *X = IsVector ? Builder.CreateExtractElement(InputVal, Idx) : InputVal;
    Value *Result = nullptr;

    if (Bits == 64) {
      // Split i64 into lo/hi i32 halves and combine the per-half results.
      Value *Lo = Builder.CreateTrunc(X, Builder.getInt32Ty());
      Value *Hi = Builder.CreateTrunc(Builder.CreateLShr(X, 32), Builder.getInt32Ty());
      Value *FblLo = Builder.CreateCall(FirstbitLo, {Lo});
      Value *FblHi = Builder.CreateCall(FirstbitLo, {Hi});
      Value *LoIsZero = Builder.CreateICmpEQ(Lo, Builder.getInt32(0));
      Value *FblHiPlus32 = Builder.CreateAdd(FblHi, Builder.getInt32(32));
      Value *Combined = Builder.CreateSelect(LoIsZero, FblHiPlus32, FblLo);
      if (!IsZeroPoison) {
        Value *XIsZero = Builder.CreateICmpEQ(X, Builder.getInt64(0));
        Combined = Builder.CreateSelect(XIsZero, Builder.getInt32(64), Combined);
      }
      Result = Builder.CreateZExt(Combined, Builder.getInt64Ty());
    } else {
      Value *X32 = (Bits == 32) ? X : Builder.CreateZExt(X, Builder.getInt32Ty());
      Value *Fbl = Builder.CreateCall(FirstbitLo, {X32});
      if (!IsZeroPoison) {
        Value *XIsZero = Builder.CreateICmpEQ(X, ConstantInt::get(ElementType, 0));
        Fbl = Builder.CreateSelect(XIsZero, Builder.getInt32(Bits), Fbl);
      }
      Result = (Bits == 32) ? Fbl : Builder.CreateTrunc(Fbl, ElementType);
    }

    if (IsVector)
      OutputVal = Builder.CreateInsertElement(OutputVal, Result, Builder.getInt32(Idx));
    else
      OutputVal = Result;
  }

  I->replaceAllUsesWith(OutputVal);
  I->eraseFromParent();
}

void ReplaceUnsupportedIntrinsics::visitIntrinsicInst(IntrinsicInst &I) {
  if (m_intrinsicToFunc.find(I.getIntrinsicID()) != m_intrinsicToFunc.end()) {
    m_instsToReplace.push_back(&I);
  }
}

bool ReplaceUnsupportedIntrinsics::runOnFunction(Function &F) {
  m_Ctx = getAnalysis<CodeGenContextWrapper>().getCodeGenContext();
  m_instsToReplace.clear();
  visit(F);
  for (auto I : m_instsToReplace) {
    (this->*m_intrinsicToFunc.at(I->getIntrinsicID()))(I);
  }
  return !m_instsToReplace.empty();
}

FunctionPass *IGC::createReplaceUnsupportedIntrinsicsPass() { return new ReplaceUnsupportedIntrinsics(); }
