diff --git a/integration-test/memory_partition/test_1d_partition/test_1d_partition.c b/integration-test/memory_partition/test_1d_partition/test_1d_partition.c new file mode 100644 index 000000000..5759daa6b --- /dev/null +++ b/integration-test/memory_partition/test_1d_partition/test_1d_partition.c @@ -0,0 +1,31 @@ +#include +#define N 32 +#include "dynamatic/Integration.h" + +// NOTE: Simple test for 1d array halve partition +void test_1d_partition(const int A[N], const int B[N], const int C[N], + int result[N]) { +#pragma DYN array_partition array = intermediate dimension = 1 style = \ + block factor = 2 + int intermediate[N]; + for (int i = 0; i < N; ++i) { + intermediate[i] = A[i] * B[i]; + } + for (int i = 0; i < N; i++) { + result[i] = intermediate[i] * C[i]; + } +} + +int main(void) { + int A[N]; + int B[N]; + int C[N]; + int result[N]; + for (int i = 0; i < N; ++i) { + A[i] = rand() % 100; + B[i] = rand() % 100; + C[i] = rand() % 100; + } + CALL_KERNEL(test_1d_partition, A, B, C, result); + return 0; +} diff --git a/integration-test/memory_partition/test_1d_partition_f4/test_1d_partition_f4.c b/integration-test/memory_partition/test_1d_partition_f4/test_1d_partition_f4.c new file mode 100644 index 000000000..46cf8b925 --- /dev/null +++ b/integration-test/memory_partition/test_1d_partition_f4/test_1d_partition_f4.c @@ -0,0 +1,31 @@ +#include +#define N 32 +#include "dynamatic/Integration.h" + +// NOTE: Test for testing larger factors +void test_1d_partition_f4(const int A[N], const int B[N], const int C[N], + int result[N]) { +#pragma DYN array_partition array = intermediate dimension = 1 style = \ + block factor = 4 + int intermediate[N]; + for (int i = 0; i < N; ++i) { + intermediate[i] = A[i] * B[i]; + } + for (int i = 0; i < N; i++) { + result[i] = intermediate[i] * C[i]; + } +} + +int main(void) { + int A[N]; + int B[N]; + int C[N]; + int result[N]; + for (int i = 0; i < N; ++i) { + A[i] = rand() % 100; + B[i] = rand() % 100; + C[i] = rand() % 100; + } + CALL_KERNEL(test_1d_partition_f4, A, B, C, result); + return 0; +} diff --git a/integration-test/memory_partition/test_1d_partition_fu/test_1d_partition_fu.c b/integration-test/memory_partition/test_1d_partition_fu/test_1d_partition_fu.c new file mode 100644 index 000000000..afcd4aab4 --- /dev/null +++ b/integration-test/memory_partition/test_1d_partition_fu/test_1d_partition_fu.c @@ -0,0 +1,31 @@ +#include +#define N 67 +#include "dynamatic/Integration.h" + +// NOTE: Test for division with remainder after doing N / factor +void test_1d_partition_fu(const int A[N], const int B[N], const int C[N], + int result[N]) { +#pragma DYN array_partition array = intermediate dimension = 1 style = \ + block factor = 8 + int intermediate[N]; + for (int i = 0; i < N; ++i) { + intermediate[i] = A[i] * B[i]; + } + for (int i = 0; i < N; i++) { + result[i] = intermediate[i] * C[i]; + } +} + +int main(void) { + int A[N]; + int B[N]; + int C[N]; + int result[N]; + for (int i = 0; i < N; ++i) { + A[i] = rand() % 100; + B[i] = rand() % 100; + C[i] = rand() % 100; + } + CALL_KERNEL(test_1d_partition_fu, A, B, C, result); + return 0; +} diff --git a/integration-test/memory_partition/test_2d_partition/test_2d_partition.c b/integration-test/memory_partition/test_2d_partition/test_2d_partition.c new file mode 100644 index 000000000..be9ec91de --- /dev/null +++ b/integration-test/memory_partition/test_2d_partition/test_2d_partition.c @@ -0,0 +1,39 @@ +#include +#define ROWS 32 +#define COLS 64 +#include "dynamatic/Integration.h" + +// NOTE: Test for outer partition of 2d array, i.e. A[N][M] -> A1[N/2][M], +// A2[N/2][M] +void test_2d_partition(const int A[ROWS][COLS], const int B[ROWS][COLS], + const int C[ROWS][COLS], int result[ROWS][COLS]) { +#pragma DYN array_partition array = intermediate dimension = 1 style = \ + block factor = 2 + int intermediate[ROWS][COLS]; + for (int i = 0; i < ROWS; ++i) { + for (int j = 0; j < COLS; ++j) { + intermediate[i][j] = A[i][j] * B[i][j]; + } + } + for (int i = 0; i < ROWS; ++i) { + for (int j = 0; j < COLS; ++j) { + result[i][j] = intermediate[i][j] * C[i][j]; + } + } +} + +int main(void) { + int A[ROWS][COLS]; + int B[ROWS][COLS]; + int C[ROWS][COLS]; + int result[ROWS][COLS]; + for (int i = 0; i < ROWS; ++i) { + for (int j = 0; j < COLS; ++j) { + A[i][j] = rand() % 100; + B[i][j] = rand() % 100; + C[i][j] = rand() % 100; + } + } + CALL_KERNEL(test_2d_partition, A, B, C, result); + return 0; +} diff --git a/integration-test/memory_partition/test_2d_partition_inner/test_2d_partition_inner.c b/integration-test/memory_partition/test_2d_partition_inner/test_2d_partition_inner.c new file mode 100644 index 000000000..e139fe107 --- /dev/null +++ b/integration-test/memory_partition/test_2d_partition_inner/test_2d_partition_inner.c @@ -0,0 +1,39 @@ +#include +#define ROWS 32 +#define COLS 64 +#include "dynamatic/Integration.h" + +// NOTE: Test for inner partition of 2d array, i.e. A[N][M] -> A1[N][M/2], +// A2[N][M/2] +void test_2d_partition_inner(const int A[ROWS][COLS], const int B[ROWS][COLS], + const int C[ROWS][COLS], int result[ROWS][COLS]) { +#pragma DYN array_partition array = intermediate dimension = 2 style = \ + block factor = 2 + int intermediate[ROWS][COLS]; + for (int i = 0; i < ROWS; ++i) { + for (int j = 0; j < COLS; ++j) { + intermediate[i][j] = A[i][j] * B[i][j]; + } + } + for (int i = 0; i < ROWS; ++i) { + for (int j = 0; j < COLS; ++j) { + result[i][j] = intermediate[i][j] * C[i][j]; + } + } +} + +int main(void) { + int A[ROWS][COLS]; + int B[ROWS][COLS]; + int C[ROWS][COLS]; + int result[ROWS][COLS]; + for (int i = 0; i < ROWS; ++i) { + for (int j = 0; j < COLS; ++j) { + A[i][j] = rand() % 100; + B[i][j] = rand() % 100; + C[i][j] = rand() % 100; + } + } + CALL_KERNEL(test_2d_partition_inner, A, B, C, result); + return 0; +} diff --git a/integration-test/memory_partition/test_zero_elem/test_zero_elem.c b/integration-test/memory_partition/test_zero_elem/test_zero_elem.c new file mode 100644 index 000000000..438dd0dd1 --- /dev/null +++ b/integration-test/memory_partition/test_zero_elem/test_zero_elem.c @@ -0,0 +1,25 @@ +#include +#define N 32 +#include "dynamatic/Integration.h" + +// NOTE: Tests for two bugs: +// 1. Reuse of GEP by two different indices +// 2. Zero indexing not producing a GEP since we simply load the address +void test_zero_elem(const int A[N], int result[1], int idx) { +#pragma DYN array_partition array = intermediate dimension = 1 style = \ + block factor = 2 + int intermediate[N]; + intermediate[idx] = A[idx]; + intermediate[0] = A[0]; + *result = intermediate[0] + intermediate[idx]; +} + +int main(void) { + int A[N]; + int result[1]; + for (int i = 0; i < N; ++i) { + A[i] = rand() % 100; + } + CALL_KERNEL(test_zero_elem, A, result, rand() % N); + return 0; +} diff --git a/lib/Transforms/LLVMIR/ArrayPartition.cpp b/lib/Transforms/LLVMIR/ArrayPartition.cpp index 0136b7abf..e7b4a1905 100644 --- a/lib/Transforms/LLVMIR/ArrayPartition.cpp +++ b/lib/Transforms/LLVMIR/ArrayPartition.cpp @@ -9,6 +9,7 @@ #include "polly/ScopPass.h" #include "polly/Support/ISLTools.h" +#include "llvm/ADT/SetVector.h" #include "llvm/ADT/StringRef.h" #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/IR/DerivedTypes.h" @@ -23,6 +24,7 @@ #include "llvm/Analysis/LoopInfo.h" #include "llvm/IR/Instruction.h" #include "llvm/IR/Instructions.h" +#include "llvm/Transforms/Utils/BasicBlockUtils.h" #include #include #include @@ -72,13 +74,21 @@ using DimInfoOfAllDimensions = std::vector; namespace { +struct PartitionInfo { + unsigned dimension; + unsigned factor; + std::string style; +}; + struct AccessInfo { std::map accessMaps; std::map instToScopId; // Base to the set of instructions storing to this base - std::map> baseToInsts; + // NOTE: llvm::SetVector here since it preserves the order in which + // instructions are inserted as well as functioning like a set + std::map> baseToInsts; bool sameScop(Instruction *i, Instruction *j) const { if (!instToScopId.count(i)) @@ -159,6 +169,19 @@ Instruction *findBaseGEP(Instruction *inst) { return findBaseGEPInternal(addr, inst); } +// Helper function for getting StringLiterals from LLVM value +std::optional extractStringLiteral(Value *v) { + auto *gv = dyn_cast(v->stripPointerCasts()); + if (!gv || !gv->hasInitializer()) + return std::nullopt; + + auto *cda = dyn_cast(gv->getInitializer()); + if (!cda || !cda->isCString()) + return std::nullopt; + + return cda->getAsCString(); +} + /// \brief: After shrinking the array to an optimal size, we also need to update /// the GEP ops to the new array. /// @@ -619,12 +642,410 @@ void partitionGlobalAlloca(Module *mod, llvm::GlobalVariable *gblConstant, } } -} // namespace +// Function that returns map of arrayNames -> partitionInfo for later partition +// has side effect of removing all call sites of pragma markers and deleting the +// pragma marker function. +// +// Pragma Syntax: +// #pragma DYN array_partition array={array name} dimension={1..array depth} +// style={block, cyclic, full} factor=int +// +// Example for int arr[10][10]; +// #pragma DYN array_partition array=arr dimension=2 style=block factor=2 +// +// NOTE: This could be made more general, i.e. providing the name of the pragma +// marker function and parsing out the function arguemnts and naming them +// later/providing a handler function that maps from argument index to struct +// field. Overkill if this is the only occurance for this +std::map collectAndErasePragmaMarkers(Function &f) { + std::map result; + std::vector callSites; + Function *markerFn = nullptr; -struct ArrayPartition : PassInfoMixin { + for (auto &bb : f) { + for (auto &inst : bb) { + auto *call = dyn_cast(&inst); + if (!call) { + continue; + } + + Function *callee = call->getCalledFunction(); + if (!callee || callee->getName() != "__dyn_array_partition") { + continue; + } + + markerFn = callee; + + if (call->arg_size() != 4) { + llvm::report_fatal_error( + Twine("__dyn_array_partition: expected 4 arguments, got ") + + Twine(call->arg_size())); + } + + auto arrName = extractStringLiteral(call->getArgOperand(0)); + auto *dimConst = dyn_cast(call->getArgOperand(1)); + auto *factorConst = dyn_cast(call->getArgOperand(2)); + auto style = extractStringLiteral(call->getArgOperand(3)); + + if (!arrName) + llvm::report_fatal_error( + "__dyn_array_partition: could not recover array name " + "string literal"); + if (!dimConst || !factorConst) + llvm::report_fatal_error( + "__dyn_array_partition: dimension/factor must be " + "constant integers"); + if (!style) + llvm::report_fatal_error( + "__dyn_array_partition: could not recover style string" + "literal"); + + LLVM_DEBUG(llvm::errs() + << "Partitioning: " << arrName << "\n\t" << dimConst << "\n\t" + << factorConst << "\n\t" << style << "\n"); + + result[arrName->str()] = PartitionInfo{ + static_cast(dimConst->getZExtValue()), + static_cast(factorConst->getZExtValue()), style->str()}; + + callSites.push_back(call); + } + } + + // Finally remove all found call sites from the function + for (auto *call : callSites) + call->eraseFromParent(); + + // Finally remove the external function declaration + if (markerFn && markerFn->use_empty()) + markerFn->eraseFromParent(); + + return result; +} + +// Transforms the code by creating branches accessing different banks for every +// array access. +// Original code: +// #pragma DYN array_partition array=arr dimension=1 style=block factor=3 +// int arr[32]; +// int sum = 0; +// for (int i = 0; i < 32; i++) +// sum += arr[i]; +// +// Transformed code: +// int arr0[10], arr1[10], arr2[12]; +// int sum = 0; +// for (int i = 0; i < 32; i++) +// if (i / 10 == 0){ +// sum += arr0[i]; +// } else if (i / 10 == 1){ +// sum += arr1[i - 10]; +// } else { +// sum += arr2[i - 20]; +// } +// NOTE: This is should be resolved/unrolled later on by LLVM automiatically +// via opitmization such that there is no branching logic remaining +void rewriteAccessWithBranching(Instruction *inst, AllocaInst *baseAlloca, + ArrayRef banks, + ArrayRef bankInfo, + const PartitionInfo &partInfo, + unsigned accessIdx) { + + StringRef arrName = baseAlloca->getName(); + + Instruction *baseGEPOrInst = findBaseGEP(inst); + auto *gepInst = dyn_cast(baseGEPOrInst); + + auto *load = dyn_cast(inst); + auto *store = dyn_cast(inst); + StringRef opKind = load ? "load" : "store"; + + if (!gepInst) { + if (baseGEPOrInst != inst) { + llvm::report_fatal_error( + "__dyn_array_partition: expected a GEP for this access"); + } + + // Early return in case we have a 0 access in to an array. i.e. there is no + // gep construction. In this case we also do not need to change the access + // pattern since the 0th element is always in the 0th bank + if (load) + load->setOperand(0, banks[0]); + else if (store) + store->setOperand(1, banks[0]); + return; + } + + // If the wanted dimension to partition from exceeds the available indices in + // the gep instruction we can not change the indices accordingly + if (partInfo.dimension >= gepInst->getNumIndices()) { + llvm::report_fatal_error("__dyn_array_partition: dimension exceeds the " + "number of indices in this access"); + } + + // For gep instruciton: + // %p = getelementptr [10 x [10 x i32]], ptr %arr, i64 0, i64 1, 5 + // and dimension = 2 we'd want origIdx to have the value of the second index, + // i.e. 5 in this example + Value *origIdx = *(gepInst->idx_begin() + partInfo.dimension); + Type *addrTy = origIdx->getType(); + + unsigned factor = static_cast(banks.size()); + unsigned totalSize = 0; + for (auto &[firstIndex, step, elems] : bankInfo) + totalSize += elems; + + unsigned chunkSize = totalSize / factor; + unsigned remainder = totalSize % factor; + + LLVMContext &ctx = inst->getContext(); + Function *f = inst->getParent()->getParent(); + + // Since LLVM might reuse gep instructions we have to insert duplicate ones to + // ensure that the later split can happen correctly. Otherwise we might split + // well before the current instruction that we'd like to split on + if (gepInst->getNextNode() != inst) { + gepInst = cast(gepInst->clone()); + gepInst->insertBefore(inst); + } + + // Splits origBB right before the GEP: everything from the GEP onward + // moves into mergeBB. + // + // origBB: + // ... + // %gep = getelementptr ...; + // %v = load ...; + // br %next + // + // -> + // + // origBB: + // ... + // br %mergeBB + // + // mergeBB: + // %gep = getelementptr ...; + // %v = load ...; + // br %next + BasicBlock *origBB = gepInst->getParent(); + BasicBlock *mergeBB = + SplitBlock(origBB, gepInst->getIterator(), (DominatorTree *)nullptr, + (LoopInfo *)nullptr, (MemorySSAUpdater *)nullptr, + "partition." + arrName + ".merge" + Twine(accessIdx)); + Instruction *placeholderBr = origBB->getTerminator(); + + IRBuilder<> preBuilder(placeholderBr); + Type *i64Ty = Type::getInt64Ty(ctx); + + // Computation of th bank that the index falls into + // NOTE: Only block partitioning needs the tooLarge addition, since cyclic + // automatically assign every index to a bin by using modular arithmetic + Value *bankIdxNative; + if (partInfo.style == "cyclic") { + bankIdxNative = preBuilder.CreateURem( + origIdx, ConstantInt::get(addrTy, factor), "bank.idx"); + } else { // "block" or "complete" + Value *raw = preBuilder.CreateUDiv( + origIdx, ConstantInt::get(addrTy, chunkSize), "bank.raw"); + if (remainder != 0) { + Value *maxBank = ConstantInt::get(addrTy, factor - 1); + Value *tooLarge = preBuilder.CreateICmpUGT(raw, maxBank); + bankIdxNative = + preBuilder.CreateSelect(tooLarge, maxBank, raw, "bank.idx"); + } else { + bankIdxNative = raw; + } + } + + // Normalize to i64 for the comparison chain below, regardless of addrTy. + // NOTE: I had issues with i32 vs i64 indices before + Value *bankIdx = + bankIdxNative->getType() == i64Ty + ? bankIdxNative + : preBuilder.CreateZExtOrTrunc(bankIdxNative, i64Ty, "bank.idx.64"); + + placeholderBr->eraseFromParent(); + + // One basic block per bank for either load/store + std::vector bankBBs; + bankBBs.reserve(factor); + for (unsigned bank = 0; bank < factor; ++bank) + bankBBs.push_back(BasicBlock::Create( + ctx, "partition." + arrName + "." + opKind + "." + Twine(bank), f)); + + if (factor == 1) { + IRBuilder<>(origBB).CreateBr(bankBBs[0]); + } else { + // if/else-if chain: + // bankIdx == 0 ? bank0 : (bankIdx == 1 ? bank1 : ... : bank[factor-1]) + BasicBlock *currentBB = origBB; + for (unsigned b = 0; b + 1 < factor; ++b) { + IRBuilder<> checkBuilder(currentBB); + Value *cmp = checkBuilder.CreateICmpEQ( + bankIdx, ConstantInt::get(i64Ty, b), "bank.cmp." + Twine(b)); + bool isLastCheck = (b + 2 == factor); + BasicBlock *elseBB = + isLastCheck + ? bankBBs[factor - 1] + : BasicBlock::Create( + ctx, "partition." + arrName + ".cmp." + Twine(b + 1), f); + checkBuilder.CreateCondBr(cmp, bankBBs[b], elseBB); + currentBB = elseBB; + } + } + + // Body for the load/store basic blocks + std::vector> incoming; + for (unsigned b = 0; b < factor; ++b) { + IRBuilder<> caseBuilder(bankBBs[b]); + auto [firstIndex, step, elems] = bankInfo[b]; + + // Creation of bank specific target index + // newTargetIdx = (origIdx - firstIndex) / step + // e.g. firstIndex=50, step=1 (block bank covering global 50..99): + // %sub = sub i64 %origIdx, 50 + // %part.idx = udiv i64 %sub, 1 ; a[73] -> bank[23] + Value *newTargetIdx = caseBuilder.CreateUDiv( + caseBuilder.CreateSub(origIdx, ConstantInt::get(addrTy, firstIndex)), + ConstantInt::get(addrTy, step), "part.idx"); + + std::vector newIndices; + newIndices.reserve(gepInst->getNumIndices()); + + // GEP indices are either the same as before or transformed exactly in the + // case where they match the target dimension + for (unsigned i = 0; i < gepInst->getNumIndices(); ++i) { + auto *newIndex = + i == partInfo.dimension ? newTargetIdx : *(gepInst->idx_begin() + i); + newIndices.push_back(newIndex); + } + + Value *newGEP = caseBuilder.CreateInBoundsGEP( + banks[b]->getAllocatedType(), banks[b], newIndices, "part.gep"); + + if (load) { + LoadInst *newLoad = caseBuilder.CreateLoad(load->getType(), newGEP, + load->getName() + ".part"); + newLoad->setAlignment(load->getAlign()); + incoming.emplace_back(bankBBs[b], newLoad); + } else if (store) { + StoreInst *newStore = + caseBuilder.CreateStore(store->getValueOperand(), newGEP); + newStore->setAlignment(store->getAlign()); + } + caseBuilder.CreateBr(mergeBB); + } + + // In case we were working with a load we need to unify using phi nodes + if (load) { + IRBuilder<> mergeBuilder(&mergeBB->front()); + PHINode *phi = mergeBuilder.CreatePHI(load->getType(), factor, + load->getName() + ".merged"); + for (auto &[bb, val] : incoming) + phi->addIncoming(val, bb); + load->replaceAllUsesWith(phi); + } + + inst->eraseFromParent(); + // Only conditionally delete the gep instruction. LLVM may reuse previous gep + // instructions to access different parts of the array, as such we should only + // delete once there is no use left for the calculated gep + if (gepInst->use_empty()) { + gepInst->eraseFromParent(); + } +} + +// Helper function for gathering type of inner arrays in multidimensional arrays +ArrayType *getTargetDimType(Type *ty, unsigned dimension) { + for (unsigned i = 1; i < dimension; ++i) { + auto *at = dyn_cast(ty); + if (!at) + llvm::report_fatal_error( + "__dyn_array_partition: dimension exceeds array rank"); + ty = at->getElementType(); + } + auto *at = dyn_cast(ty); + if (!at) + llvm::report_fatal_error( + "__dyn_array_partition: dimension exceeds array rank"); + return at; +} + +// Recursive funtion for rebuilding the array type. Go through each dimension +// and generate the corresponding array type +Type *getPartitionedArrayType(Type *ty, unsigned dimension, unsigned numElems) { + auto *at = cast(ty); + if (dimension == 1) + return ArrayType::get(at->getElementType(), numElems); + return ArrayType::get( + getPartitionedArrayType(at->getElementType(), dimension - 1, numElems), + at->getNumElements()); +} + +std::tuple, std::vector> +createPartitionBankAllocas(AllocaInst *baseAlloca, const PartitionInfo &info) { + // Get total size of the dimension we'd like to partition accross + unsigned totalSize = + getTargetDimType(baseAlloca->getAllocatedType(), info.dimension) + ->getArrayNumElements(); + + if (totalSize < 1) { + llvm::report_fatal_error( + "__dyn_array_partition: cannot partition a less than 1 length array"); + } + + unsigned factor = info.style == "complete" ? totalSize : info.factor; + + if (factor == 0 || factor > totalSize) { + llvm::report_fatal_error("__dyn_array_partition: factor must be in [1, N]"); + } - unsigned memCount = 0; + // Logic for determining bank partition numbers + unsigned chunkSize = totalSize / factor; + unsigned remainder = totalSize % factor; + + std::vector banks; + banks.reserve(factor); + if (info.style == "block" || info.style == "complete") { + unsigned offset = 0; + for (unsigned bank = 0; bank < factor; ++bank) { + // Put all remaining elements into the last bank + unsigned elems = chunkSize + (bank + 1 == factor ? remainder : 0); + banks.emplace_back(offset, 1, elems); + offset += elems; + } + } else if (info.style == "cyclic") { + for (unsigned bank = 0; bank < factor; ++bank) { + unsigned elems = chunkSize + (bank < remainder ? 1u : 0u); + banks.emplace_back(bank, factor, elems); + } + } else { + llvm::report_fatal_error(Twine("__dyn_array_partition: unknown style '") + + info.style + + "' (expected block, cyclic, or complete)"); + } + + // Insertion of allocations + std::vector bankAllocas; + bankAllocas.reserve(banks.size()); + + IRBuilder<> builder(baseAlloca->getNextNode()); + for (auto &[firstIndex, step, elems] : banks) { + Type *bankType = getPartitionedArrayType(baseAlloca->getAllocatedType(), + info.dimension, elems); + AllocaInst *newAlloca = builder.CreateAlloca( + bankType, baseAlloca->getArraySize(), baseAlloca->getName()); + newAlloca->setAlignment(baseAlloca->getAlign()); + bankAllocas.push_back(newAlloca); + } + return {bankAllocas, banks}; +} + +} // namespace + +struct ArrayPartition : PassInfoMixin { PreservedAnalyses run(Function &f, FunctionAnalysisManager &fam); }; @@ -632,27 +1053,15 @@ PreservedAnalyses ArrayPartition::run(Function &f, FunctionAnalysisManager &fam) { if (f.getName() == "main") { - llvm::errs() - << "Skipping main function for automatic array partitioning!\n"; + LLVM_DEBUG(llvm::errs() + << "Skipping main function for automatic array partitioning!\n"); return PreservedAnalyses::all(); } - auto islCtx = isl::ctx(isl_ctx_alloc()); - - auto ®ionInfoAnalysis = fam.getResult(f); - - auto &scopInfoAnalysis = fam.getResult(f); - - auto &aliasAnalysis = fam.getResult(f); - - // Needed for constructing the global constants - Module *mod = f.getParent(); + auto pragmaInfo = collectAndErasePragmaMarkers(f); AccessInfo info; - std::deque rq; - getAllRegions(*regionInfoAnalysis.getTopLevelRegion(), rq); - for (auto &bb : f) { for (auto &inst : bb) { if (isa(&inst)) { @@ -662,67 +1071,31 @@ PreservedAnalyses ArrayPartition::run(Function &f, } } - Scop *s; - unsigned scopId = 0; - for (Region *r : rq) { - if ((s = scopInfoAnalysis.getScop(r))) { - for (auto &stmt : *s) { - for (auto *memAccess : stmt) { - auto *inst = memAccess->getAccessInstruction(); - - info.instToScopId[inst] = scopId; - - // Find the access - auto *memoryAccess = stmt.getArrayAccessOrNULLFor(inst); - - if (!memoryAccess) { - continue; - } - - // Maps iteration indices to array access indices: - // example: - // stmt[i, j] -> A[i, j] - // stmt[i, j] -> B[i, j - 1] - // NOTE: - // - The base address might be different for different maps. - isl::map currentMap = memoryAccess->getLatestAccessRelation(); - - // The domain of the map, e.g., the loop bounds for the iterators. - isl::set domain = stmt.getDomain(); - - // The range of the map over the domain - // e.g., - // - input: stmt[i, j] -> A[i, j] | i \in [0, N] and j \in [0, M] - // - output: A[i, j] | i \in [0, N] and j \in [0, M] - isl::set range = currentMap.intersect_domain(domain).range(); - info.accessMaps[inst] = range; - } - } - scopId += 1; - } - } - - // For each base address of the GEPs that are allocas (i.e., a separate RAM in - // HLS circuit), compute the optimal partitioning and create separate alloca - // instructions. for (auto [base, insts] : info.baseToInsts) { - if (isa(base)) { - if (auto *allocaInst = dyn_cast(base)) { - // If the base is an alloca, we can partition it - partitionVariableAlloca(allocaInst, insts, info, aliasAnalysis, islCtx); - } else { - assert(false && - "Base address of the GEP is not an alloca, cannot partition!"); + auto it = pragmaInfo.find(base->getName().str()); + if (it != pragmaInfo.end()) { + auto *baseAlloca = dyn_cast(base); + if (!baseAlloca) { + llvm::report_fatal_error( + Twine("__dyn_array_partition: only local (alloca) arrays are " + "supported so far '") + + base->getName() + "' is not one"); } - } - if (isa(base)) { - if (auto *globVar = dyn_cast(base)) { - // If the base is a global variable, we can partition it - partitionGlobalAlloca(mod, globVar, insts, info, aliasAnalysis, islCtx); + const PartitionInfo &partInfo = it->second; + auto [bankAllocas, bankInfo] = + createPartitionBankAllocas(baseAlloca, partInfo); + + unsigned accessIdx = 0; + for (Instruction *inst : insts) { + rewriteAccessWithBranching(inst, baseAlloca, bankAllocas, bankInfo, + partInfo, accessIdx); + ++accessIdx; } + continue; } } + return PreservedAnalyses::all(); } diff --git a/test/tools/dyn-pragmas/pragma-read-erase.c b/test/tools/dyn-pragmas/pragma-read-erase.c new file mode 100644 index 000000000..6b79c2618 --- /dev/null +++ b/test/tools/dyn-pragmas/pragma-read-erase.c @@ -0,0 +1,11 @@ +// RUN: %dyn-clang-pragmas %s | opt -S -load-pass-plugin "%shlibdir/ArrayPartition.so" -passes="array-partition" | FileCheck %s + +// Check that the pragma is correctly erased after doing an array partition pass + +// CHECK-NOT: call void @__dyn_array_partition +// CHECK-NOT: declare void @__dyn_array_partition + +void kernel(void) { +#pragma DYN array_partition array=arr dimension=1 style=block factor=4 + int arr[100]; +} diff --git a/tools/dynamatic/scripts/compile.sh b/tools/dynamatic/scripts/compile.sh index 16e756950..56ebeca45 100755 --- a/tools/dynamatic/scripts/compile.sh +++ b/tools/dynamatic/scripts/compile.sh @@ -49,7 +49,9 @@ F_C_REWRITTEN="$COMP_DIR/$KERNEL_NAME.c" F_CLANG="$COMP_DIR/clang.ll" F_CLANG_OPTIMIZED="$COMP_DIR/clang.opt.ll" -F_CLANG_OPTIMIZED_DEPENDENCY="$COMP_DIR/clang.opt.dep.ll" +F_CLANG_OPTIMIZED_PARTITIONED="$COMP_DIR/clang.opt.part.ll" +F_CLANG_OPTIMIZED_PARTITIONED_CLEANED="$COMP_DIR/clang.opt.part.clean.ll" +F_CLANG_OPTIMIZED_PARTITIONED_CLEANED_DEPENDENCY="$COMP_DIR/clang.opt.part.clean.dep.ll" F_CF="$COMP_DIR/cf.mlir" F_CF_TRANSFORMED="$COMP_DIR/cf_transformed.mlir" F_CF_CONSUMED_PRAGMARKERS="$COMP_DIR/cf_consumed_pragmarkers.mlir" @@ -108,8 +110,6 @@ export_cfg() { return 0 } -# ============================================================================ # -# Compilation flow # ============================================================================ # # Reset output directory @@ -211,17 +211,33 @@ exit_on_fail "Failed to apply optimization to LLVM IR" \ # - ArrayParititon pass currently breaks the SCoP analysis in Polly. Therefore, # we need to first attach analysis results to memory ops and then apply memory # bank partition. +$LLVM_OPT -S \ + -load-pass-plugin "$DYNAMATIC_DIR/build/lib/ArrayPartition.so" \ + -passes="array-partition" \ + -polly-process-unprofitable \ + "$F_CLANG_OPTIMIZED" \ + > "$F_CLANG_OPTIMIZED_PARTITIONED" +exit_on_fail "Failed to perform array partitioning in LLVM IR" + +$LLVM_OPT -S \ + -load-pass-plugin "$DYNAMATIC_DIR/build/lib/GuardLoadStore.so" \ + -passes="function(guard-load-store,instcombine),always-inline" \ + -polly-process-unprofitable \ + "$F_CLANG_OPTIMIZED_PARTITIONED" \ + > "$F_CLANG_OPTIMIZED_PARTITIONED_CLEANED" +exit_on_fail "Failed to clean up after array partitioning in LLVM IR" + $LLVM_OPT -S \ -load-pass-plugin "$DYNAMATIC_DIR/build/lib/MemDepAnalysis.so" \ -passes="mem-dep-analysis" \ -polly-process-unprofitable \ - "$F_CLANG_OPTIMIZED" \ - > "$F_CLANG_OPTIMIZED_DEPENDENCY" + "$F_CLANG_OPTIMIZED_PARTITIONED_CLEANED" \ + > "$F_CLANG_OPTIMIZED_PARTITIONED_CLEANED_DEPENDENCY" exit_on_fail "Failed to apply memory dependency analysis to LLVM IR" \ "Applied memory dependency analysis to LLVM IR" $LLVM_TO_STD_TRANSLATION_BIN \ - "$F_CLANG_OPTIMIZED_DEPENDENCY" \ + "$F_CLANG_OPTIMIZED_PARTITIONED_CLEANED_DEPENDENCY" \ -function-name "$KERNEL_NAME" \ -csource "$F_C_SOURCE" \ -dynamatic-path "$DYNAMATIC_DIR" \ diff --git a/tools/integration/TEST_SUITE.cpp b/tools/integration/TEST_SUITE.cpp index 88b0baf54..52232c93e 100644 --- a/tools/integration/TEST_SUITE.cpp +++ b/tools/integration/TEST_SUITE.cpp @@ -290,6 +290,7 @@ class CBCSolverFixture : public BaseFixture {}; // Use FPL22 placement algorithm on a small subset of MiscBenchmarks class FPL22Fixture : public BaseFixture {}; class MemoryFixture : public BaseFixture {}; +class MemoryPartitionFixture : public BaseFixture {}; class SharingFixture : public BaseFixture {}; class SharingUnitTestFixture : public BaseFixture {}; class SpecFixture : public BaseFixture {}; @@ -373,6 +374,24 @@ TEST_P(MemoryFixture, basic) { logPerformance(config.simTime); } +TEST_P(MemoryPartitionFixture, basic) { + IntegrationTest config{ + // clang-format off + .name = GetParam(), + .testName = getVerboseOutdirSuffix(), + .benchmarkPath = fs::path(DYNAMATIC_ROOT) / "integration-test" / "memory_partition", + .testVerilog = true, + .useSharing = false, + .milpSolver = "gurobi", + .bufferAlgorithm = "fpga20", + .simTime = -1 + // clang-format on + }; + EXPECT_EQ(config.run(), 0); + RecordProperty("cycles", std::to_string(config.simTime)); + logPerformance(config.simTime); +} + /// This testing fixture runs the test with and without sharing. It checks /// whenever the sharing option is enabled, the pass can run without any /// interruption and does not penalize the latency. @@ -781,6 +800,17 @@ INSTANTIATE_TEST_SUITE_P( ), [](const auto &info) { return "memory_" + info.param; }); +INSTANTIATE_TEST_SUITE_P( + MemoryPartitionBenchmarks, MemoryPartitionFixture, + testing::Values( + "test_1d_partition", + "test_1d_partition_f4", + "test_1d_partition_fu", + "test_2d_partition_inner", + "test_2d_partition" + ), + [](const auto &info) { return "memory_partition_" + info.param; }); + INSTANTIATE_TEST_SUITE_P(SharingUnitTests, SharingUnitTestFixture, testing::Values( "share_test_1",