Dont merge - testing CI

.github/actions/setup-build/action.yml

@@ -0,0 +1,36 @@
+name: "Setup build environment"
+description: "Setup the build environment. An action so that it can be shared between in-tree/out-of-tree jobs"
+
+inputs:
+  cache-key:
+    description: |
+      Additional string that is used to compute the ccache hash.
+      Different jobs running the action need distinct values for this key,
+      but the content is irrelevant.
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+  - name: Set up Python
+    uses: actions/setup-python@v2
+    with:
+      python-version: 3.9
+  - name: Install MLIR Python depends
+    run: |
+      python -m pip install -r $GITHUB_WORKSPACE/externals/llvm-project/mlir/python/requirements.txt
+    shell: bash
+  - name: Install PyTorch nightly depends
+    run: |
+      python -m pip install -r requirements.txt
+    shell: bash
+  - name: Install Ninja
+    uses: llvm/actions/install-ninja@55d844821959226fab4911f96f37071c1d4c3268
+  - name: Get Submodule Hash
+    id: get-submodule-hash
+    run: echo "::set-output name=hash::$(md5sum $(git submodule status))"
+    shell: bash
+  - name: Ccache for C++ compilation
+    uses: hendrikmuhs/ccache-action@4687d037e4d7cf725512d9b819137a3af34d39b3
+    with:
+      key: ${{ runner.os }}-clangreleaseasserts-${{ steps.get-submodule-hash.outputs.hash }}-${{ inputs.cache-key }}

.github/workflows/buildAndTest.yml

@@ -16,33 +16,16 @@ on:
 
 jobs:
   build:
-    name: Build and Test (Release Asserts)
+    name: Build and Test and maybe Publish (Release Asserts)
     runs-on: ubuntu-20.04
     steps:
-    - name: Set up Python
-      uses: actions/setup-python@v2
-      with:
-        python-version: 3.9
     - name: Get torch-mlir
       uses: actions/checkout@v2
       with:
         submodules: 'true'
-    - name: Install MLIR Python depends
-      run: |
-        python -m pip install -r $GITHUB_WORKSPACE/externals/llvm-project/mlir/python/requirements.txt
-    - name: Install PyTorch nightly depends
-      run: |
-        python -m pip install -r requirements.txt
-    - name: Install Ninja
-      uses: llvm/actions/install-ninja@55d844821959226fab4911f96f37071c1d4c3268
-    - name: Get Submodule Hash
-      id: get-submodule-hash
-      run: echo "::set-output name=hash::$(md5sum $(git submodule status))"
-      shell: bash
-    - name: Ccache for C++ compilation
-      uses: hendrikmuhs/ccache-action@4687d037e4d7cf725512d9b819137a3af34d39b3
+    - uses: ./.github/actions/setup-build
       with:
-        key: ${{ runner.os }}-clangreleaseasserts-${{ steps.get-submodule-hash.outputs.hash }}
+        cache-key: 'in-tree'
     - name: Build and Test torch-mlir (Assert)
       run: |
         cd $GITHUB_WORKSPACE
@@ -106,3 +89,55 @@ jobs:
         GITHUB_TOKEN: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }}
       with:
         release_id: ${{ github.event.inputs.release_id }}
+
+  build-out-of-tree:
+    name: Build and Test out-of-tree (Release Asserts)
+    runs-on: ubuntu-20.04
+    steps:
+    - name: Get torch-mlir
+      uses: actions/checkout@v2
+      with:
+        submodules: 'true'
+    - uses: ./.github/actions/setup-build
+      with:
+        cache-key: 'out-of-tree'
+    - name: Build LLVM (standalone)
+      run: |
+        cd $GITHUB_WORKSPACE
+        cmake -Bllvm-build -GNinja \
+          -DCMAKE_BUILD_TYPE=Release \
+          -DCMAKE_LINKER=lld \
+          -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
+          -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ \
+          -DPython3_EXECUTABLE=$(which python) \
+          -DLLVM_ENABLE_ASSERTIONS=ON \
+          -DLLVM_ENABLE_PROJECTS=mlir \
+          -DMLIR_ENABLE_BINDINGS_PYTHON=ON \
+          -DLLVM_TARGETS_TO_BUILD=host \
+          externals/llvm-project/llvm
+        ninja -Cllvm-build
+
+    - name: Build and test torch-mlir (out-of-tree)
+      run: |
+        cd $GITHUB_WORKSPACE
+        cmake -GNinja -Bbuild \
+          -DCMAKE_LINKER=lld \
+          -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
+          -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ \
+          -DMLIR_DIR="$(pwd)/llvm-build/lib/cmake/mlir/" \
+          -DLLVM_DIR="$(pwd)/llvm-build/lib/cmake/llvm/" \
+          -DMLIR_ENABLE_BINDINGS_PYTHON=ON \
+          -DPython3_EXECUTABLE=$(which python) \
+          .
+        ninja -Cbuild check-torch-mlir-all
+
+    - name: RefBackend - TorchScript end-to-end tests
+      run: |
+        cd $GITHUB_WORKSPACE
+        export PYTHONPATH="$GITHUB_WORKSPACE/build/python_packages/torch_mlir"
+        python -m e2e_testing.torchscript.main --config=refbackend -v
+    - name: TOSA backend - TorchScript end-to-end tests
+      run: |
+        cd $GITHUB_WORKSPACE
+        export PYTHONPATH="$GITHUB_WORKSPACE/build/python_packages/torch_mlir"
+        python -m e2e_testing.torchscript.main --config=tosa -v

README.md

@@ -53,6 +53,13 @@ python -m pip install -r requirements.txt
 ```
 
 ## Build
+
+Two setups are possible to build: in-tree and out-of-tree. The in-tree setup is the most straightforward, as it will build LLVM dependencies as well.
+
+### Building torch-mlir in-tree
+
+The following command generates configuration files to build the project *in-tree*, that is, using llvm/llvm-project as the main build. This will build LLVM as well as torch-mlir and its subprojects.
+
 ```shell
 cmake -GNinja -Bbuild \
   -DCMAKE_C_COMPILER=clang \
@@ -65,23 +72,51 @@ cmake -GNinja -Bbuild \
   -DMLIR_ENABLE_BINDINGS_PYTHON=ON \
   -DLLVM_TARGETS_TO_BUILD=host \
   externals/llvm-project/llvm
-
-# Additional quality of life CMake flags:
-# Enable ccache:
-#  -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
-# Enable LLD (links in seconds compared to minutes)
-# -DCMAKE_EXE_LINKER_FLAGS_INIT="-fuse-ld=lld" -DCMAKE_MODULE_LINKER_FLAGS_INIT="-fuse-ld=lld" -DCMAKE_SHARED_LINKER_FLAGS_INIT="-fuse-ld=lld"
+```
+The following additional quality of life flags can be used to reduce build time:
+* Enabling ccache:
+```shell
+  -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
+```
+* Enabling LLD (links in seconds compared to minutes)
+```shell
+  -DCMAKE_EXE_LINKER_FLAGS_INIT="-fuse-ld=lld" -DCMAKE_MODULE_LINKER_FLAGS_INIT="-fuse-ld=lld" -DCMAKE_SHARED_LINKER_FLAGS_INIT="-fuse-ld=lld"
 # Use --ld-path= instead of -fuse-ld=lld for clang > 13
+```
 
+### Building against a pre-built LLVM
+
+If you have built llvm-project separately in the directory `$LLVM_INSTALL_DIR`, you can also build the project *out-of-tree* using the following command as template:
+```shell
+cmake -GNinja -Bbuild \
+  -DCMAKE_C_COMPILER=clang \
+  -DCMAKE_CXX_COMPILER=clang++ \
+  -DPython3_FIND_VIRTUALENV=ONLY \
+  -DMLIR_DIR="$LLVM_INSTALL_DIR/lib/cmake/mlir/" \
+  -DLLVM_DIR="$LLVM_INSTALL_DIR/lib/cmake/llvm/" \
+  -DMLIR_ENABLE_BINDINGS_PYTHON=ON \
+  -DLLVM_TARGETS_TO_BUILD=host \
+  .
+```
+The same QoL CMake flags can be used to enable ccache and lld. Be sure to have built LLVM with `-DLLVM_ENABLE_PROJECTS=mlir`.
+
+Be aware that the installed version of LLVM needs in general to match the committed version in `externals/llvm-project`. Using a different version may or may not work.
+
+
+### Build commands
+
+After either cmake run (in-tree/out-of-tree), use one of the following commands to build the project:
+```shell
 # Build just torch-mlir (not all of LLVM)
 cmake --build build --target tools/torch-mlir/all
 
 # Run unit tests.
 cmake --build build --target check-torch-mlir
 
-# Build everything (including LLVM)
+# Build everything (including LLVM if in-tree)
 cmake --build build
 ```
+
 ## Demos
 
 ## Setup Python Environment