--- /dev/null
- image: freebsd-12-1-release-amd64
+task:
+ name: freebsd
+ freebsd_instance:
- cargo_bin_cache:
- folder: ~/.cargo/bin
++ image: freebsd-13-1-release-amd64
+ setup_rust_script:
+ - pkg install -y curl git bash
+ - curl https://sh.rustup.rs -sSf --output rustup.sh
+ - sh rustup.sh --default-toolchain none -y --profile=minimal
- - # Enable backtraces for easier debugging
- - export RUST_BACKTRACE=1
- - # Reduce amount of benchmark runs as they are slow
- - export COMPILE_RUNS=2
- - export RUN_RUNS=2
+ target_cache:
+ folder: target
+ prepare_script:
+ - . $HOME/.cargo/env
+ - ./y.rs prepare
+ test_script:
+ - . $HOME/.cargo/env
+ - ./y.rs test
--- /dev/null
- - name: Cache cargo installed crates
- uses: actions/cache@v3
- with:
- path: ~/.cargo/bin
- key: ${{ runner.os }}-cargo-installed-crates
-
- - name: Cache cargo registry and index
- uses: actions/cache@v3
- with:
- path: |
- ~/.cargo/registry
- ~/.cargo/git
- key: ${{ runner.os }}-cargo-registry-and-index-${{ hashFiles('**/Cargo.lock') }}
-
+name: CI
+
+on:
+ - push
+ - pull_request
+
+jobs:
+ rustfmt:
+ runs-on: ubuntu-latest
+ timeout-minutes: 10
+
+ steps:
+ - uses: actions/checkout@v3
+
+ - name: Install rustfmt
+ run: |
+ rustup component add rustfmt
+
+ - name: Rustfmt
+ run: |
+ cargo fmt --check
+ rustfmt --check build_system/mod.rs
+
+ build:
+ runs-on: ${{ matrix.os }}
+ timeout-minutes: 60
+
++ defaults:
++ run:
++ shell: bash
++
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - os: ubuntu-20.04 # FIXME switch to ubuntu-22.04 once #1303 is fixed
+ env:
+ TARGET_TRIPLE: x86_64-unknown-linux-gnu
+ - os: macos-latest
+ env:
+ TARGET_TRIPLE: x86_64-apple-darwin
+ # cross-compile from Linux to Windows using mingw
+ - os: ubuntu-latest
+ env:
+ TARGET_TRIPLE: x86_64-pc-windows-gnu
+ - os: ubuntu-latest
+ env:
+ TARGET_TRIPLE: aarch64-unknown-linux-gnu
+ # s390x requires QEMU 6.1 or greater, we could build it from source, but ubuntu 22.04 comes with 6.2 by default
+ - os: ubuntu-latest
+ env:
+ TARGET_TRIPLE: s390x-unknown-linux-gnu
++ - os: windows-latest
++ env:
++ TARGET_TRIPLE: x86_64-pc-windows-msvc
++ - os: windows-latest
++ env:
++ TARGET_TRIPLE: x86_64-pc-windows-gnu
+
+ steps:
+ - uses: actions/checkout@v3
+
- key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('rust-toolchain', '**/Cargo.lock') }}
+ - name: Cache cargo target dir
+ uses: actions/cache@v3
+ with:
+ path: build/cg_clif
- rustup target add x86_64-pc-windows-gnu
++ key: ${{ runner.os }}-${{ matrix.env.TARGET_TRIPLE }}-cargo-build-target-${{ hashFiles('rust-toolchain', '**/Cargo.lock') }}
++
++ - name: Set MinGW as the default toolchain
++ if: matrix.os == 'windows-latest' && matrix.env.TARGET_TRIPLE == 'x86_64-pc-windows-gnu'
++ run: rustup set default-host x86_64-pc-windows-gnu
+
+ - name: Install MinGW toolchain and wine
+ if: matrix.os == 'ubuntu-latest' && matrix.env.TARGET_TRIPLE == 'x86_64-pc-windows-gnu'
+ run: |
+ sudo apt-get update
+ sudo apt-get install -y gcc-mingw-w64-x86-64 wine-stable
- run: |
- # Enable backtraces for easier debugging
- export RUST_BACKTRACE=1
-
- # Reduce amount of benchmark runs as they are slow
- export COMPILE_RUNS=2
- export RUN_RUNS=2
-
- # Enable extra checks
- export CG_CLIF_ENABLE_VERIFIER=1
-
- ./y.rs test
+
+ - name: Install AArch64 toolchain and qemu
+ if: matrix.os == 'ubuntu-latest' && matrix.env.TARGET_TRIPLE == 'aarch64-unknown-linux-gnu'
+ run: |
+ sudo apt-get update
+ sudo apt-get install -y gcc-aarch64-linux-gnu qemu-user
+
+ - name: Install s390x toolchain and qemu
+ if: matrix.env.TARGET_TRIPLE == 's390x-unknown-linux-gnu'
+ run: |
+ sudo apt-get update
+ sudo apt-get install -y gcc-s390x-linux-gnu qemu-user
+
++ - name: Use sparse cargo registry
++ run: |
++ cat >> ~/.cargo/config.toml <<EOF
++ [unstable]
++ sparse-registry = true
++ EOF
++
+ - name: Prepare dependencies
+ run: ./y.rs prepare
+
+ - name: Build without unstable features
+ env:
+ TARGET_TRIPLE: ${{ matrix.env.TARGET_TRIPLE }}
+ # This is the config rust-lang/rust uses for builds
+ run: ./y.rs build --no-unstable-features
+
+ - name: Build
+ run: ./y.rs build --sysroot none
+
+ - name: Test
+ env:
+ TARGET_TRIPLE: ${{ matrix.env.TARGET_TRIPLE }}
- if: matrix.env.TARGET_TRIPLE != 'x86_64-pc-windows-gnu'
- uses: actions/upload-artifact@v2
++ run: ./y.rs test
+
+ - name: Package prebuilt cg_clif
+ run: tar cvfJ cg_clif.tar.xz dist
+
+ - name: Upload prebuilt cg_clif
- if: matrix.env.TARGET_TRIPLE == 'x86_64-pc-windows-gnu'
++ if: matrix.os == 'windows-latest' || matrix.env.TARGET_TRIPLE != 'x86_64-pc-windows-gnu'
++ uses: actions/upload-artifact@v3
+ with:
+ name: cg_clif-${{ matrix.env.TARGET_TRIPLE }}
+ path: cg_clif.tar.xz
+
+ - name: Upload prebuilt cg_clif (cross compile)
- windows:
++ if: matrix.os != 'windows-latest' && matrix.env.TARGET_TRIPLE == 'x86_64-pc-windows-gnu'
+ uses: actions/upload-artifact@v3
+ with:
+ name: cg_clif-${{ runner.os }}-cross-x86_64-mingw
+ path: cg_clif.tar.xz
+
- fail-fast: false
++
++ abi_cafe:
+ runs-on: ${{ matrix.os }}
+ timeout-minutes: 60
+
++ defaults:
++ run:
++ shell: bash
++
+ strategy:
- # Native Windows build with MSVC
++ fail-fast: true
+ matrix:
+ include:
- # cross-compile from Windows to Windows MinGW
++ - os: ubuntu-latest
++ env:
++ TARGET_TRIPLE: x86_64-unknown-linux-gnu
++ - os: macos-latest
++ env:
++ TARGET_TRIPLE: x86_64-apple-darwin
+ - os: windows-latest
+ env:
+ TARGET_TRIPLE: x86_64-pc-windows-msvc
- - name: Cache cargo installed crates
- uses: actions/cache@v3
- with:
- path: ~/.cargo/bin
- key: ${{ runner.os }}-${{ matrix.env.TARGET_TRIPLE }}-cargo-installed-crates
-
- - name: Cache cargo registry and index
- uses: actions/cache@v3
- with:
- path: |
- ~/.cargo/registry
- ~/.cargo/git
- key: ${{ runner.os }}-${{ matrix.env.TARGET_TRIPLE }}-cargo-registry-and-index-${{ hashFiles('**/Cargo.lock') }}
-
+ - os: windows-latest
+ env:
+ TARGET_TRIPLE: x86_64-pc-windows-gnu
+
+ steps:
+ - uses: actions/checkout@v3
+
- - name: Prepare dependencies
+ - name: Cache cargo target dir
+ uses: actions/cache@v3
+ with:
+ path: build/cg_clif
+ key: ${{ runner.os }}-${{ matrix.env.TARGET_TRIPLE }}-cargo-build-target-${{ hashFiles('rust-toolchain', '**/Cargo.lock') }}
+
+ - name: Set MinGW as the default toolchain
+ if: matrix.env.TARGET_TRIPLE == 'x86_64-pc-windows-gnu'
+ run: rustup set default-host x86_64-pc-windows-gnu
+
- git config --global core.autocrlf false
- rustc y.rs -o y.exe -g
- ./y.exe prepare
++ - name: Use sparse cargo registry
+ run: |
- - name: Build without unstable features
- env:
- TARGET_TRIPLE: ${{ matrix.env.TARGET_TRIPLE }}
- # This is the config rust-lang/rust uses for builds
- run: ./y.rs build --no-unstable-features
++ cat >> ~/.cargo/config.toml <<EOF
++ [unstable]
++ sparse-registry = true
++ EOF
+
- - name: Test
- run: |
- # Enable backtraces for easier debugging
- $Env:RUST_BACKTRACE=1
-
- # Reduce amount of benchmark runs as they are slow
- $Env:COMPILE_RUNS=2
- $Env:RUN_RUNS=2
-
- # Enable extra checks
- $Env:CG_CLIF_ENABLE_VERIFIER=1
-
- # WIP Disable some tests
-
- # This fails due to some weird argument handling by hyperfine, not an actual regression
- # more of a build system issue
- (Get-Content config.txt) -replace '(bench.simple-raytracer)', '# $1' | Out-File config.txt
-
- # This fails with a different output than expected
- (Get-Content config.txt) -replace '(test.regex-shootout-regex-dna)', '# $1' | Out-File config.txt
-
- ./y.exe test
-
- - name: Package prebuilt cg_clif
- # don't use compression as xzip isn't supported by tar on windows and bzip2 hangs
- run: tar cvf cg_clif.tar dist
-
- - name: Upload prebuilt cg_clif
- uses: actions/upload-artifact@v3
- with:
- name: cg_clif-${{ matrix.env.TARGET_TRIPLE }}
- path: cg_clif.tar
++ - name: Prepare dependencies
++ run: ./y.rs prepare
+
+ - name: Build
+ run: ./y.rs build --sysroot none
+
++ - name: Test abi-cafe
++ env:
++ TARGET_TRIPLE: ${{ matrix.env.TARGET_TRIPLE }}
++ run: ./y.rs abi-cafe
--- /dev/null
- - name: Cache cargo installed crates
- uses: actions/cache@v3
- with:
- path: ~/.cargo/bin
- key: ${{ runner.os }}-cargo-installed-crates
-
- - name: Cache cargo registry and index
- uses: actions/cache@v3
- with:
- path: |
- ~/.cargo/registry
- ~/.cargo/git
- key: ${{ runner.os }}-cargo-registry-and-index-${{ hashFiles('**/Cargo.lock') }}
-
+name: Various rustc tests
+
+on:
+ - push
+
+jobs:
+ bootstrap_rustc:
+ runs-on: ubuntu-latest
+
+ steps:
+ - uses: actions/checkout@v3
+
- - name: Prepare dependencies
+ - name: Cache cargo target dir
+ uses: actions/cache@v3
+ with:
+ path: build/cg_clif
+ key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('rust-toolchain', '**/Cargo.lock') }}
+
- git config --global user.email "user@example.com"
- git config --global user.name "User"
- ./y.rs prepare
++ - name: Use sparse cargo registry
+ run: |
- - name: Test
- run: |
- # Enable backtraces for easier debugging
- export RUST_BACKTRACE=1
++ cat >> ~/.cargo/config.toml <<EOF
++ [unstable]
++ sparse-registry = true
++ EOF
+
- ./scripts/test_bootstrap.sh
++ - name: Prepare dependencies
++ run: ./y.rs prepare
+
- - name: Cache cargo installed crates
- uses: actions/cache@v3
- with:
- path: ~/.cargo/bin
- key: ${{ runner.os }}-cargo-installed-crates
-
- - name: Cache cargo registry and index
- uses: actions/cache@v3
- with:
- path: |
- ~/.cargo/registry
- ~/.cargo/git
- key: ${{ runner.os }}-cargo-registry-and-index-${{ hashFiles('**/Cargo.lock') }}
-
++ - name: Test
++ run: ./scripts/test_bootstrap.sh
+ rustc_test_suite:
+ runs-on: ubuntu-latest
+
+ steps:
+ - uses: actions/checkout@v3
+
- - name: Prepare dependencies
+ - name: Cache cargo target dir
+ uses: actions/cache@v3
+ with:
+ path: build/cg_clif
+ key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('rust-toolchain', '**/Cargo.lock') }}
+
- git config --global user.email "user@example.com"
- git config --global user.name "User"
- ./y.rs prepare
++ - name: Use sparse cargo registry
+ run: |
- - name: Test
- run: |
- # Enable backtraces for easier debugging
- export RUST_BACKTRACE=1
++ cat >> ~/.cargo/config.toml <<EOF
++ [unstable]
++ sparse-registry = true
++ EOF
+
- ./scripts/test_rustc_tests.sh
++ - name: Prepare dependencies
++ run: ./y.rs prepare
+
++ - name: Test
++ run: ./scripts/test_rustc_tests.sh
--- /dev/null
- target
++/target
+**/*.rs.bk
+*.rlib
+*.o
+perf.data
+perf.data.old
+*.events
+*.string*
+/y.bin
+/y.bin.dSYM
+/y.exe
+/y.pdb
+/build
- /build_sysroot/sysroot_src
- /build_sysroot/compiler-builtins
- /build_sysroot/rustc_version
+/dist
+/rust
+/download
--- /dev/null
- "sysroot_src": "./build_sysroot/sysroot_src/library",
+{
++ "editor.formatOnSave": true,
++
+ // source for rustc_* is not included in the rust-src component; disable the errors about this
+ "rust-analyzer.diagnostics.disabled": ["unresolved-extern-crate", "unresolved-macro-call"],
+ "rust-analyzer.imports.granularity.enforce": true,
+ "rust-analyzer.imports.granularity.group": "module",
+ "rust-analyzer.imports.prefix": "crate",
+ "rust-analyzer.cargo.features": ["unstable-features", "__check_build_system_using_ra"],
+ "rust-analyzer.linkedProjects": [
+ "./Cargo.toml",
+ {
+ "crates": [
+ {
+ "root_module": "./example/mini_core.rs",
+ "edition": "2018",
+ "deps": [],
+ "cfg": [],
+ },
+ {
+ "root_module": "./example/mini_core_hello_world.rs",
+ "edition": "2018",
+ "deps": [{ "crate": 0, "name": "mini_core" }],
+ "cfg": [],
+ },
+ {
+ "root_module": "./example/mod_bench.rs",
+ "edition": "2018",
+ "deps": [],
+ "cfg": [],
+ },
+ ]
+ },
+ {
++ "sysroot_src": "./download/sysroot/sysroot_src/library",
+ "crates": [
+ {
+ "root_module": "./example/std_example.rs",
+ "edition": "2015",
+ "deps": [],
+ "cfg": [],
+ },
+ ]
+ }
+ ]
+}
--- /dev/null
- version = "0.90.1"
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 3
+
+[[package]]
+name = "ahash"
+version = "0.7.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47"
+dependencies = [
+ "getrandom",
+ "once_cell",
+ "version_check",
+]
+
+[[package]]
+name = "anyhow"
+version = "1.0.66"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "216261ddc8289130e551ddcd5ce8a064710c0d064a4d2895c67151c92b5443f6"
+
+[[package]]
+name = "arrayvec"
+version = "0.7.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6"
+
+[[package]]
+name = "autocfg"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
+
+[[package]]
+name = "bitflags"
+version = "1.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
+
+[[package]]
+name = "bumpalo"
+version = "3.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "572f695136211188308f16ad2ca5c851a712c464060ae6974944458eb83880ba"
+
+[[package]]
+name = "byteorder"
+version = "1.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
+
+[[package]]
+name = "cfg-if"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
+
+[[package]]
+name = "cranelift-bforest"
- checksum = "b62c772976416112fa4484cbd688cb6fb35fd430005c1c586224fc014018abad"
++version = "0.92.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
- version = "0.90.1"
++checksum = "2f3d54eab028f5805ae3b26fd60eca3f3a9cfb76b989d9bab173be3f61356cc3"
+dependencies = [
+ "cranelift-entity",
+]
+
+[[package]]
+name = "cranelift-codegen"
- checksum = "9b40ed2dd13c2ac7e24f88a3090c68ad3414eb1d066a95f8f1f7b3b819cb4e46"
++version = "0.92.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
- "cranelift-egraph",
++checksum = "2be1d5f2c3cca1efb691844bc1988b89c77291f13f778499a3f3c0cf49c0ed61"
+dependencies = [
+ "arrayvec",
+ "bumpalo",
+ "cranelift-bforest",
+ "cranelift-codegen-meta",
+ "cranelift-codegen-shared",
- version = "0.90.1"
+ "cranelift-entity",
+ "cranelift-isle",
+ "gimli",
++ "hashbrown",
+ "log",
+ "regalloc2",
+ "smallvec",
+ "target-lexicon",
+]
+
+[[package]]
+name = "cranelift-codegen-meta"
- checksum = "bb927a8f1c27c34ee3759b6b0ffa528d2330405d5cc4511f0cab33fe2279f4b5"
++version = "0.92.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
- version = "0.90.1"
++checksum = "3f9b1b1089750ce4005893af7ee00bb08a2cf1c9779999c0f7164cbc8ad2e0d2"
+dependencies = [
+ "cranelift-codegen-shared",
+]
+
+[[package]]
+name = "cranelift-codegen-shared"
- checksum = "43dfa417b884a9ab488d95fd6b93b25e959321fe7bfd7a0a960ba5d7fb7ab927"
-
- [[package]]
- name = "cranelift-egraph"
- version = "0.90.1"
- source = "registry+https://github.com/rust-lang/crates.io-index"
- checksum = "e0a66b39785efd8513d2cca967ede56d6cc57c8d7986a595c7c47d0c78de8dce"
- dependencies = [
- "cranelift-entity",
- "fxhash",
- "hashbrown",
- "indexmap",
- "log",
- "smallvec",
- ]
++version = "0.92.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
- version = "0.90.1"
++checksum = "cc5fbaec51de47297fd7304986fd53c8c0030abbe69728a60d72e1c63559318d"
+
+[[package]]
+name = "cranelift-entity"
- checksum = "0637ffde963cb5d759bc4d454cfa364b6509e6c74cdaa21298add0ed9276f346"
++version = "0.92.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
- version = "0.90.1"
++checksum = "dab984c94593f876090fae92e984bdcc74d9b1acf740ab5f79036001c65cba13"
+
+[[package]]
+name = "cranelift-frontend"
- checksum = "fb72b8342685e850cb037350418f62cc4fc55d6c2eb9c7ca01b82f9f1a6f3d56"
++version = "0.92.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
- version = "0.90.1"
++checksum = "6e0cb3102d21a2fe5f3210af608748ddd0cd09825ac12d42dc56ed5ed8725fe0"
+dependencies = [
+ "cranelift-codegen",
+ "log",
+ "smallvec",
+ "target-lexicon",
+]
+
+[[package]]
+name = "cranelift-isle"
- checksum = "850579cb9e4b448f7c301f1e6e6cbad99abe3f1f1d878a4994cb66e33c6db8cd"
++version = "0.92.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
- version = "0.90.1"
++checksum = "72101dd1f441d629735143c41e00b3428f9267738176983ef588ff43382af0a0"
+
+[[package]]
+name = "cranelift-jit"
- checksum = "9add822ad66dcbe152b5ab57de10240a2df4505099f2f6c27159acb711890bd4"
++version = "0.92.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
- version = "0.90.1"
++checksum = "6557f8ce44d498777f2495aa58d9692a4a37d6f84aa445750d666cef770b6a5c"
+dependencies = [
+ "anyhow",
+ "cranelift-codegen",
+ "cranelift-entity",
+ "cranelift-module",
+ "cranelift-native",
+ "libc",
+ "log",
+ "region",
+ "target-lexicon",
+ "wasmtime-jit-icache-coherence",
+ "windows-sys",
+]
+
+[[package]]
+name = "cranelift-module"
- checksum = "406b772626fc2664864cf947f3895a23b619895c7fff635f3622e2d857f4492f"
++version = "0.92.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
- version = "0.90.1"
++checksum = "88807e1c0c47ec02fe433333ccbe56b480425418b1470e333205e11650697d72"
+dependencies = [
+ "anyhow",
+ "cranelift-codegen",
+]
+
+[[package]]
+name = "cranelift-native"
- checksum = "2d0a279e5bcba3e0466c734d8d8eb6bfc1ad29e95c37f3e4955b492b5616335e"
++version = "0.92.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
- version = "0.90.1"
++checksum = "c22b0d9fcbe3fc5a1af9e7021b44ce42b930bcefac446ce22e02e8f9a0d67120"
+dependencies = [
+ "cranelift-codegen",
+ "libc",
+ "target-lexicon",
+]
+
+[[package]]
+name = "cranelift-object"
- checksum = "39793c550f0c1d7db96c2fc1324583670c8143befe6edbfbaf1c68aba53be983"
++version = "0.92.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
- version = "0.4.2"
++checksum = "341375758d7c3fedc0b5315f552e6f0feac46baf87c450a15e9455ef47c2b261"
+dependencies = [
+ "anyhow",
+ "cranelift-codegen",
+ "cranelift-module",
+ "log",
+ "object",
+ "target-lexicon",
+]
+
+[[package]]
+name = "crc32fast"
+version = "1.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d"
+dependencies = [
+ "cfg-if",
+]
+
+[[package]]
+name = "fallible-iterator"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7"
+
+[[package]]
+name = "fxhash"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c"
+dependencies = [
+ "byteorder",
+]
+
+[[package]]
+name = "getrandom"
+version = "0.2.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "wasi",
+]
+
+[[package]]
+name = "gimli"
+version = "0.26.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "22030e2c5a68ec659fde1e949a745124b48e6fa8b045b7ed5bd1fe4ccc5c4e5d"
+dependencies = [
+ "fallible-iterator",
+ "indexmap",
+ "stable_deref_trait",
+]
+
+[[package]]
+name = "hashbrown"
+version = "0.12.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
+dependencies = [
+ "ahash",
+]
+
+[[package]]
+name = "indexmap"
+version = "1.9.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1885e79c1fc4b10f0e172c475f458b7f7b93061064d98c3293e98c5ba0c8b399"
+dependencies = [
+ "autocfg",
+ "hashbrown",
+]
+
+[[package]]
+name = "libc"
+version = "0.2.138"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "db6d7e329c562c5dfab7a46a2afabc8b987ab9a4834c9d1ca04dc54c1546cef8"
+
+[[package]]
+name = "libloading"
+version = "0.7.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f"
+dependencies = [
+ "cfg-if",
+ "winapi",
+]
+
+[[package]]
+name = "log"
+version = "0.4.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e"
+dependencies = [
+ "cfg-if",
+]
+
+[[package]]
+name = "mach"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b823e83b2affd8f40a9ee8c29dbc56404c1e34cd2710921f2801e2cf29527afa"
+dependencies = [
+ "libc",
+]
+
+[[package]]
+name = "memchr"
+version = "2.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
+
+[[package]]
+name = "object"
+version = "0.29.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "21158b2c33aa6d4561f1c0a6ea283ca92bc54802a93b263e910746d679a7eb53"
+dependencies = [
+ "crc32fast",
+ "hashbrown",
+ "indexmap",
+ "memchr",
+]
+
+[[package]]
+name = "once_cell"
+version = "1.16.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860"
+
+[[package]]
+name = "regalloc2"
- checksum = "91b2eab54204ea0117fe9a060537e0b07a4e72f7c7d182361ecc346cab2240e5"
++version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
- version = "2.0.1"
++checksum = "300d4fbfb40c1c66a78ba3ddd41c1110247cf52f97b87d0f2fc9209bd49b030c"
+dependencies = [
+ "fxhash",
+ "log",
+ "slice-group-by",
+ "smallvec",
+]
+
+[[package]]
+name = "region"
+version = "2.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "877e54ea2adcd70d80e9179344c97f93ef0dffd6b03e1f4529e6e83ab2fa9ae0"
+dependencies = [
+ "bitflags",
+ "libc",
+ "mach",
+ "winapi",
+]
+
+[[package]]
+name = "rustc_codegen_cranelift"
+version = "0.1.0"
+dependencies = [
+ "cranelift-codegen",
+ "cranelift-frontend",
+ "cranelift-jit",
+ "cranelift-module",
+ "cranelift-native",
+ "cranelift-object",
+ "gimli",
+ "indexmap",
+ "libloading",
+ "object",
+ "once_cell",
+ "smallvec",
+ "target-lexicon",
+]
+
+[[package]]
+name = "slice-group-by"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "03b634d87b960ab1a38c4fe143b508576f075e7c978bfad18217645ebfdfa2ec"
+
+[[package]]
+name = "smallvec"
+version = "1.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0"
+
+[[package]]
+name = "stable_deref_trait"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3"
+
+[[package]]
+name = "target-lexicon"
+version = "0.12.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9410d0f6853b1d94f0e519fb95df60f29d2c1eff2d921ffdf01a4c8a3b54f12d"
+
+[[package]]
+name = "version_check"
+version = "0.9.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
+
+[[package]]
+name = "wasi"
+version = "0.11.0+wasi-snapshot-preview1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
+
+[[package]]
+name = "wasmtime-jit-icache-coherence"
- checksum = "e6bbabb309c06cc238ee91b1455b748c45f0bdcab0dda2c2db85b0a1e69fcb66"
++version = "5.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
- version = "0.36.1"
++checksum = "08fcba5ebd96da2a9f0747ab6337fe9788adfb3f63fa2c180520d665562d257e"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "windows-sys",
+]
+
+[[package]]
+name = "winapi"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
+dependencies = [
+ "winapi-i686-pc-windows-gnu",
+ "winapi-x86_64-pc-windows-gnu",
+]
+
+[[package]]
+name = "winapi-i686-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
+
+[[package]]
+name = "winapi-x86_64-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
+
+[[package]]
+name = "windows-sys"
- checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2"
++version = "0.42.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
- version = "0.36.1"
++checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7"
+dependencies = [
++ "windows_aarch64_gnullvm",
+ "windows_aarch64_msvc",
+ "windows_i686_gnu",
+ "windows_i686_msvc",
+ "windows_x86_64_gnu",
++ "windows_x86_64_gnullvm",
+ "windows_x86_64_msvc",
+]
+
++[[package]]
++name = "windows_aarch64_gnullvm"
++version = "0.42.0"
++source = "registry+https://github.com/rust-lang/crates.io-index"
++checksum = "41d2aa71f6f0cbe00ae5167d90ef3cfe66527d6f613ca78ac8024c3ccab9a19e"
++
+[[package]]
+name = "windows_aarch64_msvc"
- checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47"
++version = "0.42.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
- version = "0.36.1"
++checksum = "dd0f252f5a35cac83d6311b2e795981f5ee6e67eb1f9a7f64eb4500fbc4dcdb4"
+
+[[package]]
+name = "windows_i686_gnu"
- checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6"
++version = "0.42.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
- version = "0.36.1"
++checksum = "fbeae19f6716841636c28d695375df17562ca208b2b7d0dc47635a50ae6c5de7"
+
+[[package]]
+name = "windows_i686_msvc"
- checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024"
++version = "0.42.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
- version = "0.36.1"
++checksum = "84c12f65daa39dd2babe6e442988fc329d6243fdce47d7d2d155b8d874862246"
+
+[[package]]
+name = "windows_x86_64_gnu"
- checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1"
++version = "0.42.0"
++source = "registry+https://github.com/rust-lang/crates.io-index"
++checksum = "bf7b1b21b5362cbc318f686150e5bcea75ecedc74dd157d874d754a2ca44b0ed"
++
++[[package]]
++name = "windows_x86_64_gnullvm"
++version = "0.42.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
- version = "0.36.1"
++checksum = "09d525d2ba30eeb3297665bd434a54297e4170c7f1a44cad4ef58095b4cd2028"
+
+[[package]]
+name = "windows_x86_64_msvc"
- checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680"
++version = "0.42.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
++checksum = "f40009d85759725a34da6d89a94e63d7bdc50a862acf0dbc7c8e488f1edcb6f5"
--- /dev/null
- cranelift-codegen = { version = "0.90.1", features = ["unwind", "all-arch"] }
- cranelift-frontend = "0.90.1"
- cranelift-module = "0.90.1"
- cranelift-native = "0.90.1"
- cranelift-jit = { version = "0.90.1", optional = true }
- cranelift-object = "0.90.1"
+[package]
+name = "rustc_codegen_cranelift"
+version = "0.1.0"
+edition = "2021"
+
+[[bin]]
+# This is used just to teach rust-analyzer how to check the build system. required-features is used
+# to disable it for regular builds.
+name = "y"
+path = "./y.rs"
+required-features = ["__check_build_system_using_ra"]
+
+[lib]
+crate-type = ["dylib"]
+
+[dependencies]
+# These have to be in sync with each other
++cranelift-codegen = { version = "0.92", features = ["unwind", "all-arch"] }
++cranelift-frontend = { version = "0.92" }
++cranelift-module = { version = "0.92" }
++cranelift-native = { version = "0.92" }
++cranelift-jit = { version = "0.92", optional = true }
++cranelift-object = { version = "0.92" }
+target-lexicon = "0.12.0"
+gimli = { version = "0.26.0", default-features = false, features = ["write"]}
+object = { version = "0.29.0", default-features = false, features = ["std", "read_core", "write", "archive", "coff", "elf", "macho", "pe"] }
+
+indexmap = "1.9.1"
+libloading = { version = "0.7.3", optional = true }
+once_cell = "1.10.0"
+smallvec = "1.8.1"
+
+[patch.crates-io]
+# Uncomment to use local checkout of cranelift
+#cranelift-codegen = { path = "../wasmtime/cranelift/codegen" }
+#cranelift-frontend = { path = "../wasmtime/cranelift/frontend" }
+#cranelift-module = { path = "../wasmtime/cranelift/module" }
+#cranelift-native = { path = "../wasmtime/cranelift/native" }
+#cranelift-jit = { path = "../wasmtime/cranelift/jit" }
+#cranelift-object = { path = "../wasmtime/cranelift/object" }
+
+#gimli = { path = "../" }
+
+[features]
+# Enable features not ready to be enabled when compiling as part of rustc
+unstable-features = ["jit", "inline_asm"]
+jit = ["cranelift-jit", "libloading"]
+inline_asm = []
+__check_build_system_using_ra = []
+
+[package.metadata.rust-analyzer]
+rustc_private = true
--- /dev/null
- $ git clone https://github.com/bjorn3/rustc_codegen_cranelift.git
+# Cranelift codegen backend for rust
+
+The goal of this project is to create an alternative codegen backend for the rust compiler based on [Cranelift](https://github.com/bytecodealliance/wasmtime/blob/main/cranelift).
+This has the potential to improve compilation times in debug mode.
+If your project doesn't use any of the things listed under "Not yet supported", it should work fine.
+If not please open an issue.
+
+## Building and testing
+
+```bash
- $ ./y.rs prepare # download and patch sysroot src and install hyperfine for benchmarking
++$ git clone https://github.com/bjorn3/rustc_codegen_cranelift
+$ cd rustc_codegen_cranelift
- This will implicitly build cg_clif too. Both `y.rs build` and `test.sh` accept a `--debug` argument to
- build in debug mode.
++$ ./y.rs prepare
+$ ./y.rs build
+```
+
+To run the test suite replace the last command with:
+
+```bash
+$ ./test.sh
+```
+
- Alternatively you can download a pre built version from [GHA]. It is listed in the artifacts section
++For more docs on how to build and test see [build_system/usage.txt](build_system/usage.txt) or the help message of `./y.rs`.
+
- [GHA]: https://github.com/bjorn3/rustc_codegen_cranelift/actions?query=branch%3Amaster+event%3Apush+is%3Asuccess
++Alternatively you can download a pre built version from [Github Actions]. It is listed in the artifacts section
+of workflow runs. Unfortunately due to GHA restrictions you need to be logged in to access it.
+
- * SIMD ([tracked here](https://github.com/bjorn3/rustc_codegen_cranelift/issues/171), some basic things work)
++[Github Actions]: https://github.com/bjorn3/rustc_codegen_cranelift/actions?query=branch%3Amaster+event%3Apush+is%3Asuccess
+
+## Usage
+
+rustc_codegen_cranelift can be used as a near-drop-in replacement for `cargo build` or `cargo run` for existing projects.
+
+Assuming `$cg_clif_dir` is the directory you cloned this repo into and you followed the instructions (`y.rs prepare` and `y.rs build` or `test.sh`).
+
+In the directory with your project (where you can do the usual `cargo build`), run:
+
+```bash
+$ $cg_clif_dir/dist/cargo-clif build
+```
+
+This will build your project with rustc_codegen_cranelift instead of the usual LLVM backend.
+
+For additional ways to use rustc_codegen_cranelift like the JIT mode see [usage.md](docs/usage.md).
+
+## Configuration
+
+See the documentation on the `BackendConfig` struct in [config.rs](src/config.rs) for all
+configuration options.
+
+## Not yet supported
+
+* Inline assembly ([no cranelift support](https://github.com/bytecodealliance/wasmtime/issues/1041))
+ * On UNIX there is support for invoking an external assembler for `global_asm!` and `asm!`.
++* SIMD ([tracked here](https://github.com/bjorn3/rustc_codegen_cranelift/issues/171), `std::simd` fully works, `std::arch` is partially supported)
++* Unwinding on panics ([no cranelift support](https://github.com/bytecodealliance/wasmtime/issues/1677), `-Cpanic=abort` is enabled by default)
+
+## License
+
+Licensed under either of
+
+ * Apache License, Version 2.0 ([LICENSE-APACHE](LICENSE-APACHE) or
+ http://www.apache.org/licenses/LICENSE-2.0)
+ * MIT license ([LICENSE-MIT](LICENSE-MIT) or
+ http://opensource.org/licenses/MIT)
+
+at your option.
+
+### Contribution
+
+Unless you explicitly state otherwise, any contribution intentionally submitted
+for inclusion in the work by you shall be dual licensed as above, without any
+additional terms or conditions.
--- /dev/null
- version = "1.0.77"
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 3
+
+[[package]]
+name = "addr2line"
+version = "0.17.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b9ecd88a8c8378ca913a680cd98f0f13ac67383d35993f86c90a70e3f137816b"
+dependencies = [
+ "compiler_builtins",
+ "gimli",
+ "rustc-std-workspace-alloc",
+ "rustc-std-workspace-core",
+]
+
+[[package]]
+name = "adler"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
+dependencies = [
+ "compiler_builtins",
+ "rustc-std-workspace-core",
+]
+
+[[package]]
+name = "alloc"
+version = "0.0.0"
+dependencies = [
+ "compiler_builtins",
+ "core",
+]
+
+[[package]]
+name = "cc"
- checksum = "e9f73505338f7d905b19d18738976aae232eb46b8efc15554ffc56deb5d9ebe4"
++version = "1.0.78"
+source = "registry+https://github.com/rust-lang/crates.io-index"
- version = "0.1.85"
++checksum = "a20104e2335ce8a659d6dd92a51a767a0c062599c73b343fd152cb401e828c3d"
+
+[[package]]
+name = "cfg-if"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
+dependencies = [
+ "compiler_builtins",
+ "rustc-std-workspace-core",
+]
+
+[[package]]
+name = "compiler_builtins"
- checksum = "13e81c6cd7ab79f51a0c927d22858d61ad12bd0b3865f0b13ece02a4486aeabb"
++version = "0.1.86"
+source = "registry+https://github.com/rust-lang/crates.io-index"
- version = "0.2.138"
++checksum = "5dae98c88e576098d7ab13ebcb40cc43e5114b2beafe61a87cda9200649ff205"
+dependencies = [
+ "rustc-std-workspace-core",
+]
+
+[[package]]
+name = "core"
+version = "0.0.0"
+
+[[package]]
+name = "dlmalloc"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "203540e710bfadb90e5e29930baf5d10270cec1f43ab34f46f78b147b2de715a"
+dependencies = [
+ "compiler_builtins",
+ "libc",
+ "rustc-std-workspace-core",
+]
+
+[[package]]
+name = "fortanix-sgx-abi"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "57cafc2274c10fab234f176b25903ce17e690fca7597090d50880e047a0389c5"
+dependencies = [
+ "compiler_builtins",
+ "rustc-std-workspace-core",
+]
+
+[[package]]
+name = "getopts"
+version = "0.2.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "14dbbfd5c71d70241ecf9e6f13737f7b5ce823821063188d7e46c41d371eebd5"
+dependencies = [
+ "rustc-std-workspace-core",
+ "rustc-std-workspace-std",
+ "unicode-width",
+]
+
+[[package]]
+name = "gimli"
+version = "0.26.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "22030e2c5a68ec659fde1e949a745124b48e6fa8b045b7ed5bd1fe4ccc5c4e5d"
+dependencies = [
+ "compiler_builtins",
+ "rustc-std-workspace-alloc",
+ "rustc-std-workspace-core",
+]
+
+[[package]]
+name = "hashbrown"
+version = "0.12.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
+dependencies = [
+ "compiler_builtins",
+ "rustc-std-workspace-alloc",
+ "rustc-std-workspace-core",
+]
+
+[[package]]
+name = "hermit-abi"
+version = "0.2.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7"
+dependencies = [
+ "compiler_builtins",
+ "libc",
+ "rustc-std-workspace-alloc",
+ "rustc-std-workspace-core",
+]
+
+[[package]]
+name = "libc"
- checksum = "db6d7e329c562c5dfab7a46a2afabc8b987ab9a4834c9d1ca04dc54c1546cef8"
++version = "0.2.139"
+source = "registry+https://github.com/rust-lang/crates.io-index"
++checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79"
+dependencies = [
+ "rustc-std-workspace-core",
+]
+
+[[package]]
+name = "memchr"
+version = "2.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
+dependencies = [
+ "compiler_builtins",
+ "rustc-std-workspace-core",
+]
+
+[[package]]
+name = "miniz_oxide"
+version = "0.5.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "96590ba8f175222643a85693f33d26e9c8a015f599c216509b1a6894af675d34"
+dependencies = [
+ "adler",
+ "compiler_builtins",
+ "rustc-std-workspace-alloc",
+ "rustc-std-workspace-core",
+]
+
+[[package]]
+name = "object"
+version = "0.29.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "21158b2c33aa6d4561f1c0a6ea283ca92bc54802a93b263e910746d679a7eb53"
+dependencies = [
+ "compiler_builtins",
+ "memchr",
+ "rustc-std-workspace-alloc",
+ "rustc-std-workspace-core",
+]
+
+[[package]]
+name = "panic_abort"
+version = "0.0.0"
+dependencies = [
+ "alloc",
+ "cfg-if",
+ "compiler_builtins",
+ "core",
+ "libc",
+]
+
+[[package]]
+name = "panic_unwind"
+version = "0.0.0"
+dependencies = [
+ "alloc",
+ "cfg-if",
+ "compiler_builtins",
+ "core",
+ "libc",
+ "unwind",
+]
+
+[[package]]
+name = "proc_macro"
+version = "0.0.0"
+dependencies = [
+ "core",
+ "std",
+]
+
+[[package]]
+name = "rustc-demangle"
+version = "0.1.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7ef03e0a2b150c7a90d01faf6254c9c48a41e95fb2a8c2ac1c6f0d2b9aefc342"
+dependencies = [
+ "compiler_builtins",
+ "rustc-std-workspace-core",
+]
+
+[[package]]
+name = "rustc-std-workspace-alloc"
+version = "1.99.0"
+dependencies = [
+ "alloc",
+]
+
+[[package]]
+name = "rustc-std-workspace-core"
+version = "1.99.0"
+dependencies = [
+ "core",
+]
+
+[[package]]
+name = "rustc-std-workspace-std"
+version = "1.99.0"
+dependencies = [
+ "std",
+]
+
+[[package]]
+name = "std"
+version = "0.0.0"
+dependencies = [
+ "addr2line",
+ "alloc",
+ "cfg-if",
+ "compiler_builtins",
+ "core",
+ "dlmalloc",
+ "fortanix-sgx-abi",
+ "hashbrown",
+ "hermit-abi",
+ "libc",
+ "miniz_oxide",
+ "object",
+ "panic_abort",
+ "panic_unwind",
+ "rustc-demangle",
+ "std_detect",
+ "unwind",
+ "wasi",
+]
+
+[[package]]
+name = "std_detect"
+version = "0.1.5"
+dependencies = [
+ "cfg-if",
+ "compiler_builtins",
+ "libc",
+ "rustc-std-workspace-alloc",
+ "rustc-std-workspace-core",
+]
+
+[[package]]
+name = "sysroot"
+version = "0.0.0"
+dependencies = [
+ "alloc",
+ "compiler_builtins",
+ "core",
+ "std",
+ "test",
+]
+
+[[package]]
+name = "test"
+version = "0.0.0"
+dependencies = [
+ "cfg-if",
+ "core",
+ "getopts",
+ "libc",
+ "panic_abort",
+ "panic_unwind",
+ "proc_macro",
+ "std",
+]
+
+[[package]]
+name = "unicode-width"
+version = "0.1.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b"
+dependencies = [
+ "compiler_builtins",
+ "rustc-std-workspace-core",
+ "rustc-std-workspace-std",
+]
+
+[[package]]
+name = "unwind"
+version = "0.0.0"
+dependencies = [
+ "cc",
+ "cfg-if",
+ "compiler_builtins",
+ "core",
+ "libc",
+]
+
+[[package]]
+name = "wasi"
+version = "0.11.0+wasi-snapshot-preview1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
+dependencies = [
+ "compiler_builtins",
+ "rustc-std-workspace-alloc",
+ "rustc-std-workspace-core",
+]
--- /dev/null
- use super::config;
+use std::path::Path;
+
+use super::build_sysroot;
- static ABI_CAFE: CargoProject = CargoProject::new(&ABI_CAFE_REPO.source_dir(), "abi_cafe");
+use super::path::Dirs;
+use super::prepare::GitRepo;
+use super::utils::{spawn_and_wait, CargoProject, Compiler};
+use super::SysrootKind;
+
+pub(crate) static ABI_CAFE_REPO: GitRepo =
+ GitRepo::github("Gankra", "abi-cafe", "4c6dc8c9c687e2b3a760ff2176ce236872b37212", "abi-cafe");
+
- host_triple: &str,
- target_triple: &str,
++pub(crate) static ABI_CAFE: CargoProject =
++ CargoProject::new(&ABI_CAFE_REPO.source_dir(), "abi_cafe");
+
+pub(crate) fn run(
+ channel: &str,
+ sysroot_kind: SysrootKind,
+ dirs: &Dirs,
+ cg_clif_dylib: &Path,
- if !config::get_bool("testsuite.abi-cafe") {
- eprintln!("[SKIP] abi-cafe");
- return;
- }
-
- if host_triple != target_triple {
- eprintln!("[SKIP] abi-cafe (cross-compilation not supported)");
- return;
- }
-
++ bootstrap_host_compiler: &Compiler,
+) {
- host_triple,
- target_triple,
+ eprintln!("Building sysroot for abi-cafe");
+ build_sysroot::build_sysroot(
+ dirs,
+ channel,
+ sysroot_kind,
+ cg_clif_dylib,
- let mut cmd = ABI_CAFE.run(&Compiler::host(), dirs);
++ bootstrap_host_compiler,
++ bootstrap_host_compiler.triple.clone(),
+ );
+
+ eprintln!("Running abi-cafe");
+
+ let pairs = ["rustc_calls_cgclif", "cgclif_calls_rustc", "cgclif_calls_cc", "cc_calls_cgclif"];
+
++ let mut cmd = ABI_CAFE.run(bootstrap_host_compiler, dirs);
+ cmd.arg("--");
+ cmd.arg("--pairs");
+ cmd.args(pairs);
+ cmd.arg("--add-rustc-codegen-backend");
+ cmd.arg(format!("cgclif:{}", cg_clif_dylib.display()));
+ cmd.current_dir(ABI_CAFE.source_dir(dirs));
+
+ spawn_and_wait(cmd);
+}
--- /dev/null
--- /dev/null
++use std::env;
++use std::fs;
++use std::path::Path;
++
++use super::path::{Dirs, RelPath};
++use super::prepare::GitRepo;
++use super::rustc_info::get_file_name;
++use super::utils::{hyperfine_command, is_ci, spawn_and_wait, CargoProject, Compiler};
++
++pub(crate) static SIMPLE_RAYTRACER_REPO: GitRepo = GitRepo::github(
++ "ebobby",
++ "simple-raytracer",
++ "804a7a21b9e673a482797aa289a18ed480e4d813",
++ "<none>",
++);
++
++// Use a separate target dir for the initial LLVM build to reduce unnecessary recompiles
++pub(crate) static SIMPLE_RAYTRACER_LLVM: CargoProject =
++ CargoProject::new(&SIMPLE_RAYTRACER_REPO.source_dir(), "simple_raytracer_llvm");
++
++pub(crate) static SIMPLE_RAYTRACER: CargoProject =
++ CargoProject::new(&SIMPLE_RAYTRACER_REPO.source_dir(), "simple_raytracer");
++
++pub(crate) fn benchmark(dirs: &Dirs, bootstrap_host_compiler: &Compiler) {
++ benchmark_simple_raytracer(dirs, bootstrap_host_compiler);
++}
++
++fn benchmark_simple_raytracer(dirs: &Dirs, bootstrap_host_compiler: &Compiler) {
++ if std::process::Command::new("hyperfine").output().is_err() {
++ eprintln!("Hyperfine not installed");
++ eprintln!("Hint: Try `cargo install hyperfine` to install hyperfine");
++ std::process::exit(1);
++ }
++
++ eprintln!("[LLVM BUILD] simple-raytracer");
++ let build_cmd = SIMPLE_RAYTRACER_LLVM.build(bootstrap_host_compiler, dirs);
++ spawn_and_wait(build_cmd);
++ fs::copy(
++ SIMPLE_RAYTRACER_LLVM
++ .target_dir(dirs)
++ .join(&bootstrap_host_compiler.triple)
++ .join("debug")
++ .join(get_file_name("main", "bin")),
++ RelPath::BUILD.to_path(dirs).join(get_file_name("raytracer_cg_llvm", "bin")),
++ )
++ .unwrap();
++
++ let run_runs = env::var("RUN_RUNS")
++ .unwrap_or(if is_ci() { "2" } else { "10" }.to_string())
++ .parse()
++ .unwrap();
++
++ eprintln!("[BENCH COMPILE] ebobby/simple-raytracer");
++ let cargo_clif =
++ RelPath::DIST.to_path(dirs).join(get_file_name("cargo_clif", "bin").replace('_', "-"));
++ let manifest_path = SIMPLE_RAYTRACER.manifest_path(dirs);
++ let target_dir = SIMPLE_RAYTRACER.target_dir(dirs);
++
++ let clean_cmd = format!(
++ "cargo clean --manifest-path {manifest_path} --target-dir {target_dir}",
++ manifest_path = manifest_path.display(),
++ target_dir = target_dir.display(),
++ );
++ let llvm_build_cmd = format!(
++ "cargo build --manifest-path {manifest_path} --target-dir {target_dir}",
++ manifest_path = manifest_path.display(),
++ target_dir = target_dir.display(),
++ );
++ let clif_build_cmd = format!(
++ "{cargo_clif} build --manifest-path {manifest_path} --target-dir {target_dir}",
++ cargo_clif = cargo_clif.display(),
++ manifest_path = manifest_path.display(),
++ target_dir = target_dir.display(),
++ );
++
++ let bench_compile =
++ hyperfine_command(1, run_runs, Some(&clean_cmd), &llvm_build_cmd, &clif_build_cmd);
++
++ spawn_and_wait(bench_compile);
++
++ eprintln!("[BENCH RUN] ebobby/simple-raytracer");
++ fs::copy(
++ target_dir.join("debug").join(get_file_name("main", "bin")),
++ RelPath::BUILD.to_path(dirs).join(get_file_name("raytracer_cg_clif", "bin")),
++ )
++ .unwrap();
++
++ let mut bench_run = hyperfine_command(
++ 0,
++ run_runs,
++ None,
++ Path::new(".").join(get_file_name("raytracer_cg_llvm", "bin")).to_str().unwrap(),
++ Path::new(".").join(get_file_name("raytracer_cg_clif", "bin")).to_str().unwrap(),
++ );
++ bench_run.current_dir(RelPath::BUILD.to_path(dirs));
++ spawn_and_wait(bench_run);
++}
--- /dev/null
- static CG_CLIF: CargoProject = CargoProject::new(&RelPath::SOURCE, "cg_clif");
+use std::env;
+use std::path::PathBuf;
+
+use super::path::{Dirs, RelPath};
+use super::rustc_info::get_file_name;
+use super::utils::{is_ci, CargoProject, Compiler};
+
- host_triple: &str,
++pub(crate) static CG_CLIF: CargoProject = CargoProject::new(&RelPath::SOURCE, "cg_clif");
+
+pub(crate) fn build_backend(
+ dirs: &Dirs,
+ channel: &str,
- let mut cmd = CG_CLIF.build(&Compiler::host(), dirs);
++ bootstrap_host_compiler: &Compiler,
+ use_unstable_features: bool,
+) -> PathBuf {
- .join(host_triple)
++ let mut cmd = CG_CLIF.build(&bootstrap_host_compiler, dirs);
+
+ cmd.env("CARGO_BUILD_INCREMENTAL", "true"); // Force incr comp even in release mode
+
+ let mut rustflags = env::var("RUSTFLAGS").unwrap_or_default();
+
+ if is_ci() {
+ // Deny warnings on CI
+ rustflags += " -Dwarnings";
+
+ // Disabling incr comp reduces cache size and incr comp doesn't save as much on CI anyway
+ cmd.env("CARGO_BUILD_INCREMENTAL", "false");
++
++ cmd.env("CARGO_PROFILE_RELEASE_DEBUG_ASSERTIONS", "true");
+ }
+
+ if use_unstable_features {
+ cmd.arg("--features").arg("unstable-features");
+ }
+
+ match channel {
+ "debug" => {}
+ "release" => {
+ cmd.arg("--release");
+ }
+ _ => unreachable!(),
+ }
+
+ cmd.env("RUSTFLAGS", rustflags);
+
+ eprintln!("[BUILD] rustc_codegen_cranelift");
+ super::utils::spawn_and_wait(cmd);
+
+ CG_CLIF
+ .target_dir(dirs)
++ .join(&bootstrap_host_compiler.triple)
+ .join(channel)
+ .join(get_file_name("rustc_codegen_cranelift", "dylib"))
+}
--- /dev/null
- use std::path::Path;
+use std::fs;
- use super::rustc_info::{get_file_name, get_rustc_version, get_wrapper_file_name};
- use super::utils::{spawn_and_wait, try_hard_link, CargoProject, Compiler};
++use std::path::{Path, PathBuf};
+use std::process::{self, Command};
+
+use super::path::{Dirs, RelPath};
- static RUSTLIB_DIR: RelPath = LIB_DIR.join("rustlib");
++use super::rustc_info::{get_file_name, get_rustc_version, get_toolchain_name};
++use super::utils::{remove_dir_if_exists, spawn_and_wait, try_hard_link, CargoProject, Compiler};
+use super::SysrootKind;
+
+static DIST_DIR: RelPath = RelPath::DIST;
+static BIN_DIR: RelPath = RelPath::DIST.join("bin");
+static LIB_DIR: RelPath = RelPath::DIST.join("lib");
- host_triple: &str,
- target_triple: &str,
- ) {
+
+pub(crate) fn build_sysroot(
+ dirs: &Dirs,
+ channel: &str,
+ sysroot_kind: SysrootKind,
+ cg_clif_dylib_src: &Path,
- .join(get_file_name("rustc_codegen_cranelift", "dylib"));
++ bootstrap_host_compiler: &Compiler,
++ target_triple: String,
++) -> Compiler {
+ eprintln!("[BUILD] sysroot {:?}", sysroot_kind);
+
+ DIST_DIR.ensure_fresh(dirs);
+ BIN_DIR.ensure_exists(dirs);
+ LIB_DIR.ensure_exists(dirs);
+
++ let is_native = bootstrap_host_compiler.triple == target_triple;
++
+ // Copy the backend
+ let cg_clif_dylib_path = if cfg!(windows) {
+ // Windows doesn't have rpath support, so the cg_clif dylib needs to be next to the
+ // binaries.
+ BIN_DIR
+ } else {
+ LIB_DIR
+ }
+ .to_path(dirs)
- let wrapper_name = get_wrapper_file_name(wrapper, "bin");
++ .join(cg_clif_dylib_src.file_name().unwrap());
+ try_hard_link(cg_clif_dylib_src, &cg_clif_dylib_path);
+
+ // Build and copy rustc and cargo wrappers
++ let wrapper_base_name = get_file_name("____", "bin");
++ let toolchain_name = get_toolchain_name();
+ for wrapper in ["rustc-clif", "rustdoc-clif", "cargo-clif"] {
- let mut build_cargo_wrapper_cmd = Command::new("rustc");
++ let wrapper_name = wrapper_base_name.replace("____", wrapper);
+
- .arg("-g");
++ let mut build_cargo_wrapper_cmd = Command::new(&bootstrap_host_compiler.rustc);
+ build_cargo_wrapper_cmd
++ .env("TOOLCHAIN_NAME", toolchain_name.clone())
+ .arg(RelPath::SCRIPTS.to_path(dirs).join(&format!("{wrapper}.rs")))
+ .arg("-o")
+ .arg(DIST_DIR.to_path(dirs).join(wrapper_name))
- let default_sysroot = super::rustc_info::get_default_sysroot();
++ .arg("-Cstrip=debuginfo");
+ spawn_and_wait(build_cargo_wrapper_cmd);
+ }
+
- let host_rustlib_lib = RUSTLIB_DIR.to_path(dirs).join(host_triple).join("lib");
- let target_rustlib_lib = RUSTLIB_DIR.to_path(dirs).join(target_triple).join("lib");
- fs::create_dir_all(&host_rustlib_lib).unwrap();
- fs::create_dir_all(&target_rustlib_lib).unwrap();
++ let host = build_sysroot_for_triple(
++ dirs,
++ channel,
++ bootstrap_host_compiler.clone(),
++ &cg_clif_dylib_path,
++ sysroot_kind,
++ );
++ host.install_into_sysroot(&DIST_DIR.to_path(dirs));
+
- if target_triple == "x86_64-pc-windows-gnu" {
- if !default_sysroot.join("lib").join("rustlib").join(target_triple).join("lib").exists() {
- eprintln!(
- "The x86_64-pc-windows-gnu target needs to be installed first before it is possible \
- to compile a sysroot for it.",
- );
- process::exit(1);
++ if !is_native {
++ build_sysroot_for_triple(
++ dirs,
++ channel,
++ {
++ let mut bootstrap_target_compiler = bootstrap_host_compiler.clone();
++ bootstrap_target_compiler.triple = target_triple.clone();
++ bootstrap_target_compiler.set_cross_linker_and_runner();
++ bootstrap_target_compiler
++ },
++ &cg_clif_dylib_path,
++ sysroot_kind,
++ )
++ .install_into_sysroot(&DIST_DIR.to_path(dirs));
++ }
+
- for file in fs::read_dir(
- default_sysroot.join("lib").join("rustlib").join(target_triple).join("lib"),
- )
- .unwrap()
- {
- let file = file.unwrap().path();
- if file.extension().map_or(true, |ext| ext.to_str().unwrap() != "o") {
- continue; // only copy object files
- }
- try_hard_link(&file, target_rustlib_lib.join(file.file_name().unwrap()));
++ // Copy std for the host to the lib dir. This is necessary for the jit mode to find
++ // libstd.
++ for lib in host.libs {
++ let filename = lib.file_name().unwrap().to_str().unwrap();
++ if filename.contains("std-") && !filename.contains(".rlib") {
++ try_hard_link(&lib, LIB_DIR.to_path(dirs).join(lib.file_name().unwrap()));
+ }
- match sysroot_kind {
- SysrootKind::None => {} // Nothing to do
- SysrootKind::Llvm => {
- for file in fs::read_dir(
- default_sysroot.join("lib").join("rustlib").join(host_triple).join("lib"),
- )
- .unwrap()
- {
- let file = file.unwrap().path();
- let file_name_str = file.file_name().unwrap().to_str().unwrap();
- if (file_name_str.contains("rustc_")
- && !file_name_str.contains("rustc_std_workspace_")
- && !file_name_str.contains("rustc_demangle"))
- || file_name_str.contains("chalk")
- || file_name_str.contains("tracing")
- || file_name_str.contains("regex")
- {
- // These are large crates that are part of the rustc-dev component and are not
- // necessary to run regular programs.
- continue;
- }
- try_hard_link(&file, host_rustlib_lib.join(file.file_name().unwrap()));
- }
++ }
++
++ let mut target_compiler = {
++ let dirs: &Dirs = &dirs;
++ let rustc_clif =
++ RelPath::DIST.to_path(&dirs).join(wrapper_base_name.replace("____", "rustc-clif"));
++ let rustdoc_clif =
++ RelPath::DIST.to_path(&dirs).join(wrapper_base_name.replace("____", "rustdoc-clif"));
++
++ Compiler {
++ cargo: bootstrap_host_compiler.cargo.clone(),
++ rustc: rustc_clif.clone(),
++ rustdoc: rustdoc_clif.clone(),
++ rustflags: String::new(),
++ rustdocflags: String::new(),
++ triple: target_triple,
++ runner: vec![],
+ }
++ };
++ if !is_native {
++ target_compiler.set_cross_linker_and_runner();
+ }
++ target_compiler
++}
+
- if target_triple != host_triple {
- for file in fs::read_dir(
- default_sysroot.join("lib").join("rustlib").join(target_triple).join("lib"),
- )
- .unwrap()
- {
- let file = file.unwrap().path();
- try_hard_link(&file, target_rustlib_lib.join(file.file_name().unwrap()));
- }
- }
++struct SysrootTarget {
++ triple: String,
++ libs: Vec<PathBuf>,
++}
+
- SysrootKind::Clif => {
- build_clif_sysroot_for_triple(dirs, channel, host_triple, &cg_clif_dylib_path, None);
-
- if host_triple != target_triple {
- // When cross-compiling it is often necessary to manually pick the right linker
- let linker = match target_triple {
- "aarch64-unknown-linux-gnu" => Some("aarch64-linux-gnu-gcc"),
- "s390x-unknown-linux-gnu" => Some("s390x-linux-gnu-gcc"),
- _ => None,
- };
- build_clif_sysroot_for_triple(
- dirs,
- channel,
- target_triple,
- &cg_clif_dylib_path,
- linker,
- );
- }
++impl SysrootTarget {
++ fn install_into_sysroot(&self, sysroot: &Path) {
++ if self.libs.is_empty() {
++ return;
+ }
- // Copy std for the host to the lib dir. This is necessary for the jit mode to find
- // libstd.
- for file in fs::read_dir(host_rustlib_lib).unwrap() {
- let file = file.unwrap().path();
- let filename = file.file_name().unwrap().to_str().unwrap();
- if filename.contains("std-") && !filename.contains(".rlib") {
- try_hard_link(&file, LIB_DIR.to_path(dirs).join(file.file_name().unwrap()));
- }
- }
+
- // FIXME move to download/ or dist/
- pub(crate) static SYSROOT_RUSTC_VERSION: RelPath = RelPath::BUILD_SYSROOT.join("rustc_version");
- pub(crate) static SYSROOT_SRC: RelPath = RelPath::BUILD_SYSROOT.join("sysroot_src");
- static STANDARD_LIBRARY: CargoProject = CargoProject::new(&RelPath::BUILD_SYSROOT, "build_sysroot");
++ let target_rustlib_lib = sysroot.join("lib").join("rustlib").join(&self.triple).join("lib");
++ fs::create_dir_all(&target_rustlib_lib).unwrap();
++
++ for lib in &self.libs {
++ try_hard_link(lib, target_rustlib_lib.join(lib.file_name().unwrap()));
+ }
+ }
+}
+
- triple: &str,
++pub(crate) static ORIG_BUILD_SYSROOT: RelPath = RelPath::SOURCE.join("build_sysroot");
++pub(crate) static BUILD_SYSROOT: RelPath = RelPath::DOWNLOAD.join("sysroot");
++pub(crate) static SYSROOT_RUSTC_VERSION: RelPath = BUILD_SYSROOT.join("rustc_version");
++pub(crate) static SYSROOT_SRC: RelPath = BUILD_SYSROOT.join("sysroot_src");
++pub(crate) static STANDARD_LIBRARY: CargoProject =
++ CargoProject::new(&BUILD_SYSROOT, "build_sysroot");
++pub(crate) static RTSTARTUP_SYSROOT: RelPath = RelPath::BUILD.join("rtstartup");
+
++#[must_use]
++fn build_sysroot_for_triple(
++ dirs: &Dirs,
++ channel: &str,
++ compiler: Compiler,
++ cg_clif_dylib_path: &Path,
++ sysroot_kind: SysrootKind,
++) -> SysrootTarget {
++ match sysroot_kind {
++ SysrootKind::None => build_rtstartup(dirs, &compiler)
++ .unwrap_or(SysrootTarget { triple: compiler.triple, libs: vec![] }),
++ SysrootKind::Llvm => build_llvm_sysroot_for_triple(compiler),
++ SysrootKind::Clif => {
++ build_clif_sysroot_for_triple(dirs, channel, compiler, &cg_clif_dylib_path)
++ }
++ }
++}
++
++#[must_use]
++fn build_llvm_sysroot_for_triple(compiler: Compiler) -> SysrootTarget {
++ let default_sysroot = super::rustc_info::get_default_sysroot(&compiler.rustc);
++
++ let mut target_libs = SysrootTarget { triple: compiler.triple, libs: vec![] };
++
++ for entry in fs::read_dir(
++ default_sysroot.join("lib").join("rustlib").join(&target_libs.triple).join("lib"),
++ )
++ .unwrap()
++ {
++ let entry = entry.unwrap();
++ if entry.file_type().unwrap().is_dir() {
++ continue;
++ }
++ let file = entry.path();
++ let file_name_str = file.file_name().unwrap().to_str().unwrap();
++ if (file_name_str.contains("rustc_")
++ && !file_name_str.contains("rustc_std_workspace_")
++ && !file_name_str.contains("rustc_demangle"))
++ || file_name_str.contains("chalk")
++ || file_name_str.contains("tracing")
++ || file_name_str.contains("regex")
++ {
++ // These are large crates that are part of the rustc-dev component and are not
++ // necessary to run regular programs.
++ continue;
++ }
++ target_libs.libs.push(file);
++ }
++
++ target_libs
++}
++
++#[must_use]
+fn build_clif_sysroot_for_triple(
+ dirs: &Dirs,
+ channel: &str,
- linker: Option<&str>,
- ) {
++ mut compiler: Compiler,
+ cg_clif_dylib_path: &Path,
- let rustc_version = get_rustc_version();
++) -> SysrootTarget {
+ match fs::read_to_string(SYSROOT_RUSTC_VERSION.to_path(dirs)) {
+ Err(e) => {
+ eprintln!("Failed to get rustc version for patched sysroot source: {}", e);
+ eprintln!("Hint: Try `./y.rs prepare` to patch the sysroot source");
+ process::exit(1);
+ }
+ Ok(source_version) => {
- let build_dir = STANDARD_LIBRARY.target_dir(dirs).join(triple).join(channel);
++ let rustc_version = get_rustc_version(&compiler.rustc);
+ if source_version != rustc_version {
+ eprintln!("The patched sysroot source is outdated");
+ eprintln!("Source version: {}", source_version.trim());
+ eprintln!("Rustc version: {}", rustc_version.trim());
+ eprintln!("Hint: Try `./y.rs prepare` to update the patched sysroot source");
+ process::exit(1);
+ }
+ }
+ }
+
- if build_dir.join("deps").exists() {
- fs::remove_dir_all(build_dir.join("deps")).unwrap();
- }
++ let mut target_libs = SysrootTarget { triple: compiler.triple.clone(), libs: vec![] };
++
++ if let Some(rtstartup_target_libs) = build_rtstartup(dirs, &compiler) {
++ rtstartup_target_libs.install_into_sysroot(&RTSTARTUP_SYSROOT.to_path(dirs));
++
++ target_libs.libs.extend(rtstartup_target_libs.libs);
++ }
++
++ let build_dir = STANDARD_LIBRARY.target_dir(dirs).join(&compiler.triple).join(channel);
+
+ if !super::config::get_bool("keep_sysroot") {
+ // Cleanup the deps dir, but keep build scripts and the incremental cache for faster
+ // recompilation as they are not affected by changes in cg_clif.
- let mut rustflags = "-Zforce-unstable-if-unmarked -Cpanic=abort".to_string();
++ remove_dir_if_exists(&build_dir.join("deps"));
+ }
+
+ // Build sysroot
- rustflags.push_str(&format!(" --sysroot={}", DIST_DIR.to_path(dirs).to_str().unwrap()));
++ let mut rustflags = " -Zforce-unstable-if-unmarked -Cpanic=abort".to_string();
+ rustflags.push_str(&format!(" -Zcodegen-backend={}", cg_clif_dylib_path.to_str().unwrap()));
- if let Some(linker) = linker {
- use std::fmt::Write;
- write!(rustflags, " -Clinker={}", linker).unwrap();
- }
- let mut compiler = Compiler::with_triple(triple.to_owned());
- compiler.rustflags = rustflags;
++ // Necessary for MinGW to find rsbegin.o and rsend.o
++ rustflags
++ .push_str(&format!(" --sysroot={}", RTSTARTUP_SYSROOT.to_path(dirs).to_str().unwrap()));
+ if channel == "release" {
+ rustflags.push_str(" -Zmir-opt-level=3");
+ }
- // Copy all relevant files to the sysroot
++ compiler.rustflags += &rustflags;
+ let mut build_cmd = STANDARD_LIBRARY.build(&compiler, dirs);
+ if channel == "release" {
+ build_cmd.arg("--release");
+ }
+ build_cmd.env("__CARGO_DEFAULT_LIB_METADATA", "cg_clif");
+ spawn_and_wait(build_cmd);
+
- try_hard_link(
- entry.path(),
- RUSTLIB_DIR.to_path(dirs).join(triple).join("lib").join(entry.file_name()),
- );
+ for entry in fs::read_dir(build_dir.join("deps")).unwrap() {
+ let entry = entry.unwrap();
+ if let Some(ext) = entry.path().extension() {
+ if ext == "rmeta" || ext == "d" || ext == "dSYM" || ext == "clif" {
+ continue;
+ }
+ } else {
+ continue;
+ };
++ target_libs.libs.push(entry.path());
++ }
++
++ target_libs
++}
++
++fn build_rtstartup(dirs: &Dirs, compiler: &Compiler) -> Option<SysrootTarget> {
++ if !compiler.triple.ends_with("windows-gnu") {
++ return None;
+ }
++
++ RTSTARTUP_SYSROOT.ensure_fresh(dirs);
++
++ let rtstartup_src = SYSROOT_SRC.to_path(dirs).join("library").join("rtstartup");
++ let mut target_libs = SysrootTarget { triple: compiler.triple.clone(), libs: vec![] };
++
++ for file in ["rsbegin", "rsend"] {
++ let obj = RTSTARTUP_SYSROOT.to_path(dirs).join(format!("{file}.o"));
++ let mut build_rtstartup_cmd = Command::new(&compiler.rustc);
++ build_rtstartup_cmd
++ .arg("--target")
++ .arg(&compiler.triple)
++ .arg("--emit=obj")
++ .arg("-o")
++ .arg(&obj)
++ .arg(rtstartup_src.join(format!("{file}.rs")));
++ spawn_and_wait(build_rtstartup_cmd);
++ target_libs.libs.push(obj.clone());
++ }
++
++ Some(target_libs)
+}
--- /dev/null
- use self::utils::is_ci;
+use std::env;
+use std::path::PathBuf;
+use std::process;
+
- const USAGE: &str = r#"The build system of cg_clif.
-
- USAGE:
- ./y.rs prepare [--out-dir DIR]
- ./y.rs build [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--no-unstable-features]
- ./y.rs test [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--no-unstable-features]
-
- OPTIONS:
- --sysroot none|clif|llvm
- Which sysroot libraries to use:
- `none` will not include any standard library in the sysroot.
- `clif` will build the standard library using Cranelift.
- `llvm` will use the pre-compiled standard library of rustc which is compiled with LLVM.
-
- --out-dir DIR
- Specify the directory in which the download, build and dist directories are stored.
- By default this is the working directory.
-
- --no-unstable-features
- fSome features are not yet ready for production usage. This option will disable these
- features. This includes the JIT mode and inline assembly support.
- "#;
-
++use self::utils::{is_ci, Compiler};
+
+mod abi_cafe;
++mod bench;
+mod build_backend;
+mod build_sysroot;
+mod config;
+mod path;
+mod prepare;
+mod rustc_info;
+mod tests;
+mod utils;
+
- eprintln!("{USAGE}");
+fn usage() {
- env::set_var("CG_CLIF_DISPLAY_CG_TIME", "1");
++ eprintln!("{}", include_str!("usage.txt"));
+}
+
+macro_rules! arg_error {
+ ($($err:tt)*) => {{
+ eprintln!($($err)*);
+ usage();
+ std::process::exit(1);
+ }};
+}
+
+#[derive(PartialEq, Debug)]
+enum Command {
+ Prepare,
+ Build,
+ Test,
++ AbiCafe,
++ Bench,
+}
+
+#[derive(Copy, Clone, Debug)]
+pub(crate) enum SysrootKind {
+ None,
+ Clif,
+ Llvm,
+}
+
+pub fn main() {
- let host_triple = if let Ok(host_triple) = std::env::var("HOST_TRIPLE") {
- host_triple
- } else if let Some(host_triple) = config::get_value("host") {
- host_triple
- } else {
- rustc_info::get_host_triple()
- };
- let target_triple = if let Ok(target_triple) = std::env::var("TARGET_TRIPLE") {
- if target_triple != "" {
- target_triple
- } else {
- host_triple.clone() // Empty target triple can happen on GHA
- }
- } else if let Some(target_triple) = config::get_value("target") {
- target_triple
- } else {
- host_triple.clone()
- };
++ if env::var("RUST_BACKTRACE").is_err() {
++ env::set_var("RUST_BACKTRACE", "1");
++ }
+ env::set_var("CG_CLIF_DISABLE_INCR_CACHE", "1");
+
+ if is_ci() {
+ // Disabling incr comp reduces cache size and incr comp doesn't save as much on CI anyway
+ env::set_var("CARGO_BUILD_INCREMENTAL", "false");
++
++ // Enable the Cranelift verifier
++ env::set_var("CG_CLIF_ENABLE_VERIFIER", "1");
+ }
+
+ let mut args = env::args().skip(1);
+ let command = match args.next().as_deref() {
+ Some("prepare") => Command::Prepare,
+ Some("build") => Command::Build,
+ Some("test") => Command::Test,
++ Some("abi-cafe") => Command::AbiCafe,
++ Some("bench") => Command::Bench,
+ Some(flag) if flag.starts_with('-') => arg_error!("Expected command found flag {}", flag),
+ Some(command) => arg_error!("Unknown command {}", command),
+ None => {
+ usage();
+ process::exit(0);
+ }
+ };
+
+ let mut out_dir = PathBuf::from(".");
+ let mut channel = "release";
+ let mut sysroot_kind = SysrootKind::Clif;
+ let mut use_unstable_features = true;
+ while let Some(arg) = args.next().as_deref() {
+ match arg {
+ "--out-dir" => {
+ out_dir = PathBuf::from(args.next().unwrap_or_else(|| {
+ arg_error!("--out-dir requires argument");
+ }))
+ }
+ "--debug" => channel = "debug",
+ "--sysroot" => {
+ sysroot_kind = match args.next().as_deref() {
+ Some("none") => SysrootKind::None,
+ Some("clif") => SysrootKind::Clif,
+ Some("llvm") => SysrootKind::Llvm,
+ Some(arg) => arg_error!("Unknown sysroot kind {}", arg),
+ None => arg_error!("--sysroot requires argument"),
+ }
+ }
+ "--no-unstable-features" => use_unstable_features = false,
+ flag if flag.starts_with("-") => arg_error!("Unknown flag {}", flag),
+ arg => arg_error!("Unexpected argument {}", arg),
+ }
+ }
+
- let cg_clif_dylib =
- build_backend::build_backend(&dirs, channel, &host_triple, use_unstable_features);
++ let bootstrap_host_compiler = Compiler::bootstrap_with_triple(
++ std::env::var("HOST_TRIPLE")
++ .ok()
++ .or_else(|| config::get_value("host"))
++ .unwrap_or_else(|| rustc_info::get_host_triple()),
++ );
++ let target_triple = std::env::var("TARGET_TRIPLE")
++ .ok()
++ .or_else(|| config::get_value("target"))
++ .unwrap_or_else(|| bootstrap_host_compiler.triple.clone());
+
+ // FIXME allow changing the location of these dirs using cli arguments
+ let current_dir = std::env::current_dir().unwrap();
+ out_dir = current_dir.join(out_dir);
+ let dirs = path::Dirs {
+ source_dir: current_dir.clone(),
+ download_dir: out_dir.join("download"),
+ build_dir: out_dir.join("build"),
+ dist_dir: out_dir.join("dist"),
+ };
+
+ path::RelPath::BUILD.ensure_exists(&dirs);
+
+ {
+ // Make sure we always explicitly specify the target dir
+ let target =
+ path::RelPath::BUILD.join("target_dir_should_be_set_explicitly").to_path(&dirs);
+ env::set_var("CARGO_TARGET_DIR", &target);
+ let _ = std::fs::remove_file(&target);
+ std::fs::File::create(target).unwrap();
+ }
+
+ if command == Command::Prepare {
+ prepare::prepare(&dirs);
+ process::exit(0);
+ }
+
- &host_triple,
- &target_triple,
++ env::set_var("RUSTC", "rustc_should_be_set_explicitly");
++ env::set_var("RUSTDOC", "rustdoc_should_be_set_explicitly");
++
++ let cg_clif_dylib = build_backend::build_backend(
++ &dirs,
++ channel,
++ &bootstrap_host_compiler,
++ use_unstable_features,
++ );
+ match command {
+ Command::Prepare => {
+ // Handled above
+ }
+ Command::Test => {
+ tests::run_tests(
+ &dirs,
+ channel,
+ sysroot_kind,
+ &cg_clif_dylib,
-
- abi_cafe::run(
++ &bootstrap_host_compiler,
++ target_triple.clone(),
+ );
- &dirs,
++ }
++ Command::AbiCafe => {
++ if bootstrap_host_compiler.triple != target_triple {
++ eprintln!("Abi-cafe doesn't support cross-compilation");
++ process::exit(1);
++ }
++ abi_cafe::run(channel, sysroot_kind, &dirs, &cg_clif_dylib, &bootstrap_host_compiler);
++ }
++ Command::Build => {
++ build_sysroot::build_sysroot(
++ &dirs,
+ channel,
+ sysroot_kind,
- &host_triple,
- &target_triple,
+ &cg_clif_dylib,
- Command::Build => {
++ &bootstrap_host_compiler,
++ target_triple,
+ );
+ }
- &host_triple,
- &target_triple,
++ Command::Bench => {
+ build_sysroot::build_sysroot(
+ &dirs,
+ channel,
+ sysroot_kind,
+ &cg_clif_dylib,
++ &bootstrap_host_compiler,
++ target_triple,
+ );
++ bench::benchmark(&dirs, &bootstrap_host_compiler);
+ }
+ }
+}
--- /dev/null
- pub(crate) const BUILD_SYSROOT: RelPath = RelPath::SOURCE.join("build_sysroot");
+use std::fs;
+use std::path::PathBuf;
+
++use super::utils::remove_dir_if_exists;
++
+#[derive(Debug, Clone)]
+pub(crate) struct Dirs {
+ pub(crate) source_dir: PathBuf,
+ pub(crate) download_dir: PathBuf,
+ pub(crate) build_dir: PathBuf,
+ pub(crate) dist_dir: PathBuf,
+}
+
+#[doc(hidden)]
+#[derive(Debug, Copy, Clone)]
+pub(crate) enum PathBase {
+ Source,
+ Download,
+ Build,
+ Dist,
+}
+
+impl PathBase {
+ fn to_path(self, dirs: &Dirs) -> PathBuf {
+ match self {
+ PathBase::Source => dirs.source_dir.clone(),
+ PathBase::Download => dirs.download_dir.clone(),
+ PathBase::Build => dirs.build_dir.clone(),
+ PathBase::Dist => dirs.dist_dir.clone(),
+ }
+ }
+}
+
+#[derive(Debug, Copy, Clone)]
+pub(crate) enum RelPath {
+ Base(PathBase),
+ Join(&'static RelPath, &'static str),
+}
+
+impl RelPath {
+ pub(crate) const SOURCE: RelPath = RelPath::Base(PathBase::Source);
+ pub(crate) const DOWNLOAD: RelPath = RelPath::Base(PathBase::Download);
+ pub(crate) const BUILD: RelPath = RelPath::Base(PathBase::Build);
+ pub(crate) const DIST: RelPath = RelPath::Base(PathBase::Dist);
+
+ pub(crate) const SCRIPTS: RelPath = RelPath::SOURCE.join("scripts");
- if path.exists() {
- fs::remove_dir_all(&path).unwrap();
- }
+ pub(crate) const PATCHES: RelPath = RelPath::SOURCE.join("patches");
+
+ pub(crate) const fn join(&'static self, suffix: &'static str) -> RelPath {
+ RelPath::Join(self, suffix)
+ }
+
+ pub(crate) fn to_path(&self, dirs: &Dirs) -> PathBuf {
+ match self {
+ RelPath::Base(base) => base.to_path(dirs),
+ RelPath::Join(base, suffix) => base.to_path(dirs).join(suffix),
+ }
+ }
+
+ pub(crate) fn ensure_exists(&self, dirs: &Dirs) {
+ fs::create_dir_all(self.to_path(dirs)).unwrap();
+ }
+
+ pub(crate) fn ensure_fresh(&self, dirs: &Dirs) {
+ let path = self.to_path(dirs);
++ remove_dir_if_exists(&path);
+ fs::create_dir_all(path).unwrap();
+ }
+}
--- /dev/null
- use super::build_sysroot::{SYSROOT_RUSTC_VERSION, SYSROOT_SRC};
+use std::ffi::OsStr;
+use std::fs;
+use std::path::{Path, PathBuf};
+use std::process::Command;
+
- use super::rustc_info::{get_file_name, get_rustc_path, get_rustc_version};
- use super::utils::{copy_dir_recursively, spawn_and_wait, Compiler};
++use super::build_sysroot::{BUILD_SYSROOT, ORIG_BUILD_SYSROOT, SYSROOT_RUSTC_VERSION, SYSROOT_SRC};
+use super::path::{Dirs, RelPath};
- if RelPath::DOWNLOAD.to_path(dirs).exists() {
- std::fs::remove_dir_all(RelPath::DOWNLOAD.to_path(dirs)).unwrap();
- }
- std::fs::create_dir_all(RelPath::DOWNLOAD.to_path(dirs)).unwrap();
++use super::rustc_info::{get_default_sysroot, get_rustc_version};
++use super::utils::{copy_dir_recursively, git_command, retry_spawn_and_wait, spawn_and_wait};
+
+pub(crate) fn prepare(dirs: &Dirs) {
- prepare_sysroot(dirs);
++ RelPath::DOWNLOAD.ensure_fresh(dirs);
+
- // FIXME maybe install this only locally?
- eprintln!("[INSTALL] hyperfine");
- Command::new("cargo")
- .arg("install")
- .arg("hyperfine")
- .env_remove("CARGO_TARGET_DIR")
- .spawn()
- .unwrap()
- .wait()
- .unwrap();
++ spawn_and_wait(super::build_backend::CG_CLIF.fetch("cargo", dirs));
+
- super::tests::SIMPLE_RAYTRACER_REPO.fetch(dirs);
-
- eprintln!("[LLVM BUILD] simple-raytracer");
- let host_compiler = Compiler::host();
- let build_cmd = super::tests::SIMPLE_RAYTRACER.build(&host_compiler, dirs);
- spawn_and_wait(build_cmd);
- fs::copy(
- super::tests::SIMPLE_RAYTRACER
- .target_dir(dirs)
- .join(&host_compiler.triple)
- .join("debug")
- .join(get_file_name("main", "bin")),
- RelPath::BUILD.to_path(dirs).join(get_file_name("raytracer_cg_llvm", "bin")),
- )
- .unwrap();
++ prepare_sysroot(dirs);
++ spawn_and_wait(super::build_sysroot::STANDARD_LIBRARY.fetch("cargo", dirs));
++ spawn_and_wait(super::tests::LIBCORE_TESTS.fetch("cargo", dirs));
+
+ super::abi_cafe::ABI_CAFE_REPO.fetch(dirs);
++ spawn_and_wait(super::abi_cafe::ABI_CAFE.fetch("cargo", dirs));
+ super::tests::RAND_REPO.fetch(dirs);
++ spawn_and_wait(super::tests::RAND.fetch("cargo", dirs));
+ super::tests::REGEX_REPO.fetch(dirs);
++ spawn_and_wait(super::tests::REGEX.fetch("cargo", dirs));
+ super::tests::PORTABLE_SIMD_REPO.fetch(dirs);
- let rustc_path = get_rustc_path();
- let sysroot_src_orig = rustc_path.parent().unwrap().join("../lib/rustlib/src/rust");
- let sysroot_src = SYSROOT_SRC;
-
++ spawn_and_wait(super::tests::PORTABLE_SIMD.fetch("cargo", dirs));
++ super::bench::SIMPLE_RAYTRACER_REPO.fetch(dirs);
++ spawn_and_wait(super::bench::SIMPLE_RAYTRACER.fetch("cargo", dirs));
+}
+
+fn prepare_sysroot(dirs: &Dirs) {
- sysroot_src.ensure_fresh(dirs);
- fs::create_dir_all(sysroot_src.to_path(dirs).join("library")).unwrap();
++ let sysroot_src_orig = get_default_sysroot(Path::new("rustc")).join("lib/rustlib/src/rust");
+ assert!(sysroot_src_orig.exists());
+
- &sysroot_src.to_path(dirs).join("library"),
+ eprintln!("[COPY] sysroot src");
++
++ // FIXME ensure builds error out or update the copy if any of the files copied here change
++ BUILD_SYSROOT.ensure_fresh(dirs);
++ copy_dir_recursively(&ORIG_BUILD_SYSROOT.to_path(dirs), &BUILD_SYSROOT.to_path(dirs));
++
++ fs::create_dir_all(SYSROOT_SRC.to_path(dirs).join("library")).unwrap();
+ copy_dir_recursively(
+ &sysroot_src_orig.join("library"),
- let rustc_version = get_rustc_version();
++ &SYSROOT_SRC.to_path(dirs).join("library"),
+ );
+
- init_git_repo(&sysroot_src.to_path(dirs));
++ let rustc_version = get_rustc_version(Path::new("rustc"));
+ fs::write(SYSROOT_RUSTC_VERSION.to_path(dirs), &rustc_version).unwrap();
+
+ eprintln!("[GIT] init");
- apply_patches(dirs, "sysroot", &sysroot_src.to_path(dirs));
++ init_git_repo(&SYSROOT_SRC.to_path(dirs));
+
- Command::new("git").arg("clone").arg(repo).arg(&download_dir).spawn().unwrap().wait().unwrap();
++ apply_patches(dirs, "sysroot", &SYSROOT_SRC.to_path(dirs));
+}
+
+pub(crate) struct GitRepo {
+ url: GitRepoUrl,
+ rev: &'static str,
+ patch_name: &'static str,
+}
+
+enum GitRepoUrl {
+ Github { user: &'static str, repo: &'static str },
+}
+
+impl GitRepo {
+ pub(crate) const fn github(
+ user: &'static str,
+ repo: &'static str,
+ rev: &'static str,
+ patch_name: &'static str,
+ ) -> GitRepo {
+ GitRepo { url: GitRepoUrl::Github { user, repo }, rev, patch_name }
+ }
+
+ pub(crate) const fn source_dir(&self) -> RelPath {
+ match self.url {
+ GitRepoUrl::Github { user: _, repo } => RelPath::DOWNLOAD.join(repo),
+ }
+ }
+
+ fn fetch(&self, dirs: &Dirs) {
+ match self.url {
+ GitRepoUrl::Github { user, repo } => {
+ clone_repo_shallow_github(
+ dirs,
+ &self.source_dir().to_path(dirs),
+ user,
+ repo,
+ self.rev,
+ );
+ }
+ }
+ apply_patches(dirs, self.patch_name, &self.source_dir().to_path(dirs));
+ }
+}
+
+#[allow(dead_code)]
+fn clone_repo(download_dir: &Path, repo: &str, rev: &str) {
+ eprintln!("[CLONE] {}", repo);
+ // Ignore exit code as the repo may already have been checked out
- let mut clean_cmd = Command::new("git");
- clean_cmd.arg("checkout").arg("--").arg(".").current_dir(&download_dir);
++ git_command(None, "clone").arg(repo).arg(download_dir).spawn().unwrap().wait().unwrap();
+
- let mut checkout_cmd = Command::new("git");
- checkout_cmd.arg("checkout").arg("-q").arg(rev).current_dir(download_dir);
++ let mut clean_cmd = git_command(download_dir, "checkout");
++ clean_cmd.arg("--").arg(".");
+ spawn_and_wait(clean_cmd);
+
- download_cmd.arg("--location").arg("--output").arg(&archive_file).arg(archive_url);
- spawn_and_wait(download_cmd);
++ let mut checkout_cmd = git_command(download_dir, "checkout");
++ checkout_cmd.arg("-q").arg(rev);
+ spawn_and_wait(checkout_cmd);
+}
+
+fn clone_repo_shallow_github(dirs: &Dirs, download_dir: &Path, user: &str, repo: &str, rev: &str) {
+ if cfg!(windows) {
+ // Older windows doesn't have tar or curl by default. Fall back to using git.
+ clone_repo(download_dir, &format!("https://github.com/{}/{}.git", user, repo), rev);
+ return;
+ }
+
+ let archive_url = format!("https://github.com/{}/{}/archive/{}.tar.gz", user, repo, rev);
+ let archive_file = RelPath::DOWNLOAD.to_path(dirs).join(format!("{}.tar.gz", rev));
+ let archive_dir = RelPath::DOWNLOAD.to_path(dirs).join(format!("{}-{}", repo, rev));
+
+ eprintln!("[DOWNLOAD] {}/{} from {}", user, repo, archive_url);
+
+ // Remove previous results if they exists
+ let _ = std::fs::remove_file(&archive_file);
+ let _ = std::fs::remove_dir_all(&archive_dir);
+ let _ = std::fs::remove_dir_all(&download_dir);
+
+ // Download zip archive
+ let mut download_cmd = Command::new("curl");
- let mut git_init_cmd = Command::new("git");
- git_init_cmd.arg("init").arg("-q").current_dir(repo_dir);
++ download_cmd
++ .arg("--max-time")
++ .arg("600")
++ .arg("-y")
++ .arg("30")
++ .arg("-Y")
++ .arg("10")
++ .arg("--connect-timeout")
++ .arg("30")
++ .arg("--continue-at")
++ .arg("-")
++ .arg("--location")
++ .arg("--output")
++ .arg(&archive_file)
++ .arg(archive_url);
++ retry_spawn_and_wait(5, download_cmd);
+
+ // Unpack tar archive
+ let mut unpack_cmd = Command::new("tar");
+ unpack_cmd.arg("xf").arg(&archive_file).current_dir(RelPath::DOWNLOAD.to_path(dirs));
+ spawn_and_wait(unpack_cmd);
+
+ // Rename unpacked dir to the expected name
+ std::fs::rename(archive_dir, &download_dir).unwrap();
+
+ init_git_repo(&download_dir);
+
+ // Cleanup
+ std::fs::remove_file(archive_file).unwrap();
+}
+
+fn init_git_repo(repo_dir: &Path) {
- let mut git_add_cmd = Command::new("git");
- git_add_cmd.arg("add").arg(".").current_dir(repo_dir);
++ let mut git_init_cmd = git_command(repo_dir, "init");
++ git_init_cmd.arg("-q");
+ spawn_and_wait(git_init_cmd);
+
- let mut git_commit_cmd = Command::new("git");
- git_commit_cmd
- .arg("-c")
- .arg("user.name=Dummy")
- .arg("-c")
- .arg("user.email=dummy@example.com")
- .arg("commit")
- .arg("-m")
- .arg("Initial commit")
- .arg("-q")
- .current_dir(repo_dir);
++ let mut git_add_cmd = git_command(repo_dir, "add");
++ git_add_cmd.arg(".");
+ spawn_and_wait(git_add_cmd);
+
- let mut apply_patch_cmd = Command::new("git");
- apply_patch_cmd
- .arg("-c")
- .arg("user.name=Dummy")
- .arg("-c")
- .arg("user.email=dummy@example.com")
- .arg("am")
- .arg(patch)
- .arg("-q")
- .current_dir(target_dir);
++ let mut git_commit_cmd = git_command(repo_dir, "commit");
++ git_commit_cmd.arg("-m").arg("Initial commit").arg("-q");
+ spawn_and_wait(git_commit_cmd);
+}
+
+fn get_patches(dirs: &Dirs, crate_name: &str) -> Vec<PathBuf> {
+ let mut patches: Vec<_> = fs::read_dir(RelPath::PATCHES.to_path(dirs))
+ .unwrap()
+ .map(|entry| entry.unwrap().path())
+ .filter(|path| path.extension() == Some(OsStr::new("patch")))
+ .filter(|path| {
+ path.file_name()
+ .unwrap()
+ .to_str()
+ .unwrap()
+ .split_once("-")
+ .unwrap()
+ .1
+ .starts_with(crate_name)
+ })
+ .collect();
+ patches.sort();
+ patches
+}
+
+fn apply_patches(dirs: &Dirs, crate_name: &str, target_dir: &Path) {
+ if crate_name == "<none>" {
+ return;
+ }
+
+ for patch in get_patches(dirs, crate_name) {
+ eprintln!(
+ "[PATCH] {:?} <- {:?}",
+ target_dir.file_name().unwrap(),
+ patch.file_name().unwrap()
+ );
++ let mut apply_patch_cmd = git_command(target_dir, "am");
++ apply_patch_cmd.arg(patch).arg("-q");
+ spawn_and_wait(apply_patch_cmd);
+ }
+}
--- /dev/null
- pub(crate) fn get_rustc_version() -> String {
+use std::path::{Path, PathBuf};
+use std::process::{Command, Stdio};
+
- Command::new("rustc").stderr(Stdio::inherit()).args(&["-V"]).output().unwrap().stdout;
++pub(crate) fn get_rustc_version(rustc: &Path) -> String {
+ let version_info =
- pub(crate) fn get_default_sysroot() -> PathBuf {
- let default_sysroot = Command::new("rustc")
++ Command::new(rustc).stderr(Stdio::inherit()).args(&["-V"]).output().unwrap().stdout;
+ String::from_utf8(version_info).unwrap()
+}
+
+pub(crate) fn get_host_triple() -> String {
+ let version_info =
+ Command::new("rustc").stderr(Stdio::inherit()).args(&["-vV"]).output().unwrap().stdout;
+ String::from_utf8(version_info)
+ .unwrap()
+ .lines()
+ .to_owned()
+ .find(|line| line.starts_with("host"))
+ .unwrap()
+ .split(":")
+ .nth(1)
+ .unwrap()
+ .trim()
+ .to_owned()
+}
+
++pub(crate) fn get_toolchain_name() -> String {
++ let active_toolchain = Command::new("rustup")
++ .stderr(Stdio::inherit())
++ .args(&["show", "active-toolchain"])
++ .output()
++ .unwrap()
++ .stdout;
++ String::from_utf8(active_toolchain).unwrap().trim().split_once(' ').unwrap().0.to_owned()
++}
++
+pub(crate) fn get_cargo_path() -> PathBuf {
+ let cargo_path = Command::new("rustup")
+ .stderr(Stdio::inherit())
+ .args(&["which", "cargo"])
+ .output()
+ .unwrap()
+ .stdout;
+ Path::new(String::from_utf8(cargo_path).unwrap().trim()).to_owned()
+}
+
+pub(crate) fn get_rustc_path() -> PathBuf {
+ let rustc_path = Command::new("rustup")
+ .stderr(Stdio::inherit())
+ .args(&["which", "rustc"])
+ .output()
+ .unwrap()
+ .stdout;
+ Path::new(String::from_utf8(rustc_path).unwrap().trim()).to_owned()
+}
+
+pub(crate) fn get_rustdoc_path() -> PathBuf {
+ let rustc_path = Command::new("rustup")
+ .stderr(Stdio::inherit())
+ .args(&["which", "rustdoc"])
+ .output()
+ .unwrap()
+ .stdout;
+ Path::new(String::from_utf8(rustc_path).unwrap().trim()).to_owned()
+}
+
-
- /// Similar to `get_file_name`, but converts any dashes (`-`) in the `crate_name` to
- /// underscores (`_`). This is specially made for the rustc and cargo wrappers
- /// which have a dash in the name, and that is not allowed in a crate name.
- pub(crate) fn get_wrapper_file_name(crate_name: &str, crate_type: &str) -> String {
- let crate_name = crate_name.replace('-', "_");
- let wrapper_name = get_file_name(&crate_name, crate_type);
- wrapper_name.replace('_', "-")
- }
++pub(crate) fn get_default_sysroot(rustc: &Path) -> PathBuf {
++ let default_sysroot = Command::new(rustc)
+ .stderr(Stdio::inherit())
+ .args(&["--print", "sysroot"])
+ .output()
+ .unwrap()
+ .stdout;
+ Path::new(String::from_utf8(default_sysroot).unwrap().trim()).to_owned()
+}
+
+pub(crate) fn get_file_name(crate_name: &str, crate_type: &str) -> String {
+ let file_name = Command::new("rustc")
+ .stderr(Stdio::inherit())
+ .args(&[
+ "--crate-name",
+ crate_name,
+ "--crate-type",
+ crate_type,
+ "--print",
+ "file-names",
+ "-",
+ ])
+ .output()
+ .unwrap()
+ .stdout;
+ let file_name = String::from_utf8(file_name).unwrap().trim().to_owned();
+ assert!(!file_name.contains('\n'));
+ assert!(file_name.contains(crate_name));
+ file_name
+}
--- /dev/null
- use super::build_sysroot;
++use super::bench::SIMPLE_RAYTRACER;
++use super::build_sysroot::{self, SYSROOT_SRC};
+use super::config;
+use super::path::{Dirs, RelPath};
+use super::prepare::GitRepo;
- use super::rustc_info::{get_cargo_path, get_wrapper_file_name};
- use super::utils::{
- hyperfine_command, spawn_and_wait, spawn_and_wait_with_input, CargoProject, Compiler,
- };
++use super::rustc_info::get_host_triple;
++use super::utils::{spawn_and_wait, spawn_and_wait_with_input, CargoProject, Compiler};
+use super::SysrootKind;
+use std::env;
+use std::ffi::OsStr;
+use std::fs;
+use std::path::Path;
+use std::process::Command;
+
+static BUILD_EXAMPLE_OUT_DIR: RelPath = RelPath::BUILD.join("example");
+
+struct TestCase {
+ config: &'static str,
- func: &'static dyn Fn(&TestRunner),
++ cmd: TestCaseCmd,
++}
++
++enum TestCaseCmd {
++ Custom { func: &'static dyn Fn(&TestRunner) },
++ BuildLib { source: &'static str, crate_types: &'static str },
++ BuildBinAndRun { source: &'static str, args: &'static [&'static str] },
++ JitBin { source: &'static str, args: &'static str },
+}
+
+impl TestCase {
- const fn new(config: &'static str, func: &'static dyn Fn(&TestRunner)) -> Self {
- Self { config, func }
++ // FIXME reduce usage of custom test case commands
++ const fn custom(config: &'static str, func: &'static dyn Fn(&TestRunner)) -> Self {
++ Self { config, cmd: TestCaseCmd::Custom { func } }
++ }
++
++ const fn build_lib(
++ config: &'static str,
++ source: &'static str,
++ crate_types: &'static str,
++ ) -> Self {
++ Self { config, cmd: TestCaseCmd::BuildLib { source, crate_types } }
++ }
++
++ const fn build_bin_and_run(
++ config: &'static str,
++ source: &'static str,
++ args: &'static [&'static str],
++ ) -> Self {
++ Self { config, cmd: TestCaseCmd::BuildBinAndRun { source, args } }
++ }
++
++ const fn jit_bin(config: &'static str, source: &'static str, args: &'static str) -> Self {
++ Self { config, cmd: TestCaseCmd::JitBin { source, args } }
+ }
+}
+
+const NO_SYSROOT_SUITE: &[TestCase] = &[
- TestCase::new("build.mini_core", &|runner| {
- runner.run_rustc([
- "example/mini_core.rs",
- "--crate-name",
- "mini_core",
- "--crate-type",
- "lib,dylib",
- "--target",
- &runner.target_compiler.triple,
- ]);
- }),
- TestCase::new("build.example", &|runner| {
- runner.run_rustc([
- "example/example.rs",
- "--crate-type",
- "lib",
- "--target",
- &runner.target_compiler.triple,
- ]);
- }),
- TestCase::new("jit.mini_core_hello_world", &|runner| {
- let mut jit_cmd = runner.rustc_command([
- "-Zunstable-options",
- "-Cllvm-args=mode=jit",
- "-Cprefer-dynamic",
- "example/mini_core_hello_world.rs",
- "--cfg",
- "jit",
- "--target",
- &runner.target_compiler.triple,
- ]);
- jit_cmd.env("CG_CLIF_JIT_ARGS", "abc bcd");
- spawn_and_wait(jit_cmd);
-
- eprintln!("[JIT-lazy] mini_core_hello_world");
- let mut jit_cmd = runner.rustc_command([
- "-Zunstable-options",
- "-Cllvm-args=mode=jit-lazy",
- "-Cprefer-dynamic",
- "example/mini_core_hello_world.rs",
- "--cfg",
- "jit",
- "--target",
- &runner.target_compiler.triple,
- ]);
- jit_cmd.env("CG_CLIF_JIT_ARGS", "abc bcd");
- spawn_and_wait(jit_cmd);
- }),
- TestCase::new("aot.mini_core_hello_world", &|runner| {
- runner.run_rustc([
- "example/mini_core_hello_world.rs",
- "--crate-name",
- "mini_core_hello_world",
- "--crate-type",
- "bin",
- "-g",
- "--target",
- &runner.target_compiler.triple,
- ]);
- runner.run_out_command("mini_core_hello_world", ["abc", "bcd"]);
- }),
++ TestCase::build_lib("build.mini_core", "example/mini_core.rs", "lib,dylib"),
++ TestCase::build_lib("build.example", "example/example.rs", "lib"),
++ TestCase::jit_bin("jit.mini_core_hello_world", "example/mini_core_hello_world.rs", "abc bcd"),
++ TestCase::build_bin_and_run(
++ "aot.mini_core_hello_world",
++ "example/mini_core_hello_world.rs",
++ &["abc", "bcd"],
++ ),
+];
+
+const BASE_SYSROOT_SUITE: &[TestCase] = &[
- TestCase::new("aot.arbitrary_self_types_pointers_and_wrappers", &|runner| {
- runner.run_rustc([
- "example/arbitrary_self_types_pointers_and_wrappers.rs",
- "--crate-name",
- "arbitrary_self_types_pointers_and_wrappers",
- "--crate-type",
- "bin",
- "--target",
- &runner.target_compiler.triple,
- ]);
- runner.run_out_command("arbitrary_self_types_pointers_and_wrappers", []);
- }),
- TestCase::new("aot.issue_91827_extern_types", &|runner| {
- runner.run_rustc([
- "example/issue-91827-extern-types.rs",
- "--crate-name",
- "issue_91827_extern_types",
- "--crate-type",
- "bin",
- "--target",
- &runner.target_compiler.triple,
- ]);
- runner.run_out_command("issue_91827_extern_types", []);
- }),
- TestCase::new("build.alloc_system", &|runner| {
- runner.run_rustc([
- "example/alloc_system.rs",
- "--crate-type",
- "lib",
- "--target",
- &runner.target_compiler.triple,
- ]);
- }),
- TestCase::new("aot.alloc_example", &|runner| {
- runner.run_rustc([
- "example/alloc_example.rs",
- "--crate-type",
- "bin",
- "--target",
- &runner.target_compiler.triple,
- ]);
- runner.run_out_command("alloc_example", []);
- }),
- TestCase::new("jit.std_example", &|runner| {
- runner.run_rustc([
- "-Zunstable-options",
- "-Cllvm-args=mode=jit",
- "-Cprefer-dynamic",
- "example/std_example.rs",
- "--target",
- &runner.target_compiler.triple,
- ]);
-
- eprintln!("[JIT-lazy] std_example");
- runner.run_rustc([
- "-Zunstable-options",
- "-Cllvm-args=mode=jit-lazy",
- "-Cprefer-dynamic",
- "example/std_example.rs",
- "--target",
- &runner.target_compiler.triple,
- ]);
- }),
- TestCase::new("aot.std_example", &|runner| {
- runner.run_rustc([
- "example/std_example.rs",
- "--crate-type",
- "bin",
- "--target",
- &runner.target_compiler.triple,
- ]);
- runner.run_out_command("std_example", ["arg"]);
- }),
- TestCase::new("aot.dst_field_align", &|runner| {
- runner.run_rustc([
- "example/dst-field-align.rs",
- "--crate-name",
- "dst_field_align",
- "--crate-type",
- "bin",
- "--target",
- &runner.target_compiler.triple,
- ]);
- runner.run_out_command("dst_field_align", []);
- }),
- TestCase::new("aot.subslice-patterns-const-eval", &|runner| {
- runner.run_rustc([
- "example/subslice-patterns-const-eval.rs",
- "--crate-type",
- "bin",
- "-Cpanic=abort",
- "--target",
- &runner.target_compiler.triple,
- ]);
- runner.run_out_command("subslice-patterns-const-eval", []);
- }),
- TestCase::new("aot.track-caller-attribute", &|runner| {
- runner.run_rustc([
- "example/track-caller-attribute.rs",
- "--crate-type",
- "bin",
- "-Cpanic=abort",
- "--target",
- &runner.target_compiler.triple,
- ]);
- runner.run_out_command("track-caller-attribute", []);
- }),
- TestCase::new("aot.float-minmax-pass", &|runner| {
- runner.run_rustc([
- "example/float-minmax-pass.rs",
- "--crate-type",
- "bin",
- "-Cpanic=abort",
- "--target",
- &runner.target_compiler.triple,
- ]);
- runner.run_out_command("float-minmax-pass", []);
- }),
- TestCase::new("aot.mod_bench", &|runner| {
- runner.run_rustc([
- "example/mod_bench.rs",
- "--crate-type",
- "bin",
- "--target",
- &runner.target_compiler.triple,
- ]);
- runner.run_out_command("mod_bench", []);
- }),
- TestCase::new("aot.issue-72793", &|runner| {
- runner.run_rustc([
- "example/issue-72793.rs",
- "--crate-type",
- "bin",
- "--target",
- &runner.target_compiler.triple,
- ]);
- runner.run_out_command("issue-72793", []);
- }),
++ TestCase::build_bin_and_run(
++ "aot.arbitrary_self_types_pointers_and_wrappers",
++ "example/arbitrary_self_types_pointers_and_wrappers.rs",
++ &[],
++ ),
++ TestCase::build_bin_and_run(
++ "aot.issue_91827_extern_types",
++ "example/issue-91827-extern-types.rs",
++ &[],
++ ),
++ TestCase::build_lib("build.alloc_system", "example/alloc_system.rs", "lib"),
++ TestCase::build_bin_and_run("aot.alloc_example", "example/alloc_example.rs", &[]),
++ TestCase::jit_bin("jit.std_example", "example/std_example.rs", ""),
++ TestCase::build_bin_and_run("aot.std_example", "example/std_example.rs", &["arg"]),
++ TestCase::build_bin_and_run("aot.dst_field_align", "example/dst-field-align.rs", &[]),
++ TestCase::build_bin_and_run(
++ "aot.subslice-patterns-const-eval",
++ "example/subslice-patterns-const-eval.rs",
++ &[],
++ ),
++ TestCase::build_bin_and_run(
++ "aot.track-caller-attribute",
++ "example/track-caller-attribute.rs",
++ &[],
++ ),
++ TestCase::build_bin_and_run("aot.float-minmax-pass", "example/float-minmax-pass.rs", &[]),
++ TestCase::build_bin_and_run("aot.mod_bench", "example/mod_bench.rs", &[]),
++ TestCase::build_bin_and_run("aot.issue-72793", "example/issue-72793.rs", &[]),
+];
+
+pub(crate) static RAND_REPO: GitRepo =
+ GitRepo::github("rust-random", "rand", "0f933f9c7176e53b2a3c7952ded484e1783f0bf1", "rand");
+
- static RAND: CargoProject = CargoProject::new(&RAND_REPO.source_dir(), "rand");
++pub(crate) static RAND: CargoProject = CargoProject::new(&RAND_REPO.source_dir(), "rand");
+
+pub(crate) static REGEX_REPO: GitRepo =
+ GitRepo::github("rust-lang", "regex", "341f207c1071f7290e3f228c710817c280c8dca1", "regex");
+
- static REGEX: CargoProject = CargoProject::new(®EX_REPO.source_dir(), "regex");
++pub(crate) static REGEX: CargoProject = CargoProject::new(®EX_REPO.source_dir(), "regex");
+
+pub(crate) static PORTABLE_SIMD_REPO: GitRepo = GitRepo::github(
+ "rust-lang",
+ "portable-simd",
- "d5cd4a8112d958bd3a252327e0d069a6363249bd",
++ "582239ac3b32007613df04d7ffa78dc30f4c5645",
+ "portable-simd",
+);
+
- static PORTABLE_SIMD: CargoProject =
++pub(crate) static PORTABLE_SIMD: CargoProject =
+ CargoProject::new(&PORTABLE_SIMD_REPO.source_dir(), "portable_simd");
+
- pub(crate) static SIMPLE_RAYTRACER_REPO: GitRepo = GitRepo::github(
- "ebobby",
- "simple-raytracer",
- "804a7a21b9e673a482797aa289a18ed480e4d813",
- "<none>",
- );
-
- pub(crate) static SIMPLE_RAYTRACER: CargoProject =
- CargoProject::new(&SIMPLE_RAYTRACER_REPO.source_dir(), "simple_raytracer");
-
- static LIBCORE_TESTS: CargoProject =
- CargoProject::new(&RelPath::BUILD_SYSROOT.join("sysroot_src/library/core/tests"), "core_tests");
++pub(crate) static LIBCORE_TESTS: CargoProject =
++ CargoProject::new(&SYSROOT_SRC.join("library/core/tests"), "core_tests");
+
+const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[
- TestCase::new("test.rust-random/rand", &|runner| {
- spawn_and_wait(RAND.clean(&runner.target_compiler.cargo, &runner.dirs));
++ TestCase::custom("test.rust-random/rand", &|runner| {
++ RAND.clean(&runner.dirs);
+
+ if runner.is_native {
+ eprintln!("[TEST] rust-random/rand");
+ let mut test_cmd = RAND.test(&runner.target_compiler, &runner.dirs);
+ test_cmd.arg("--workspace");
+ spawn_and_wait(test_cmd);
+ } else {
+ eprintln!("[AOT] rust-random/rand");
+ let mut build_cmd = RAND.build(&runner.target_compiler, &runner.dirs);
+ build_cmd.arg("--workspace").arg("--tests");
+ spawn_and_wait(build_cmd);
+ }
+ }),
- TestCase::new("bench.simple-raytracer", &|runner| {
- let run_runs = env::var("RUN_RUNS").unwrap_or("10".to_string()).parse().unwrap();
-
- if runner.is_native {
- eprintln!("[BENCH COMPILE] ebobby/simple-raytracer");
- let cargo_clif = RelPath::DIST
- .to_path(&runner.dirs)
- .join(get_wrapper_file_name("cargo-clif", "bin"));
- let manifest_path = SIMPLE_RAYTRACER.manifest_path(&runner.dirs);
- let target_dir = SIMPLE_RAYTRACER.target_dir(&runner.dirs);
-
- let clean_cmd = format!(
- "cargo clean --manifest-path {manifest_path} --target-dir {target_dir}",
- manifest_path = manifest_path.display(),
- target_dir = target_dir.display(),
- );
- let llvm_build_cmd = format!(
- "cargo build --manifest-path {manifest_path} --target-dir {target_dir}",
- manifest_path = manifest_path.display(),
- target_dir = target_dir.display(),
- );
- let clif_build_cmd = format!(
- "{cargo_clif} build --manifest-path {manifest_path} --target-dir {target_dir}",
- cargo_clif = cargo_clif.display(),
- manifest_path = manifest_path.display(),
- target_dir = target_dir.display(),
- );
-
- let bench_compile =
- hyperfine_command(1, run_runs, Some(&clean_cmd), &llvm_build_cmd, &clif_build_cmd);
-
- spawn_and_wait(bench_compile);
-
- eprintln!("[BENCH RUN] ebobby/simple-raytracer");
- fs::copy(
- target_dir.join("debug").join("main"),
- RelPath::BUILD.to_path(&runner.dirs).join("raytracer_cg_clif"),
- )
- .unwrap();
-
- let mut bench_run =
- hyperfine_command(0, run_runs, None, "./raytracer_cg_llvm", "./raytracer_cg_clif");
- bench_run.current_dir(RelPath::BUILD.to_path(&runner.dirs));
- spawn_and_wait(bench_run);
- } else {
- spawn_and_wait(SIMPLE_RAYTRACER.clean(&runner.target_compiler.cargo, &runner.dirs));
- eprintln!("[BENCH COMPILE] ebobby/simple-raytracer (skipped)");
- eprintln!("[COMPILE] ebobby/simple-raytracer");
- spawn_and_wait(SIMPLE_RAYTRACER.build(&runner.target_compiler, &runner.dirs));
- eprintln!("[BENCH RUN] ebobby/simple-raytracer (skipped)");
- }
++ TestCase::custom("test.simple-raytracer", &|runner| {
++ SIMPLE_RAYTRACER.clean(&runner.dirs);
++ spawn_and_wait(SIMPLE_RAYTRACER.build(&runner.target_compiler, &runner.dirs));
+ }),
- TestCase::new("test.libcore", &|runner| {
- spawn_and_wait(LIBCORE_TESTS.clean(&runner.host_compiler.cargo, &runner.dirs));
++ TestCase::custom("test.libcore", &|runner| {
++ LIBCORE_TESTS.clean(&runner.dirs);
+
+ if runner.is_native {
+ spawn_and_wait(LIBCORE_TESTS.test(&runner.target_compiler, &runner.dirs));
+ } else {
+ eprintln!("Cross-Compiling: Not running tests");
+ let mut build_cmd = LIBCORE_TESTS.build(&runner.target_compiler, &runner.dirs);
+ build_cmd.arg("--tests");
+ spawn_and_wait(build_cmd);
+ }
+ }),
- TestCase::new("test.regex-shootout-regex-dna", &|runner| {
- spawn_and_wait(REGEX.clean(&runner.target_compiler.cargo, &runner.dirs));
++ TestCase::custom("test.regex-shootout-regex-dna", &|runner| {
++ REGEX.clean(&runner.dirs);
+
+ // newer aho_corasick versions throw a deprecation warning
+ let lint_rust_flags = format!("{} --cap-lints warn", runner.target_compiler.rustflags);
+
+ let mut build_cmd = REGEX.build(&runner.target_compiler, &runner.dirs);
+ build_cmd.arg("--example").arg("shootout-regex-dna");
+ build_cmd.env("RUSTFLAGS", lint_rust_flags.clone());
+ spawn_and_wait(build_cmd);
+
+ if runner.is_native {
+ let mut run_cmd = REGEX.run(&runner.target_compiler, &runner.dirs);
+ run_cmd.arg("--example").arg("shootout-regex-dna");
+ run_cmd.env("RUSTFLAGS", lint_rust_flags);
+
+ let input = fs::read_to_string(
+ REGEX.source_dir(&runner.dirs).join("examples").join("regexdna-input.txt"),
+ )
+ .unwrap();
- let expected_path =
- REGEX.source_dir(&runner.dirs).join("examples").join("regexdna-output.txt");
- let expected = fs::read_to_string(&expected_path).unwrap();
++ let expected = fs::read_to_string(
++ REGEX.source_dir(&runner.dirs).join("examples").join("regexdna-output.txt"),
++ )
++ .unwrap();
+
+ let output = spawn_and_wait_with_input(run_cmd, input);
+ // Make sure `[codegen mono items] start` doesn't poison the diff
+ let output = output
+ .lines()
+ .filter(|line| !line.contains("codegen mono items"))
+ .chain(Some("")) // This just adds the trailing newline
+ .collect::<Vec<&str>>()
+ .join("\r\n");
+
+ let output_matches = expected.lines().eq(output.lines());
+ if !output_matches {
- let res_path = REGEX.source_dir(&runner.dirs).join("res.txt");
- fs::write(&res_path, &output).unwrap();
-
- if cfg!(windows) {
- println!("Output files don't match!");
- println!("Expected Output:\n{}", expected);
- println!("Actual Output:\n{}", output);
- } else {
- let mut diff = Command::new("diff");
- diff.arg("-u");
- diff.arg(res_path);
- diff.arg(expected_path);
- spawn_and_wait(diff);
- }
++ println!("Output files don't match!");
++ println!("Expected Output:\n{}", expected);
++ println!("Actual Output:\n{}", output);
+
+ std::process::exit(1);
+ }
+ }
+ }),
- TestCase::new("test.regex", &|runner| {
- spawn_and_wait(REGEX.clean(&runner.host_compiler.cargo, &runner.dirs));
++ TestCase::custom("test.regex", &|runner| {
++ REGEX.clean(&runner.dirs);
+
+ // newer aho_corasick versions throw a deprecation warning
+ let lint_rust_flags = format!("{} --cap-lints warn", runner.target_compiler.rustflags);
+
+ if runner.is_native {
+ let mut run_cmd = REGEX.test(&runner.target_compiler, &runner.dirs);
+ run_cmd.args([
+ "--tests",
+ "--",
+ "--exclude-should-panic",
+ "--test-threads",
+ "1",
+ "-Zunstable-options",
+ "-q",
+ ]);
+ run_cmd.env("RUSTFLAGS", lint_rust_flags);
+ spawn_and_wait(run_cmd);
+ } else {
+ eprintln!("Cross-Compiling: Not running tests");
+ let mut build_cmd = REGEX.build(&runner.target_compiler, &runner.dirs);
+ build_cmd.arg("--tests");
+ build_cmd.env("RUSTFLAGS", lint_rust_flags.clone());
+ spawn_and_wait(build_cmd);
+ }
+ }),
- TestCase::new("test.portable-simd", &|runner| {
- spawn_and_wait(PORTABLE_SIMD.clean(&runner.host_compiler.cargo, &runner.dirs));
++ TestCase::custom("test.portable-simd", &|runner| {
++ PORTABLE_SIMD.clean(&runner.dirs);
+
+ let mut build_cmd = PORTABLE_SIMD.build(&runner.target_compiler, &runner.dirs);
+ build_cmd.arg("--all-targets");
+ spawn_and_wait(build_cmd);
+
+ if runner.is_native {
+ let mut test_cmd = PORTABLE_SIMD.test(&runner.target_compiler, &runner.dirs);
+ test_cmd.arg("-q");
+ spawn_and_wait(test_cmd);
+ }
+ }),
+];
+
+pub(crate) fn run_tests(
+ dirs: &Dirs,
+ channel: &str,
+ sysroot_kind: SysrootKind,
+ cg_clif_dylib: &Path,
- host_triple: &str,
- target_triple: &str,
++ bootstrap_host_compiler: &Compiler,
++ target_triple: String,
+) {
- let runner = TestRunner::new(dirs.clone(), host_triple.to_string(), target_triple.to_string());
-
+ if config::get_bool("testsuite.no_sysroot") {
- build_sysroot::build_sysroot(
++ let target_compiler = build_sysroot::build_sysroot(
+ dirs,
+ channel,
+ SysrootKind::None,
+ cg_clif_dylib,
- &host_triple,
- &target_triple,
++ bootstrap_host_compiler,
++ target_triple.clone(),
+ );
+
++ let runner =
++ TestRunner::new(dirs.clone(), target_compiler, get_host_triple() == target_triple);
++
+ BUILD_EXAMPLE_OUT_DIR.ensure_fresh(dirs);
+ runner.run_testsuite(NO_SYSROOT_SUITE);
+ } else {
+ eprintln!("[SKIP] no_sysroot tests");
+ }
+
+ let run_base_sysroot = config::get_bool("testsuite.base_sysroot");
+ let run_extended_sysroot = config::get_bool("testsuite.extended_sysroot");
+
+ if run_base_sysroot || run_extended_sysroot {
- build_sysroot::build_sysroot(
++ let target_compiler = build_sysroot::build_sysroot(
+ dirs,
+ channel,
+ sysroot_kind,
+ cg_clif_dylib,
- &host_triple,
- &target_triple,
++ bootstrap_host_compiler,
++ target_triple.clone(),
+ );
- }
+
- if run_base_sysroot {
- runner.run_testsuite(BASE_SYSROOT_SUITE);
- } else {
- eprintln!("[SKIP] base_sysroot tests");
- }
++ let runner =
++ TestRunner::new(dirs.clone(), target_compiler, get_host_triple() == target_triple);
+
- if run_extended_sysroot {
- runner.run_testsuite(EXTENDED_SYSROOT_SUITE);
- } else {
- eprintln!("[SKIP] extended_sysroot tests");
++ if run_base_sysroot {
++ runner.run_testsuite(BASE_SYSROOT_SUITE);
++ } else {
++ eprintln!("[SKIP] base_sysroot tests");
++ }
++
++ if run_extended_sysroot {
++ runner.run_testsuite(EXTENDED_SYSROOT_SUITE);
++ } else {
++ eprintln!("[SKIP] extended_sysroot tests");
++ }
+ }
+}
+
+struct TestRunner {
+ is_native: bool,
+ jit_supported: bool,
+ dirs: Dirs,
- host_compiler: Compiler,
+ target_compiler: Compiler,
+}
+
+impl TestRunner {
- pub fn new(dirs: Dirs, host_triple: String, target_triple: String) -> Self {
- let is_native = host_triple == target_triple;
- let jit_supported =
- target_triple.contains("x86_64") && is_native && !host_triple.contains("windows");
-
- let rustc_clif =
- RelPath::DIST.to_path(&dirs).join(get_wrapper_file_name("rustc-clif", "bin"));
- let rustdoc_clif =
- RelPath::DIST.to_path(&dirs).join(get_wrapper_file_name("rustdoc-clif", "bin"));
-
- let mut rustflags = env::var("RUSTFLAGS").ok().unwrap_or("".to_string());
- let mut runner = vec![];
-
- if !is_native {
- match target_triple.as_str() {
- "aarch64-unknown-linux-gnu" => {
- // We are cross-compiling for aarch64. Use the correct linker and run tests in qemu.
- rustflags = format!("-Clinker=aarch64-linux-gnu-gcc{}", rustflags);
- runner = vec![
- "qemu-aarch64".to_owned(),
- "-L".to_owned(),
- "/usr/aarch64-linux-gnu".to_owned(),
- ];
- }
- "s390x-unknown-linux-gnu" => {
- // We are cross-compiling for s390x. Use the correct linker and run tests in qemu.
- rustflags = format!("-Clinker=s390x-linux-gnu-gcc{}", rustflags);
- runner = vec![
- "qemu-s390x".to_owned(),
- "-L".to_owned(),
- "/usr/s390x-linux-gnu".to_owned(),
- ];
- }
- "x86_64-pc-windows-gnu" => {
- // We are cross-compiling for Windows. Run tests in wine.
- runner = vec!["wine".to_owned()];
- }
- _ => {
- println!("Unknown non-native platform");
- }
- }
++ pub fn new(dirs: Dirs, mut target_compiler: Compiler, is_native: bool) -> Self {
++ if let Ok(rustflags) = env::var("RUSTFLAGS") {
++ target_compiler.rustflags.push(' ');
++ target_compiler.rustflags.push_str(&rustflags);
++ }
++ if let Ok(rustdocflags) = env::var("RUSTDOCFLAGS") {
++ target_compiler.rustdocflags.push(' ');
++ target_compiler.rustdocflags.push_str(&rustdocflags);
+ }
+
+ // FIXME fix `#[linkage = "extern_weak"]` without this
- if target_triple.contains("darwin") {
- rustflags = format!("{} -Clink-arg=-undefined -Clink-arg=dynamic_lookup", rustflags);
++ if target_compiler.triple.contains("darwin") {
++ target_compiler.rustflags.push_str(" -Clink-arg=-undefined -Clink-arg=dynamic_lookup");
+ }
+
- let host_compiler = Compiler {
- cargo: get_cargo_path(),
- rustc: rustc_clif.clone(),
- rustdoc: rustdoc_clif.clone(),
- rustflags: String::new(),
- rustdocflags: String::new(),
- triple: host_triple,
- runner: vec![],
- };
-
- let target_compiler = Compiler {
- cargo: get_cargo_path(),
- rustc: rustc_clif,
- rustdoc: rustdoc_clif,
- rustflags: rustflags.clone(),
- rustdocflags: rustflags,
- triple: target_triple,
- runner,
- };
-
- Self { is_native, jit_supported, dirs, host_compiler, target_compiler }
++ let jit_supported = is_native
++ && target_compiler.triple.contains("x86_64")
++ && !target_compiler.triple.contains("windows");
++
++ Self { is_native, jit_supported, dirs, target_compiler }
+ }
+
+ pub fn run_testsuite(&self, tests: &[TestCase]) {
- for &TestCase { config, func } in tests {
++ for TestCase { config, cmd } in tests {
+ let (tag, testname) = config.split_once('.').unwrap();
+ let tag = tag.to_uppercase();
+ let is_jit_test = tag == "JIT";
+
+ if !config::get_bool(config) || (is_jit_test && !self.jit_supported) {
+ eprintln!("[{tag}] {testname} (skipped)");
+ continue;
+ } else {
+ eprintln!("[{tag}] {testname}");
+ }
+
- func(self);
++ match *cmd {
++ TestCaseCmd::Custom { func } => func(self),
++ TestCaseCmd::BuildLib { source, crate_types } => {
++ self.run_rustc([source, "--crate-type", crate_types]);
++ }
++ TestCaseCmd::BuildBinAndRun { source, args } => {
++ self.run_rustc([source]);
++ self.run_out_command(
++ source.split('/').last().unwrap().split('.').next().unwrap(),
++ args,
++ );
++ }
++ TestCaseCmd::JitBin { source, args } => {
++ let mut jit_cmd = self.rustc_command([
++ "-Zunstable-options",
++ "-Cllvm-args=mode=jit",
++ "-Cprefer-dynamic",
++ source,
++ "--cfg",
++ "jit",
++ ]);
++ if !args.is_empty() {
++ jit_cmd.env("CG_CLIF_JIT_ARGS", args);
++ }
++ spawn_and_wait(jit_cmd);
++
++ eprintln!("[JIT-lazy] {testname}");
++ let mut jit_cmd = self.rustc_command([
++ "-Zunstable-options",
++ "-Cllvm-args=mode=jit-lazy",
++ "-Cprefer-dynamic",
++ source,
++ "--cfg",
++ "jit",
++ ]);
++ if !args.is_empty() {
++ jit_cmd.env("CG_CLIF_JIT_ARGS", args);
++ }
++ spawn_and_wait(jit_cmd);
++ }
++ }
+ }
+ }
+
+ #[must_use]
+ fn rustc_command<I, S>(&self, args: I) -> Command
+ where
+ I: IntoIterator<Item = S>,
+ S: AsRef<OsStr>,
+ {
+ let mut cmd = Command::new(&self.target_compiler.rustc);
+ cmd.args(self.target_compiler.rustflags.split_whitespace());
+ cmd.arg("-L");
+ cmd.arg(format!("crate={}", BUILD_EXAMPLE_OUT_DIR.to_path(&self.dirs).display()));
+ cmd.arg("--out-dir");
+ cmd.arg(format!("{}", BUILD_EXAMPLE_OUT_DIR.to_path(&self.dirs).display()));
+ cmd.arg("-Cdebuginfo=2");
++ cmd.arg("--target");
++ cmd.arg(&self.target_compiler.triple);
++ cmd.arg("-Cpanic=abort");
+ cmd.args(args);
+ cmd
+ }
+
+ fn run_rustc<I, S>(&self, args: I)
+ where
+ I: IntoIterator<Item = S>,
+ S: AsRef<OsStr>,
+ {
+ spawn_and_wait(self.rustc_command(args));
+ }
+
- fn run_out_command<'a, I>(&self, name: &str, args: I)
- where
- I: IntoIterator<Item = &'a str>,
- {
++ fn run_out_command<'a>(&self, name: &str, args: &[&str]) {
+ let mut full_cmd = vec![];
+
+ // Prepend the RUN_WRAPPER's
+ if !self.target_compiler.runner.is_empty() {
+ full_cmd.extend(self.target_compiler.runner.iter().cloned());
+ }
+
+ full_cmd.push(
+ BUILD_EXAMPLE_OUT_DIR.to_path(&self.dirs).join(name).to_str().unwrap().to_string(),
+ );
+
- for arg in args.into_iter() {
++ for arg in args {
+ full_cmd.push(arg.to_string());
+ }
+
+ let mut cmd_iter = full_cmd.into_iter();
+ let first = cmd_iter.next().unwrap();
+
+ let mut cmd = Command::new(first);
+ cmd.args(cmd_iter);
+
+ spawn_and_wait(cmd);
+ }
+}
--- /dev/null
--- /dev/null
++The build system of cg_clif.
++
++USAGE:
++ ./y.rs prepare [--out-dir DIR]
++ ./y.rs build [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--no-unstable-features]
++ ./y.rs test [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--no-unstable-features]
++ ./y.rs abi-cafe [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--no-unstable-features]
++ ./y.rs bench [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--no-unstable-features]
++
++OPTIONS:
++ --debug
++ Build cg_clif and the standard library in debug mode rather than release mode.
++ Warning: An unoptimized cg_clif is very slow.
++
++ --sysroot none|clif|llvm
++ Which sysroot libraries to use:
++ `none` will not include any standard library in the sysroot.
++ `clif` will build the standard library using Cranelift.
++ `llvm` will use the pre-compiled standard library of rustc which is compiled with LLVM.
++
++ --out-dir DIR
++ Specify the directory in which the download, build and dist directories are stored.
++ By default this is the working directory.
++
++ --no-unstable-features
++ Some features are not yet ready for production usage. This option will disable these
++ features. This includes the JIT mode and inline assembly support.
++
++REQUIREMENTS:
++ * Rustup: The build system has a hard coded dependency on rustup to install the right nightly
++ version and make sure it is used where necessary.
++ * Git: `./y.rs prepare` uses git for applying patches and on Windows for downloading test repos.
++ * Curl and tar (non-Windows only): Used by `./y.rs prepare` to download a single commit for
++ repos. Git will be used to clone the whole repo when using Windows.
++ * [Hyperfine](https://github.com/sharkdp/hyperfine/): Used for benchmarking with `./y.rs bench`.
--- /dev/null
- use std::io::Write;
+use std::env;
+use std::fs;
- use super::rustc_info::{get_cargo_path, get_host_triple, get_rustc_path, get_rustdoc_path};
++use std::io::{self, Write};
+use std::path::{Path, PathBuf};
+use std::process::{self, Command, Stdio};
+
+use super::path::{Dirs, RelPath};
- pub(crate) fn host() -> Compiler {
++use super::rustc_info::{get_cargo_path, get_rustc_path, get_rustdoc_path};
+
++#[derive(Clone, Debug)]
+pub(crate) struct Compiler {
+ pub(crate) cargo: PathBuf,
+ pub(crate) rustc: PathBuf,
+ pub(crate) rustdoc: PathBuf,
+ pub(crate) rustflags: String,
+ pub(crate) rustdocflags: String,
+ pub(crate) triple: String,
+ pub(crate) runner: Vec<String>,
+}
+
+impl Compiler {
- triple: get_host_triple(),
++ pub(crate) fn bootstrap_with_triple(triple: String) -> Compiler {
+ Compiler {
+ cargo: get_cargo_path(),
+ rustc: get_rustc_path(),
+ rustdoc: get_rustdoc_path(),
+ rustflags: String::new(),
+ rustdocflags: String::new(),
- pub(crate) fn with_triple(triple: String) -> Compiler {
- Compiler {
- cargo: get_cargo_path(),
- rustc: get_rustc_path(),
- rustdoc: get_rustdoc_path(),
- rustflags: String::new(),
- rustdocflags: String::new(),
- triple,
- runner: vec![],
++ triple,
+ runner: vec![],
+ }
+ }
+
- .arg(self.target_dir(dirs));
++ pub(crate) fn set_cross_linker_and_runner(&mut self) {
++ match self.triple.as_str() {
++ "aarch64-unknown-linux-gnu" => {
++ // We are cross-compiling for aarch64. Use the correct linker and run tests in qemu.
++ self.rustflags += " -Clinker=aarch64-linux-gnu-gcc";
++ self.rustdocflags += " -Clinker=aarch64-linux-gnu-gcc";
++ self.runner = vec![
++ "qemu-aarch64".to_owned(),
++ "-L".to_owned(),
++ "/usr/aarch64-linux-gnu".to_owned(),
++ ];
++ }
++ "s390x-unknown-linux-gnu" => {
++ // We are cross-compiling for s390x. Use the correct linker and run tests in qemu.
++ self.rustflags += " -Clinker=s390x-linux-gnu-gcc";
++ self.rustdocflags += " -Clinker=s390x-linux-gnu-gcc";
++ self.runner = vec![
++ "qemu-s390x".to_owned(),
++ "-L".to_owned(),
++ "/usr/s390x-linux-gnu".to_owned(),
++ ];
++ }
++ "x86_64-pc-windows-gnu" => {
++ // We are cross-compiling for Windows. Run tests in wine.
++ self.runner = vec!["wine".to_owned()];
++ }
++ _ => {
++ println!("Unknown non-native platform");
++ }
+ }
+ }
+}
+
+pub(crate) struct CargoProject {
+ source: &'static RelPath,
+ target: &'static str,
+}
+
+impl CargoProject {
+ pub(crate) const fn new(path: &'static RelPath, target: &'static str) -> CargoProject {
+ CargoProject { source: path, target }
+ }
+
+ pub(crate) fn source_dir(&self, dirs: &Dirs) -> PathBuf {
+ self.source.to_path(dirs)
+ }
+
+ pub(crate) fn manifest_path(&self, dirs: &Dirs) -> PathBuf {
+ self.source_dir(dirs).join("Cargo.toml")
+ }
+
+ pub(crate) fn target_dir(&self, dirs: &Dirs) -> PathBuf {
+ RelPath::BUILD.join(self.target).to_path(dirs)
+ }
+
++ #[must_use]
+ fn base_cmd(&self, command: &str, cargo: &Path, dirs: &Dirs) -> Command {
+ let mut cmd = Command::new(cargo);
+
+ cmd.arg(command)
+ .arg("--manifest-path")
+ .arg(self.manifest_path(dirs))
+ .arg("--target-dir")
- #[must_use]
- pub(crate) fn clean(&self, cargo: &Path, dirs: &Dirs) -> Command {
- self.base_cmd("clean", cargo, dirs)
++ .arg(self.target_dir(dirs))
++ .arg("--frozen");
+
+ cmd
+ }
+
++ #[must_use]
+ fn build_cmd(&self, command: &str, compiler: &Compiler, dirs: &Dirs) -> Command {
+ let mut cmd = self.base_cmd(command, &compiler.cargo, dirs);
+
+ cmd.arg("--target").arg(&compiler.triple);
+
+ cmd.env("RUSTC", &compiler.rustc);
+ cmd.env("RUSTDOC", &compiler.rustdoc);
+ cmd.env("RUSTFLAGS", &compiler.rustflags);
+ cmd.env("RUSTDOCFLAGS", &compiler.rustdocflags);
+ if !compiler.runner.is_empty() {
+ cmd.env(
+ format!("CARGO_TARGET_{}_RUNNER", compiler.triple.to_uppercase().replace('-', "_")),
+ compiler.runner.join(" "),
+ );
+ }
+
+ cmd
+ }
+
+ #[must_use]
+ pub(crate) fn fetch(&self, cargo: impl AsRef<Path>, dirs: &Dirs) -> Command {
+ let mut cmd = Command::new(cargo.as_ref());
+
+ cmd.arg("fetch").arg("--manifest-path").arg(self.manifest_path(dirs));
+
+ cmd
+ }
+
++ pub(crate) fn clean(&self, dirs: &Dirs) {
++ let _ = fs::remove_dir_all(self.target_dir(dirs));
+ }
+
+ #[must_use]
+ pub(crate) fn build(&self, compiler: &Compiler, dirs: &Dirs) -> Command {
+ self.build_cmd("build", compiler, dirs)
+ }
+
+ #[must_use]
+ pub(crate) fn test(&self, compiler: &Compiler, dirs: &Dirs) -> Command {
+ self.build_cmd("test", compiler, dirs)
+ }
+
+ #[must_use]
+ pub(crate) fn run(&self, compiler: &Compiler, dirs: &Dirs) -> Command {
+ self.build_cmd("run", compiler, dirs)
+ }
+}
+
+#[must_use]
+pub(crate) fn hyperfine_command(
+ warmup: u64,
+ runs: u64,
+ prepare: Option<&str>,
+ a: &str,
+ b: &str,
+) -> Command {
+ let mut bench = Command::new("hyperfine");
+
+ if warmup != 0 {
+ bench.arg("--warmup").arg(warmup.to_string());
+ }
+
+ if runs != 0 {
+ bench.arg("--runs").arg(runs.to_string());
+ }
+
+ if let Some(prepare) = prepare {
+ bench.arg("--prepare").arg(prepare);
+ }
+
+ bench.arg(a).arg(b);
+
+ bench
+}
+
++#[must_use]
++pub(crate) fn git_command<'a>(repo_dir: impl Into<Option<&'a Path>>, cmd: &str) -> Command {
++ let mut git_cmd = Command::new("git");
++ git_cmd
++ .arg("-c")
++ .arg("user.name=Dummy")
++ .arg("-c")
++ .arg("user.email=dummy@example.com")
++ .arg("-c")
++ .arg("core.autocrlf=false")
++ .arg(cmd);
++ if let Some(repo_dir) = repo_dir.into() {
++ git_cmd.current_dir(repo_dir);
++ }
++ git_cmd
++}
++
+#[track_caller]
+pub(crate) fn try_hard_link(src: impl AsRef<Path>, dst: impl AsRef<Path>) {
+ let src = src.as_ref();
+ let dst = dst.as_ref();
+ if let Err(_) = fs::hard_link(src, dst) {
+ fs::copy(src, dst).unwrap(); // Fallback to copying if hardlinking failed
+ }
+}
+
+#[track_caller]
+pub(crate) fn spawn_and_wait(mut cmd: Command) {
+ if !cmd.spawn().unwrap().wait().unwrap().success() {
+ process::exit(1);
+ }
+}
+
++// Based on the retry function in rust's src/ci/shared.sh
++#[track_caller]
++pub(crate) fn retry_spawn_and_wait(tries: u64, mut cmd: Command) {
++ for i in 1..tries + 1 {
++ if i != 1 {
++ println!("Command failed. Attempt {i}/{tries}:");
++ }
++ if cmd.spawn().unwrap().wait().unwrap().success() {
++ return;
++ }
++ std::thread::sleep(std::time::Duration::from_secs(i * 5));
++ }
++ println!("The command has failed after {tries} attempts.");
++ process::exit(1);
++}
++
+#[track_caller]
+pub(crate) fn spawn_and_wait_with_input(mut cmd: Command, input: String) -> String {
+ let mut child = cmd
+ .stdin(Stdio::piped())
+ .stdout(Stdio::piped())
+ .spawn()
+ .expect("Failed to spawn child process");
+
+ let mut stdin = child.stdin.take().expect("Failed to open stdin");
+ std::thread::spawn(move || {
+ stdin.write_all(input.as_bytes()).expect("Failed to write to stdin");
+ });
+
+ let output = child.wait_with_output().expect("Failed to read stdout");
+ if !output.status.success() {
+ process::exit(1);
+ }
+
+ String::from_utf8(output.stdout).unwrap()
+}
+
++pub(crate) fn remove_dir_if_exists(path: &Path) {
++ match fs::remove_dir_all(&path) {
++ Ok(()) => {}
++ Err(err) if err.kind() == io::ErrorKind::NotFound => {}
++ Err(err) => panic!("Failed to remove {path}: {err}", path = path.display()),
++ }
++}
++
+pub(crate) fn copy_dir_recursively(from: &Path, to: &Path) {
+ for entry in fs::read_dir(from).unwrap() {
+ let entry = entry.unwrap();
+ let filename = entry.file_name();
+ if filename == "." || filename == ".." {
+ continue;
+ }
+ if entry.metadata().unwrap().is_dir() {
+ fs::create_dir(to.join(&filename)).unwrap();
+ copy_dir_recursively(&from.join(&filename), &to.join(&filename));
+ } else {
+ fs::copy(from.join(&filename), to.join(&filename)).unwrap();
+ }
+ }
+}
+
+pub(crate) fn is_ci() -> bool {
+ env::var("CI").as_deref() == Ok("true")
+}
--- /dev/null
- rm -rf build_sysroot/{sysroot_src/,target/,compiler-builtins/,rustc_version}
- rm -rf target/ build/ dist/ perf.data{,.old} y.bin
- rm -rf download/
+#!/usr/bin/env bash
+set -e
+
++rm -rf target/ download/ build/ dist/ y.bin y.bin.dSYM y.exe y.pdb
+
+# Kept for now in case someone updates their checkout of cg_clif before running clean_all.sh
+# FIXME remove at some point in the future
+rm -rf rand/ regex/ simple-raytracer/ portable-simd/ abi-checker/ abi-cafe/
++rm -rf build_sysroot/{sysroot_src/,target/,compiler-builtins/,rustc_version}
--- /dev/null
- bench.simple-raytracer
+# This file allows configuring the build system.
+
+# Which triple to produce a compiler toolchain for.
+#
+# Defaults to the default triple of rustc on the host system.
+#host = x86_64-unknown-linux-gnu
+
+# Which triple to build libraries (core/alloc/std/test/proc_macro) for.
+#
+# Defaults to `host`.
+#target = x86_64-unknown-linux-gnu
+
+# Disables cleaning of the sysroot dir. This will cause old compiled artifacts to be re-used when
+# the sysroot source hasn't changed. This is useful when the codegen backend hasn't been modified.
+# This option can be changed while the build system is already running for as long as sysroot
+# building hasn't started yet.
+#keep_sysroot
+
+
+# Testsuite
+#
+# Each test suite item has a corresponding key here. The default is to run all tests.
+# Comment any of these lines to skip individual tests.
+
+testsuite.no_sysroot
+build.mini_core
+build.example
+jit.mini_core_hello_world
+aot.mini_core_hello_world
+
+testsuite.base_sysroot
+aot.arbitrary_self_types_pointers_and_wrappers
+aot.issue_91827_extern_types
+build.alloc_system
+aot.alloc_example
+jit.std_example
+aot.std_example
+aot.dst_field_align
+aot.subslice-patterns-const-eval
+aot.track-caller-attribute
+aot.float-minmax-pass
+aot.mod_bench
+aot.issue-72793
+
+testsuite.extended_sysroot
+test.rust-random/rand
-
- testsuite.abi-cafe
++test.simple-raytracer
+test.libcore
+test.regex-shootout-regex-dna
+test.regex
+test.portable-simd
--- /dev/null
- @@ -0,0 +1,11 @@
+From f6befc4bb51d84f5f1cf35938a168c953d421350 Mon Sep 17 00:00:00 2001
+From: bjorn3 <bjorn3@users.noreply.github.com>
+Date: Sun, 24 Nov 2019 15:10:23 +0100
+Subject: [PATCH] [core] Disable not compiling tests
+
+---
+ library/core/tests/Cargo.toml | 8 ++++++++
+ library/core/tests/num/flt2dec/mod.rs | 1 -
+ library/core/tests/num/int_macros.rs | 2 ++
+ library/core/tests/num/uint_macros.rs | 2 ++
+ library/core/tests/ptr.rs | 2 ++
+ library/core/tests/slice.rs | 2 ++
+ 6 files changed, 16 insertions(+), 1 deletion(-)
+ create mode 100644 library/core/tests/Cargo.toml
+
+diff --git a/library/core/tests/Cargo.toml b/library/core/tests/Cargo.toml
+new file mode 100644
+index 0000000..46fd999
+--- /dev/null
++++ b/library/core/tests/Cargo.toml
- +rand = "0.7"
++@@ -0,0 +1,12 @@
++[package]
++name = "core"
++version = "0.0.0"
++edition = "2021"
++
++[lib]
++name = "coretests"
++path = "lib.rs"
++
++[dependencies]
+++rand = { version = "0.8.5", default-features = false }
+++rand_xorshift = { version = "0.3.0", default-features = false }
+--
+2.21.0 (Apple Git-122)
--- /dev/null
- channel = "nightly-2022-12-13"
+[toolchain]
++channel = "nightly-2023-01-20"
+components = ["rust-src", "rustc-dev", "llvm-tools-preview"]
--- /dev/null
- env::set_var("RUSTUP_TOOLCHAIN", env!("RUSTUP_TOOLCHAIN"));
+use std::env;
+#[cfg(unix)]
+use std::os::unix::process::CommandExt;
+use std::path::PathBuf;
+use std::process::Command;
+
+fn main() {
+ let sysroot = PathBuf::from(env::current_exe().unwrap().parent().unwrap());
+
+ let mut rustflags = String::new();
+ rustflags.push_str(" -Cpanic=abort -Zpanic-abort-tests -Zcodegen-backend=");
+ rustflags.push_str(
+ sysroot
+ .join(if cfg!(windows) { "bin" } else { "lib" })
+ .join(
+ env::consts::DLL_PREFIX.to_string()
+ + "rustc_codegen_cranelift"
+ + env::consts::DLL_SUFFIX,
+ )
+ .to_str()
+ .unwrap(),
+ );
+ rustflags.push_str(" --sysroot ");
+ rustflags.push_str(sysroot.to_str().unwrap());
+ env::set_var("RUSTFLAGS", env::var("RUSTFLAGS").unwrap_or(String::new()) + &rustflags);
+ env::set_var("RUSTDOCFLAGS", env::var("RUSTDOCFLAGS").unwrap_or(String::new()) + &rustflags);
+
+ // Ensure that the right toolchain is used
++ env::set_var("RUSTUP_TOOLCHAIN", env!("TOOLCHAIN_NAME"));
+
+ let args: Vec<_> = match env::args().nth(1).as_deref() {
+ Some("jit") => {
+ env::set_var(
+ "RUSTFLAGS",
+ env::var("RUSTFLAGS").unwrap_or(String::new()) + " -Cprefer-dynamic",
+ );
+ IntoIterator::into_iter(["rustc".to_string()])
+ .chain(env::args().skip(2))
+ .chain([
+ "--".to_string(),
+ "-Zunstable-options".to_string(),
+ "-Cllvm-args=mode=jit".to_string(),
+ ])
+ .collect()
+ }
+ Some("lazy-jit") => {
+ env::set_var(
+ "RUSTFLAGS",
+ env::var("RUSTFLAGS").unwrap_or(String::new()) + " -Cprefer-dynamic",
+ );
+ IntoIterator::into_iter(["rustc".to_string()])
+ .chain(env::args().skip(2))
+ .chain([
+ "--".to_string(),
+ "-Zunstable-options".to_string(),
+ "-Cllvm-args=mode=jit-lazy".to_string(),
+ ])
+ .collect()
+ }
+ _ => env::args().skip(1).collect(),
+ };
+
+ #[cfg(unix)]
+ Command::new("cargo").args(args).exec();
+
+ #[cfg(not(unix))]
+ std::process::exit(
+ Command::new("cargo").args(args).spawn().unwrap().wait().unwrap().code().unwrap_or(1),
+ );
+}
--- /dev/null
- env::set_var("RUSTUP_TOOLCHAIN", env!("RUSTUP_TOOLCHAIN"));
+use std::env;
+use std::ffi::OsString;
+#[cfg(unix)]
+use std::os::unix::process::CommandExt;
+use std::path::PathBuf;
+use std::process::Command;
+
+fn main() {
+ let sysroot = PathBuf::from(env::current_exe().unwrap().parent().unwrap());
+
+ let cg_clif_dylib_path = sysroot.join(if cfg!(windows) { "bin" } else { "lib" }).join(
+ env::consts::DLL_PREFIX.to_string() + "rustc_codegen_cranelift" + env::consts::DLL_SUFFIX,
+ );
+
+ let mut args = std::env::args_os().skip(1).collect::<Vec<_>>();
+ args.push(OsString::from("-Cpanic=abort"));
+ args.push(OsString::from("-Zpanic-abort-tests"));
+ let mut codegen_backend_arg = OsString::from("-Zcodegen-backend=");
+ codegen_backend_arg.push(cg_clif_dylib_path);
+ args.push(codegen_backend_arg);
+ if !args.contains(&OsString::from("--sysroot")) {
+ args.push(OsString::from("--sysroot"));
+ args.push(OsString::from(sysroot.to_str().unwrap()));
+ }
+
+ // Ensure that the right toolchain is used
++ env::set_var("RUSTUP_TOOLCHAIN", env!("TOOLCHAIN_NAME"));
+
+ #[cfg(unix)]
+ Command::new("rustc").args(args).exec();
+
+ #[cfg(not(unix))]
+ std::process::exit(
+ Command::new("rustc").args(args).spawn().unwrap().wait().unwrap().code().unwrap_or(1),
+ );
+}
--- /dev/null
- env::set_var("RUSTUP_TOOLCHAIN", env!("RUSTUP_TOOLCHAIN"));
+use std::env;
+use std::ffi::OsString;
+#[cfg(unix)]
+use std::os::unix::process::CommandExt;
+use std::path::PathBuf;
+use std::process::Command;
+
+fn main() {
+ let sysroot = PathBuf::from(env::current_exe().unwrap().parent().unwrap());
+
+ let cg_clif_dylib_path = sysroot.join(if cfg!(windows) { "bin" } else { "lib" }).join(
+ env::consts::DLL_PREFIX.to_string() + "rustc_codegen_cranelift" + env::consts::DLL_SUFFIX,
+ );
+
+ let mut args = std::env::args_os().skip(1).collect::<Vec<_>>();
+ args.push(OsString::from("-Cpanic=abort"));
+ args.push(OsString::from("-Zpanic-abort-tests"));
+ let mut codegen_backend_arg = OsString::from("-Zcodegen-backend=");
+ codegen_backend_arg.push(cg_clif_dylib_path);
+ args.push(codegen_backend_arg);
+ if !args.contains(&OsString::from("--sysroot")) {
+ args.push(OsString::from("--sysroot"));
+ args.push(OsString::from(sysroot.to_str().unwrap()));
+ }
+
+ // Ensure that the right toolchain is used
++ env::set_var("RUSTUP_TOOLCHAIN", env!("TOOLCHAIN_NAME"));
+
+ #[cfg(unix)]
+ Command::new("rustdoc").args(args).exec();
+
+ #[cfg(not(unix))]
+ std::process::exit(
+ Command::new("rustdoc").args(args).spawn().unwrap().wait().unwrap().code().unwrap_or(1),
+ );
+}
--- /dev/null
- ./y.rs prepare
+#!/usr/bin/env bash
+
+set -e
+
+case $1 in
+ "prepare")
+ TOOLCHAIN=$(date +%Y-%m-%d)
+
+ echo "=> Installing new nightly"
+ rustup toolchain install --profile minimal "nightly-${TOOLCHAIN}" # Sanity check to see if the nightly exists
+ sed -i "s/\"nightly-.*\"/\"nightly-${TOOLCHAIN}\"/" rust-toolchain
+ rustup component add rustfmt || true
+
+ echo "=> Uninstalling all old nightlies"
+ for nightly in $(rustup toolchain list | grep nightly | grep -v "$TOOLCHAIN" | grep -v nightly-x86_64); do
+ rustup toolchain uninstall "$nightly"
+ done
+
+ ./clean_all.sh
- (cd build_sysroot && cargo update)
+
++ ./y.rs prepare
+
++ (cd download/sysroot && cargo update && cargo fetch && cp Cargo.lock ../../build_sysroot/)
+ ;;
+ "commit")
+ git add rust-toolchain build_sysroot/Cargo.lock
+ git commit -m "Rustup to $(rustc -V)"
+ ;;
+ "push")
+ cg_clif=$(pwd)
+ pushd ../rust
+ git pull origin master
+ branch=sync_cg_clif-$(date +%Y-%m-%d)
+ git checkout -b "$branch"
+ git subtree pull --prefix=compiler/rustc_codegen_cranelift/ https://github.com/bjorn3/rustc_codegen_cranelift.git master
+ git push -u my "$branch"
+
+ # immediately merge the merge commit into cg_clif to prevent merge conflicts when syncing
+ # from rust-lang/rust later
+ git subtree push --prefix=compiler/rustc_codegen_cranelift/ "$cg_clif" sync_from_rust
+ popd
+ git merge sync_from_rust
+ ;;
+ "pull")
+ cg_clif=$(pwd)
+ pushd ../rust
+ git pull origin master
+ rust_vers="$(git rev-parse HEAD)"
+ git subtree push --prefix=compiler/rustc_codegen_cranelift/ "$cg_clif" sync_from_rust
+ popd
+ git merge sync_from_rust -m "Sync from rust $rust_vers"
+ git branch -d sync_from_rust
+ ;;
+ *)
+ echo "Unknown command '$1'"
+ echo "Usage: ./rustup.sh prepare|commit"
+ ;;
+esac
--- /dev/null
- git am ../patches/*-sysroot-*.patch
+#!/usr/bin/env bash
+set -e
+
+./y.rs build --no-unstable-features
+
+echo "[SETUP] Rust fork"
+git clone https://github.com/rust-lang/rust.git || true
+pushd rust
+git fetch
+git checkout -- .
+git checkout "$(rustc -V | cut -d' ' -f3 | tr -d '(')"
+
- rand = "0.7"
- rand_xorshift = "0.2"
++git -c user.name=Dummy -c user.email=dummy@example.com am ../patches/*-sysroot-*.patch
+
+git apply - <<EOF
+diff --git a/library/alloc/Cargo.toml b/library/alloc/Cargo.toml
+index d95b5b7f17f..00b6f0e3635 100644
+--- a/library/alloc/Cargo.toml
++++ b/library/alloc/Cargo.toml
+@@ -8,7 +8,7 @@ edition = "2018"
+
+ [dependencies]
+ core = { path = "../core" }
+-compiler_builtins = { version = "0.1.40", features = ['rustc-dep-of-std'] }
++compiler_builtins = { version = "0.1.66", features = ['rustc-dep-of-std', 'no-asm'] }
+
+ [dev-dependencies]
- export CFG_VIRTUAL_RUST_SOURCE_BASE_DIR="$(cd build_sysroot/sysroot_src; pwd)"
++ rand = { version = "0.8.5", default-features = false, features = ["alloc"] }
++ rand_xorshift = "0.3.0"
+EOF
+
+cat > config.toml <<EOF
+changelog-seen = 2
+
+[llvm]
+ninja = false
+
+[build]
+rustc = "$(pwd)/../dist/rustc-clif"
+cargo = "$(rustup which cargo)"
+full-bootstrap = true
+local-rebuild = true
+
+[rust]
+codegen-backends = ["cranelift"]
+deny-warnings = false
+verbose-tests = false
+EOF
+popd
+
+# FIXME remove once inline asm is fully supported
+export RUSTFLAGS="$RUSTFLAGS --cfg=rustix_use_libc"
+
++export CFG_VIRTUAL_RUST_SOURCE_BASE_DIR="$(cd download/sysroot/sysroot_src; pwd)"
+
+# Allow the testsuite to use llvm tools
+host_triple=$(rustc -vV | grep host | cut -d: -f2 | tr -d " ")
+export LLVM_BIN_DIR="$(rustc --print sysroot)/lib/rustlib/$host_triple/bin"
--- /dev/null
- for test in $(rg --files-with-matches "lto|// needs-asm-support|// needs-unwind" tests/{ui,incremental}); do
+#!/usr/bin/env bash
+set -e
+
+cd $(dirname "$0")/../
+
+source ./scripts/setup_rust_fork.sh
+
+echo "[TEST] Test suite of rustc"
+pushd rust
+
+command -v rg >/dev/null 2>&1 || cargo install ripgrep
+
+rm -r tests/ui/{extern/,unsized-locals/,lto/,linkage*} || true
- rm tests/ui/simd/intrinsic/generic-bitmask-pass.rs # simd_bitmask unimplemented
- rm tests/ui/simd/intrinsic/generic-as.rs # simd_as unimplemented
- rm tests/ui/simd/intrinsic/generic-arithmetic-saturating-pass.rs # simd_saturating_add unimplemented
++for test in $(rg --files-with-matches "lto|// needs-asm-support|// needs-unwind" tests/{codegen-units,ui,incremental}); do
+ rm $test
+done
+
+for test in $(rg -i --files-with-matches "//(\[\w+\])?~[^\|]*\s*ERR|// error-pattern:|// build-fail|// run-fail|-Cllvm-args" tests/ui); do
+ rm $test
+done
+
+git checkout -- tests/ui/issues/auxiliary/issue-3136-a.rs # contains //~ERROR, but shouldn't be removed
+git checkout -- tests/ui/proc-macro/pretty-print-hack/
+
+# missing features
+# ================
+
+# requires stack unwinding
+rm tests/incremental/change_crate_dep_kind.rs
+rm tests/incremental/issue-80691-bad-eval-cache.rs # -Cpanic=abort causes abort instead of exit(101)
+
+# requires compiling with -Cpanic=unwind
+rm -r tests/ui/macros/rfc-2011-nicer-assert-messages/
+rm -r tests/run-make/test-benches
++rm tests/ui/test-attrs/test-type.rs
+
+# vendor intrinsics
+rm tests/ui/sse2.rs # cpuid not supported, so sse2 not detected
+rm tests/ui/intrinsics/const-eval-select-x86_64.rs # requires x86_64 vendor intrinsics
+rm tests/ui/simd/array-type.rs # "Index argument for `simd_insert` is not a constant"
- rm tests/ui/simd/intrinsic/generic-gather-pass.rs # simd_gather unimplemented
- rm tests/ui/simd/intrinsic/generic-select-pass.rs # simd_select_bitmask unimplemented
- rm tests/ui/simd/issue-85915-simd-ptrs.rs # simd_gather unimplemented
- rm tests/ui/simd/issue-89193.rs # simd_gather unimplemented
- rm tests/ui/simd/simd-bitmask.rs # simd_bitmask unimplemented
+rm tests/ui/simd/intrinsic/float-math-pass.rs # simd_fcos unimplemented
- rm tests/ui/abi/stack-probes.rs # stack probes not yet implemented
- rm tests/ui/simd/intrinsic/ptr-cast.rs # simd_expose_addr intrinsic unimplemented
+
+# exotic linkages
+rm tests/ui/issues/issue-33992.rs # unsupported linkages
+rm tests/incremental/hashes/function_interfaces.rs # same
+rm tests/incremental/hashes/statics.rs # same
+
+# variadic arguments
+rm tests/ui/abi/mir/mir_codegen_calls_variadic.rs # requires float varargs
+rm tests/ui/abi/variadic-ffi.rs # requires callee side vararg support
+
+# unsized locals
+rm -r tests/run-pass-valgrind/unsized-locals
+
+# misc unimplemented things
+rm tests/ui/intrinsics/intrinsic-nearby.rs # unimplemented nearbyintf32 and nearbyintf64 intrinsics
+rm tests/ui/target-feature/missing-plusminus.rs # error not implemented
+rm tests/ui/fn/dyn-fn-alignment.rs # wants a 256 byte alignment
+rm -r tests/run-make/emit-named-files # requires full --emit support
- rm tests/codegen-units/item-collection/asm-sym.rs # requires support for sym in asm!()
+rm -r tests/run-make/repr128-dwarf # debuginfo test
- rm tests/ui/issues/issue-74564-if-expr-stack-overflow.rs # gives a stackoverflow before the backend runs
- rm tests/ui/mir/ssa-analysis-regression-50041.rs # produces ICE
- rm tests/ui/type-alias-impl-trait/assoc-projection-ice.rs # produces ICE
+
+# optimization tests
+# ==================
+rm tests/ui/codegen/issue-28950.rs # depends on stack size optimizations
+rm tests/ui/codegen/init-large-type.rs # same
+rm tests/ui/issues/issue-40883.rs # same
+rm -r tests/run-make/fmt-write-bloat/ # tests an optimization
+
+# backend specific tests
+# ======================
+rm tests/incremental/thinlto/cgu_invalidated_when_import_{added,removed}.rs # requires LLVM
+rm tests/ui/abi/stack-protector.rs # requires stack protector support
+
+# giving different but possibly correct results
+# =============================================
+rm tests/ui/mir/mir_misc_casts.rs # depends on deduplication of constants
+rm tests/ui/mir/mir_raw_fat_ptr.rs # same
+rm tests/ui/consts/issue-33537.rs # same
+rm tests/ui/layout/valid_range_oob.rs # different ICE message
+
++rm tests/ui/consts/issue-miri-1910.rs # different error message
++rm tests/ui/consts/offset_ub.rs # same
++rm tests/ui/intrinsics/panic-uninitialized-zeroed.rs # same
++rm tests/ui/lint/lint-const-item-mutation.rs # same
++rm tests/ui/pattern/usefulness/doc-hidden-non-exhaustive.rs # same
++rm tests/ui/suggestions/derive-trait-for-method-call.rs # same
++rm tests/ui/typeck/issue-46112.rs # same
++
++rm tests/ui/proc-macro/crt-static.rs # extra warning about -Cpanic=abort for proc macros
++rm tests/ui/proc-macro/proc-macro-deprecated-attr.rs # same
++rm tests/ui/proc-macro/quote-debug.rs # same
++rm tests/ui/proc-macro/no-missing-docs.rs # same
++rm tests/ui/rust-2018/proc-macro-crate-in-paths.rs # same
++
+# doesn't work due to the way the rustc test suite is invoked.
+# should work when using ./x.py test the way it is intended
+# ============================================================
+rm -r tests/run-make/emit-shared-files # requires the rustdoc executable in dist/bin/
+rm -r tests/run-make/unstable-flag-required # same
+rm -r tests/run-make/rustdoc-* # same
+rm -r tests/run-make/issue-88756-default-output # same
+rm -r tests/run-make/remap-path-prefix-dwarf # requires llvm-dwarfdump
+rm -r tests/ui/consts/missing_span_in_backtrace.rs # expects sysroot source to be elsewhere
+
+# genuine bugs
+# ============
+rm tests/incremental/spike-neg1.rs # errors out for some reason
+rm tests/incremental/spike-neg2.rs # same
- rm tests/ui/runtime/out-of-stack.rs # SIGSEGV instead of SIGABRT for some reason (#1301)
+
+rm tests/ui/simd/intrinsic/generic-reduction-pass.rs # simd_reduce_add_unordered doesn't accept an accumulator for integer vectors
+
- rm tests/ui/test-attrs/test-type.rs # TODO panic message on stderr. correct stdout
- # not sure if this is actually a bug in the test suite, but the symbol list shows the function without leading _ for some reason
- rm -r tests/run-make/native-link-modifier-bundle
++rm tests/ui/simd/intrinsic/generic-as.rs # crash when accessing vector type filed (#1318)
++rm tests/ui/simd/simd-bitmask.rs # crash
++
++rm tests/ui/dyn-star/dyn-star-to-dyn.rs
++rm tests/ui/dyn-star/dispatch-on-pin-mut.rs
+
+# bugs in the test suite
+# ======================
+rm tests/ui/backtrace.rs # TODO warning
+rm tests/ui/simple_global_asm.rs # TODO add needs-asm-support
- rm tests/ui/dyn-star/dispatch-on-pin-mut.rs # TODO failed assertion in vtable::get_ptr_and_method_ref
+rm tests/ui/process/nofile-limit.rs # TODO some AArch64 linking issue
+
+rm tests/ui/stdio-is-blocking.rs # really slow with unoptimized libstd
+
+echo "[TEST] rustc test suite"
+RUST_TEST_NOCAPTURE=1 COMPILETEST_FORCE_STAGE0=1 ./x.py test --stage 0 tests/{codegen-units,run-make,run-pass-valgrind,ui,incremental}
+popd
--- /dev/null
- let call_conv = conv_to_call_conv(fn_abi.conv, default_call_conv);
+//! Handling of everything related to the calling convention. Also fills `fx.local_map`.
+
+mod comments;
+mod pass_mode;
+mod returning;
+
+use cranelift_module::ModuleError;
+use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
+use rustc_middle::ty::layout::FnAbiOf;
++use rustc_session::Session;
+use rustc_target::abi::call::{Conv, FnAbi};
+use rustc_target::spec::abi::Abi;
+
+use cranelift_codegen::ir::{AbiParam, SigRef};
+
+use self::pass_mode::*;
+use crate::prelude::*;
+
+pub(crate) use self::returning::codegen_return;
+
+fn clif_sig_from_fn_abi<'tcx>(
+ tcx: TyCtxt<'tcx>,
+ default_call_conv: CallConv,
+ fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
+) -> Signature {
- pub(crate) fn conv_to_call_conv(c: Conv, default_call_conv: CallConv) -> CallConv {
++ let call_conv = conv_to_call_conv(tcx.sess, fn_abi.conv, default_call_conv);
+
+ let inputs = fn_abi.args.iter().map(|arg_abi| arg_abi.get_abi_param(tcx).into_iter()).flatten();
+
+ let (return_ptr, returns) = fn_abi.ret.get_abi_return(tcx);
+ // Sometimes the first param is an pointer to the place where the return value needs to be stored.
+ let params: Vec<_> = return_ptr.into_iter().chain(inputs).collect();
+
+ Signature { params, returns, call_conv }
+}
+
- Conv::ArmAapcs
- | Conv::CCmseNonSecureCall
- | Conv::Msp430Intr
++pub(crate) fn conv_to_call_conv(sess: &Session, c: Conv, default_call_conv: CallConv) -> CallConv {
+ match c {
+ Conv::Rust | Conv::C => default_call_conv,
+ Conv::RustCold => CallConv::Cold,
+ Conv::X86_64SysV => CallConv::SystemV,
+ Conv::X86_64Win64 => CallConv::WindowsFastcall,
- | Conv::X86Fastcall
- | Conv::X86Intr
- | Conv::X86Stdcall
- | Conv::X86ThisCall
- | Conv::X86VectorCall
++
++ // Should already get a back compat warning
++ Conv::X86Fastcall | Conv::X86Stdcall | Conv::X86ThisCall | Conv::X86VectorCall => {
++ default_call_conv
++ }
++
++ Conv::X86Intr => sess.fatal("x86-interrupt call conv not yet implemented"),
++
++ Conv::ArmAapcs => sess.fatal("aapcs call conv not yet implemented"),
++ Conv::CCmseNonSecureCall => {
++ sess.fatal("C-cmse-nonsecure-call call conv is not yet implemented");
++ }
++
++ Conv::Msp430Intr
+ | Conv::PtxKernel
- | Conv::AvrNonBlockingInterrupt => todo!("{:?}", c),
+ | Conv::AmdGpuKernel
+ | Conv::AvrInterrupt
++ | Conv::AvrNonBlockingInterrupt => {
++ unreachable!("tried to use {c:?} call conv which only exists on an unsupported target");
++ }
+ }
+}
+
+pub(crate) fn get_function_sig<'tcx>(
+ tcx: TyCtxt<'tcx>,
+ default_call_conv: CallConv,
+ inst: Instance<'tcx>,
+) -> Signature {
+ assert!(!inst.substs.needs_infer());
+ clif_sig_from_fn_abi(
+ tcx,
+ default_call_conv,
+ &RevealAllLayoutCx(tcx).fn_abi_of_instance(inst, ty::List::empty()),
+ )
+}
+
+/// Instance must be monomorphized
+pub(crate) fn import_function<'tcx>(
+ tcx: TyCtxt<'tcx>,
+ module: &mut dyn Module,
+ inst: Instance<'tcx>,
+) -> FuncId {
+ let name = tcx.symbol_name(inst).name;
+ let sig = get_function_sig(tcx, module.target_config().default_call_conv, inst);
+ match module.declare_function(name, Linkage::Import, &sig) {
+ Ok(func_id) => func_id,
+ Err(ModuleError::IncompatibleDeclaration(_)) => tcx.sess.fatal(&format!(
+ "attempt to declare `{name}` as function, but it was already declared as static"
+ )),
+ Err(ModuleError::IncompatibleSignature(_, prev_sig, new_sig)) => tcx.sess.fatal(&format!(
+ "attempt to declare `{name}` with signature {new_sig:?}, \
+ but it was already declared with signature {prev_sig:?}"
+ )),
+ Err(err) => Err::<_, _>(err).unwrap(),
+ }
+}
+
+impl<'tcx> FunctionCx<'_, '_, 'tcx> {
+ /// Instance must be monomorphized
+ pub(crate) fn get_function_ref(&mut self, inst: Instance<'tcx>) -> FuncRef {
+ let func_id = import_function(self.tcx, self.module, inst);
+ let func_ref = self.module.declare_func_in_func(func_id, &mut self.bcx.func);
+
+ if self.clif_comments.enabled() {
+ self.add_comment(func_ref, format!("{:?}", inst));
+ }
+
+ func_ref
+ }
+
+ pub(crate) fn lib_call(
+ &mut self,
+ name: &str,
+ params: Vec<AbiParam>,
+ returns: Vec<AbiParam>,
+ args: &[Value],
+ ) -> &[Value] {
+ let sig = Signature { params, returns, call_conv: self.target_config.default_call_conv };
+ let func_id = self.module.declare_function(name, Linkage::Import, &sig).unwrap();
+ let func_ref = self.module.declare_func_in_func(func_id, &mut self.bcx.func);
+ if self.clif_comments.enabled() {
+ self.add_comment(func_ref, format!("{:?}", name));
+ }
+ let call_inst = self.bcx.ins().call(func_ref, args);
+ if self.clif_comments.enabled() {
+ self.add_comment(call_inst, format!("easy_call {}", name));
+ }
+ let results = self.bcx.inst_results(call_inst);
+ assert!(results.len() <= 2, "{}", results.len());
+ results
+ }
+
+ pub(crate) fn easy_call(
+ &mut self,
+ name: &str,
+ args: &[CValue<'tcx>],
+ return_ty: Ty<'tcx>,
+ ) -> CValue<'tcx> {
+ let (input_tys, args): (Vec<_>, Vec<_>) = args
+ .iter()
+ .map(|arg| {
+ (AbiParam::new(self.clif_type(arg.layout().ty).unwrap()), arg.load_scalar(self))
+ })
+ .unzip();
+ let return_layout = self.layout_of(return_ty);
+ let return_tys = if let ty::Tuple(tup) = return_ty.kind() {
+ tup.iter().map(|ty| AbiParam::new(self.clif_type(ty).unwrap())).collect()
+ } else {
+ vec![AbiParam::new(self.clif_type(return_ty).unwrap())]
+ };
+ let ret_vals = self.lib_call(name, input_tys, return_tys, &args);
+ match *ret_vals {
+ [] => CValue::by_ref(
+ Pointer::const_addr(self, i64::from(self.pointer_type.bytes())),
+ return_layout,
+ ),
+ [val] => CValue::by_val(val, return_layout),
+ [val, extra] => CValue::by_val_pair(val, extra, return_layout),
+ _ => unreachable!(),
+ }
+ }
+}
+
+/// Make a [`CPlace`] capable of holding value of the specified type.
+fn make_local_place<'tcx>(
+ fx: &mut FunctionCx<'_, '_, 'tcx>,
+ local: Local,
+ layout: TyAndLayout<'tcx>,
+ is_ssa: bool,
+) -> CPlace<'tcx> {
++ if layout.is_unsized() {
++ fx.tcx.sess.span_fatal(
++ fx.mir.local_decls[local].source_info.span,
++ "unsized locals are not yet supported",
++ );
++ }
+ let place = if is_ssa {
+ if let rustc_target::abi::Abi::ScalarPair(_, _) = layout.abi {
+ CPlace::new_var_pair(fx, local, layout)
+ } else {
+ CPlace::new_var(fx, local, layout)
+ }
+ } else {
+ CPlace::new_stack_slot(fx, layout)
+ };
+
+ self::comments::add_local_place_comments(fx, place, local);
+
+ place
+}
+
+pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_block: Block) {
+ fx.bcx.append_block_params_for_function_params(start_block);
+
+ fx.bcx.switch_to_block(start_block);
+ fx.bcx.ins().nop();
+
+ let ssa_analyzed = crate::analyze::analyze(fx);
+
+ self::comments::add_args_header_comment(fx);
+
+ let mut block_params_iter = fx.bcx.func.dfg.block_params(start_block).to_vec().into_iter();
+ let ret_place =
+ self::returning::codegen_return_param(fx, &ssa_analyzed, &mut block_params_iter);
+ assert_eq!(fx.local_map.push(ret_place), RETURN_PLACE);
+
+ // None means pass_mode == NoPass
+ enum ArgKind<'tcx> {
+ Normal(Option<CValue<'tcx>>),
+ Spread(Vec<Option<CValue<'tcx>>>),
+ }
+
+ let fn_abi = fx.fn_abi.take().unwrap();
+
+ // FIXME implement variadics in cranelift
+ if fn_abi.c_variadic {
+ fx.tcx.sess.span_fatal(
+ fx.mir.span,
+ "Defining variadic functions is not yet supported by Cranelift",
+ );
+ }
+
+ let mut arg_abis_iter = fn_abi.args.iter();
+
+ let func_params = fx
+ .mir
+ .args_iter()
+ .map(|local| {
+ let arg_ty = fx.monomorphize(fx.mir.local_decls[local].ty);
+
+ // Adapted from https://github.com/rust-lang/rust/blob/145155dc96757002c7b2e9de8489416e2fdbbd57/src/librustc_codegen_llvm/mir/mod.rs#L442-L482
+ if Some(local) == fx.mir.spread_arg {
+ // This argument (e.g. the last argument in the "rust-call" ABI)
+ // is a tuple that was spread at the ABI level and now we have
+ // to reconstruct it into a tuple local variable, from multiple
+ // individual function arguments.
+
+ let tupled_arg_tys = match arg_ty.kind() {
+ ty::Tuple(ref tys) => tys,
+ _ => bug!("spread argument isn't a tuple?! but {:?}", arg_ty),
+ };
+
+ let mut params = Vec::new();
+ for (i, _arg_ty) in tupled_arg_tys.iter().enumerate() {
+ let arg_abi = arg_abis_iter.next().unwrap();
+ let param =
+ cvalue_for_param(fx, Some(local), Some(i), arg_abi, &mut block_params_iter);
+ params.push(param);
+ }
+
+ (local, ArgKind::Spread(params), arg_ty)
+ } else {
+ let arg_abi = arg_abis_iter.next().unwrap();
+ let param =
+ cvalue_for_param(fx, Some(local), None, arg_abi, &mut block_params_iter);
+ (local, ArgKind::Normal(param), arg_ty)
+ }
+ })
+ .collect::<Vec<(Local, ArgKind<'tcx>, Ty<'tcx>)>>();
+
+ assert!(fx.caller_location.is_none());
+ if fx.instance.def.requires_caller_location(fx.tcx) {
+ // Store caller location for `#[track_caller]`.
+ let arg_abi = arg_abis_iter.next().unwrap();
+ fx.caller_location =
+ Some(cvalue_for_param(fx, None, None, arg_abi, &mut block_params_iter).unwrap());
+ }
+
+ assert!(arg_abis_iter.next().is_none(), "ArgAbi left behind");
+ fx.fn_abi = Some(fn_abi);
+ assert!(block_params_iter.next().is_none(), "arg_value left behind");
+
+ self::comments::add_locals_header_comment(fx);
+
+ for (local, arg_kind, ty) in func_params {
+ let layout = fx.layout_of(ty);
+
+ let is_ssa = ssa_analyzed[local] == crate::analyze::SsaKind::Ssa;
+
+ // While this is normally an optimization to prevent an unnecessary copy when an argument is
+ // not mutated by the current function, this is necessary to support unsized arguments.
+ if let ArgKind::Normal(Some(val)) = arg_kind {
+ if let Some((addr, meta)) = val.try_to_ptr() {
+ // Ownership of the value at the backing storage for an argument is passed to the
+ // callee per the ABI, so it is fine to borrow the backing storage of this argument
+ // to prevent a copy.
+
+ let place = if let Some(meta) = meta {
+ CPlace::for_ptr_with_extra(addr, meta, val.layout())
+ } else {
+ CPlace::for_ptr(addr, val.layout())
+ };
+
+ self::comments::add_local_place_comments(fx, place, local);
+
+ assert_eq!(fx.local_map.push(place), local);
+ continue;
+ }
+ }
+
+ let place = make_local_place(fx, local, layout, is_ssa);
+ assert_eq!(fx.local_map.push(place), local);
+
+ match arg_kind {
+ ArgKind::Normal(param) => {
+ if let Some(param) = param {
+ place.write_cvalue(fx, param);
+ }
+ }
+ ArgKind::Spread(params) => {
+ for (i, param) in params.into_iter().enumerate() {
+ if let Some(param) = param {
+ place.place_field(fx, mir::Field::new(i)).write_cvalue(fx, param);
+ }
+ }
+ }
+ }
+ }
+
+ for local in fx.mir.vars_and_temps_iter() {
+ let ty = fx.monomorphize(fx.mir.local_decls[local].ty);
+ let layout = fx.layout_of(ty);
+
+ let is_ssa = ssa_analyzed[local] == crate::analyze::SsaKind::Ssa;
+
+ let place = make_local_place(fx, local, layout, is_ssa);
+ assert_eq!(fx.local_map.push(place), local);
+ }
+
+ fx.bcx.ins().jump(*fx.block_map.get(START_BLOCK).unwrap(), &[]);
+}
+
+struct CallArgument<'tcx> {
+ value: CValue<'tcx>,
+ is_owned: bool,
+}
+
+// FIXME avoid intermediate `CValue` before calling `adjust_arg_for_abi`
+fn codegen_call_argument_operand<'tcx>(
+ fx: &mut FunctionCx<'_, '_, 'tcx>,
+ operand: &Operand<'tcx>,
+) -> CallArgument<'tcx> {
+ CallArgument {
+ value: codegen_operand(fx, operand),
+ is_owned: matches!(operand, Operand::Move(_)),
+ }
+}
+
+pub(crate) fn codegen_terminator_call<'tcx>(
+ fx: &mut FunctionCx<'_, '_, 'tcx>,
+ source_info: mir::SourceInfo,
+ func: &Operand<'tcx>,
+ args: &[Operand<'tcx>],
+ destination: Place<'tcx>,
+ target: Option<BasicBlock>,
+) {
+ let func = codegen_operand(fx, func);
+ let fn_sig = func.layout().ty.fn_sig(fx.tcx);
+
+ let ret_place = codegen_place(fx, destination);
+
+ // Handle special calls like intrinsics and empty drop glue.
+ let instance = if let ty::FnDef(def_id, substs) = *func.layout().ty.kind() {
+ let instance =
+ ty::Instance::expect_resolve(fx.tcx, ty::ParamEnv::reveal_all(), def_id, substs)
+ .polymorphize(fx.tcx);
+
+ if fx.tcx.symbol_name(instance).name.starts_with("llvm.") {
+ crate::intrinsics::codegen_llvm_intrinsic_call(
+ fx,
+ &fx.tcx.symbol_name(instance).name,
+ substs,
+ args,
+ ret_place,
+ target,
+ );
+ return;
+ }
+
+ match instance.def {
+ InstanceDef::Intrinsic(_) => {
+ crate::intrinsics::codegen_intrinsic_call(
+ fx,
+ instance,
+ args,
+ ret_place,
+ target,
+ source_info,
+ );
+ return;
+ }
+ InstanceDef::DropGlue(_, None) => {
+ // empty drop glue - a nop.
+ let dest = target.expect("Non terminating drop_in_place_real???");
+ let ret_block = fx.get_block(dest);
+ fx.bcx.ins().jump(ret_block, &[]);
+ return;
+ }
+ _ => Some(instance),
+ }
+ } else {
+ None
+ };
+
+ let extra_args = &args[fn_sig.inputs().skip_binder().len()..];
+ let extra_args = fx
+ .tcx
+ .mk_type_list(extra_args.iter().map(|op_arg| fx.monomorphize(op_arg.ty(fx.mir, fx.tcx))));
+ let fn_abi = if let Some(instance) = instance {
+ RevealAllLayoutCx(fx.tcx).fn_abi_of_instance(instance, extra_args)
+ } else {
+ RevealAllLayoutCx(fx.tcx).fn_abi_of_fn_ptr(fn_sig, extra_args)
+ };
+
+ let is_cold = if fn_sig.abi() == Abi::RustCold {
+ true
+ } else {
+ instance
+ .map(|inst| {
+ fx.tcx.codegen_fn_attrs(inst.def_id()).flags.contains(CodegenFnAttrFlags::COLD)
+ })
+ .unwrap_or(false)
+ };
+ if is_cold {
+ fx.bcx.set_cold_block(fx.bcx.current_block().unwrap());
+ if let Some(destination_block) = target {
+ fx.bcx.set_cold_block(fx.get_block(destination_block));
+ }
+ }
+
+ // Unpack arguments tuple for closures
+ let mut args = if fn_sig.abi() == Abi::RustCall {
+ assert_eq!(args.len(), 2, "rust-call abi requires two arguments");
+ let self_arg = codegen_call_argument_operand(fx, &args[0]);
+ let pack_arg = codegen_call_argument_operand(fx, &args[1]);
+
+ let tupled_arguments = match pack_arg.value.layout().ty.kind() {
+ ty::Tuple(ref tupled_arguments) => tupled_arguments,
+ _ => bug!("argument to function with \"rust-call\" ABI is not a tuple"),
+ };
+
+ let mut args = Vec::with_capacity(1 + tupled_arguments.len());
+ args.push(self_arg);
+ for i in 0..tupled_arguments.len() {
+ args.push(CallArgument {
+ value: pack_arg.value.value_field(fx, mir::Field::new(i)),
+ is_owned: pack_arg.is_owned,
+ });
+ }
+ args
+ } else {
+ args.iter().map(|arg| codegen_call_argument_operand(fx, arg)).collect::<Vec<_>>()
+ };
+
+ // Pass the caller location for `#[track_caller]`.
+ if instance.map(|inst| inst.def.requires_caller_location(fx.tcx)).unwrap_or(false) {
+ let caller_location = fx.get_caller_location(source_info);
+ args.push(CallArgument { value: caller_location, is_owned: false });
+ }
+
+ let args = args;
+ assert_eq!(fn_abi.args.len(), args.len());
+
+ enum CallTarget {
+ Direct(FuncRef),
+ Indirect(SigRef, Value),
+ }
+
+ let (func_ref, first_arg_override) = match instance {
+ // Trait object call
+ Some(Instance { def: InstanceDef::Virtual(_, idx), .. }) => {
+ if fx.clif_comments.enabled() {
+ let nop_inst = fx.bcx.ins().nop();
+ fx.add_comment(
+ nop_inst,
+ format!("virtual call; self arg pass mode: {:?}", &fn_abi.args[0]),
+ );
+ }
+
+ let (ptr, method) = crate::vtable::get_ptr_and_method_ref(fx, args[0].value, idx);
+ let sig = clif_sig_from_fn_abi(fx.tcx, fx.target_config.default_call_conv, &fn_abi);
+ let sig = fx.bcx.import_signature(sig);
+
+ (CallTarget::Indirect(sig, method), Some(ptr.get_addr(fx)))
+ }
+
+ // Normal call
+ Some(instance) => {
+ let func_ref = fx.get_function_ref(instance);
+ (CallTarget::Direct(func_ref), None)
+ }
+
+ // Indirect call
+ None => {
+ if fx.clif_comments.enabled() {
+ let nop_inst = fx.bcx.ins().nop();
+ fx.add_comment(nop_inst, "indirect call");
+ }
+
+ let func = func.load_scalar(fx);
+ let sig = clif_sig_from_fn_abi(fx.tcx, fx.target_config.default_call_conv, &fn_abi);
+ let sig = fx.bcx.import_signature(sig);
+
+ (CallTarget::Indirect(sig, func), None)
+ }
+ };
+
+ self::returning::codegen_with_call_return_arg(fx, &fn_abi.ret, ret_place, |fx, return_ptr| {
+ let call_args = return_ptr
+ .into_iter()
+ .chain(first_arg_override.into_iter())
+ .chain(
+ args.into_iter()
+ .enumerate()
+ .skip(if first_arg_override.is_some() { 1 } else { 0 })
+ .map(|(i, arg)| {
+ adjust_arg_for_abi(fx, arg.value, &fn_abi.args[i], arg.is_owned).into_iter()
+ })
+ .flatten(),
+ )
+ .collect::<Vec<Value>>();
+
+ let call_inst = match func_ref {
+ CallTarget::Direct(func_ref) => fx.bcx.ins().call(func_ref, &call_args),
+ CallTarget::Indirect(sig, func_ptr) => {
+ fx.bcx.ins().call_indirect(sig, func_ptr, &call_args)
+ }
+ };
+
+ // FIXME find a cleaner way to support varargs
+ if fn_sig.c_variadic() {
+ if !matches!(fn_sig.abi(), Abi::C { .. }) {
+ fx.tcx.sess.span_fatal(
+ source_info.span,
+ &format!("Variadic call for non-C abi {:?}", fn_sig.abi()),
+ );
+ }
+ let sig_ref = fx.bcx.func.dfg.call_signature(call_inst).unwrap();
+ let abi_params = call_args
+ .into_iter()
+ .map(|arg| {
+ let ty = fx.bcx.func.dfg.value_type(arg);
+ if !ty.is_int() {
+ // FIXME set %al to upperbound on float args once floats are supported
+ fx.tcx.sess.span_fatal(
+ source_info.span,
+ &format!("Non int ty {:?} for variadic call", ty),
+ );
+ }
+ AbiParam::new(ty)
+ })
+ .collect::<Vec<AbiParam>>();
+ fx.bcx.func.dfg.signatures[sig_ref].params = abi_params;
+ }
+
+ call_inst
+ });
+
+ if let Some(dest) = target {
+ let ret_block = fx.get_block(dest);
+ fx.bcx.ins().jump(ret_block, &[]);
+ } else {
+ fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
+ }
+}
+
+pub(crate) fn codegen_drop<'tcx>(
+ fx: &mut FunctionCx<'_, '_, 'tcx>,
+ source_info: mir::SourceInfo,
+ drop_place: CPlace<'tcx>,
+) {
+ let ty = drop_place.layout().ty;
+ let drop_instance = Instance::resolve_drop_in_place(fx.tcx, ty).polymorphize(fx.tcx);
+
+ if let ty::InstanceDef::DropGlue(_, None) = drop_instance.def {
+ // we don't actually need to drop anything
+ } else {
+ match ty.kind() {
+ ty::Dynamic(_, _, ty::Dyn) => {
+ // IN THIS ARM, WE HAVE:
+ // ty = *mut (dyn Trait)
+ // which is: exists<T> ( *mut T, Vtable<T: Trait> )
+ // args[0] args[1]
+ //
+ // args = ( Data, Vtable )
+ // |
+ // v
+ // /-------\
+ // | ... |
+ // \-------/
+ //
+ let (ptr, vtable) = drop_place.to_ptr_maybe_unsized();
+ let ptr = ptr.get_addr(fx);
+ let drop_fn = crate::vtable::drop_fn_of_obj(fx, vtable.unwrap());
+
+ // FIXME(eddyb) perhaps move some of this logic into
+ // `Instance::resolve_drop_in_place`?
+ let virtual_drop = Instance {
+ def: ty::InstanceDef::Virtual(drop_instance.def_id(), 0),
+ substs: drop_instance.substs,
+ };
+ let fn_abi =
+ RevealAllLayoutCx(fx.tcx).fn_abi_of_instance(virtual_drop, ty::List::empty());
+
+ let sig = clif_sig_from_fn_abi(fx.tcx, fx.target_config.default_call_conv, &fn_abi);
+ let sig = fx.bcx.import_signature(sig);
+ fx.bcx.ins().call_indirect(sig, drop_fn, &[ptr]);
+ }
+ ty::Dynamic(_, _, ty::DynStar) => {
+ // IN THIS ARM, WE HAVE:
+ // ty = *mut (dyn* Trait)
+ // which is: *mut exists<T: sizeof(T) == sizeof(usize)> (T, Vtable<T: Trait>)
+ //
+ // args = [ * ]
+ // |
+ // v
+ // ( Data, Vtable )
+ // |
+ // v
+ // /-------\
+ // | ... |
+ // \-------/
+ //
+ //
+ // WE CAN CONVERT THIS INTO THE ABOVE LOGIC BY DOING
+ //
+ // data = &(*args[0]).0 // gives a pointer to Data above (really the same pointer)
+ // vtable = (*args[0]).1 // loads the vtable out
+ // (data, vtable) // an equivalent Rust `*mut dyn Trait`
+ //
+ // SO THEN WE CAN USE THE ABOVE CODE.
+ let (data, vtable) = drop_place.to_cvalue(fx).dyn_star_force_data_on_stack(fx);
+ let drop_fn = crate::vtable::drop_fn_of_obj(fx, vtable);
+
+ let virtual_drop = Instance {
+ def: ty::InstanceDef::Virtual(drop_instance.def_id(), 0),
+ substs: drop_instance.substs,
+ };
+ let fn_abi =
+ RevealAllLayoutCx(fx.tcx).fn_abi_of_instance(virtual_drop, ty::List::empty());
+
+ let sig = clif_sig_from_fn_abi(fx.tcx, fx.target_config.default_call_conv, &fn_abi);
+ let sig = fx.bcx.import_signature(sig);
+ fx.bcx.ins().call_indirect(sig, drop_fn, &[data]);
+ }
+ _ => {
+ assert!(!matches!(drop_instance.def, InstanceDef::Virtual(_, _)));
+
+ let fn_abi =
+ RevealAllLayoutCx(fx.tcx).fn_abi_of_instance(drop_instance, ty::List::empty());
+
+ let arg_value = drop_place.place_ref(
+ fx,
+ fx.layout_of(fx.tcx.mk_ref(
+ fx.tcx.lifetimes.re_erased,
+ TypeAndMut { ty, mutbl: crate::rustc_hir::Mutability::Mut },
+ )),
+ );
+ let arg_value = adjust_arg_for_abi(fx, arg_value, &fn_abi.args[0], true);
+
+ let mut call_args: Vec<Value> = arg_value.into_iter().collect::<Vec<_>>();
+
+ if drop_instance.def.requires_caller_location(fx.tcx) {
+ // Pass the caller location for `#[track_caller]`.
+ let caller_location = fx.get_caller_location(source_info);
+ call_args.extend(
+ adjust_arg_for_abi(fx, caller_location, &fn_abi.args[1], false).into_iter(),
+ );
+ }
+
+ let func_ref = fx.get_function_ref(drop_instance);
+ fx.bcx.ins().call(func_ref, &call_args);
+ }
+ }
+ }
+}
--- /dev/null
- TerminatorKind::Resume | TerminatorKind::Abort => {
+//! Codegen of a single function
+
+use rustc_ast::InlineAsmOptions;
+use rustc_index::vec::IndexVec;
+use rustc_middle::ty::adjustment::PointerCast;
+use rustc_middle::ty::layout::FnAbiOf;
+use rustc_middle::ty::print::with_no_trimmed_paths;
+
+use cranelift_codegen::ir::UserFuncName;
+
+use crate::constant::ConstantCx;
+use crate::debuginfo::FunctionDebugContext;
+use crate::prelude::*;
+use crate::pretty_clif::CommentWriter;
+
+pub(crate) struct CodegenedFunction {
+ symbol_name: String,
+ func_id: FuncId,
+ func: Function,
+ clif_comments: CommentWriter,
+ func_debug_cx: Option<FunctionDebugContext>,
+}
+
+#[cfg_attr(not(feature = "jit"), allow(dead_code))]
+pub(crate) fn codegen_and_compile_fn<'tcx>(
+ tcx: TyCtxt<'tcx>,
+ cx: &mut crate::CodegenCx,
+ cached_context: &mut Context,
+ module: &mut dyn Module,
+ instance: Instance<'tcx>,
+) {
+ let _inst_guard =
+ crate::PrintOnPanic(|| format!("{:?} {}", instance, tcx.symbol_name(instance).name));
+
+ let cached_func = std::mem::replace(&mut cached_context.func, Function::new());
+ let codegened_func = codegen_fn(tcx, cx, cached_func, module, instance);
+
+ compile_fn(cx, cached_context, module, codegened_func);
+}
+
+pub(crate) fn codegen_fn<'tcx>(
+ tcx: TyCtxt<'tcx>,
+ cx: &mut crate::CodegenCx,
+ cached_func: Function,
+ module: &mut dyn Module,
+ instance: Instance<'tcx>,
+) -> CodegenedFunction {
+ debug_assert!(!instance.substs.needs_infer());
+
+ let mir = tcx.instance_mir(instance.def);
+ let _mir_guard = crate::PrintOnPanic(|| {
+ let mut buf = Vec::new();
+ with_no_trimmed_paths!({
+ rustc_middle::mir::pretty::write_mir_fn(tcx, mir, &mut |_, _| Ok(()), &mut buf)
+ .unwrap();
+ });
+ String::from_utf8_lossy(&buf).into_owned()
+ });
+
+ // Declare function
+ let symbol_name = tcx.symbol_name(instance).name.to_string();
+ let sig = get_function_sig(tcx, module.target_config().default_call_conv, instance);
+ let func_id = module.declare_function(&symbol_name, Linkage::Local, &sig).unwrap();
+
+ // Make the FunctionBuilder
+ let mut func_ctx = FunctionBuilderContext::new();
+ let mut func = cached_func;
+ func.clear();
+ func.name = UserFuncName::user(0, func_id.as_u32());
+ func.signature = sig;
+ func.collect_debug_info();
+
+ let mut bcx = FunctionBuilder::new(&mut func, &mut func_ctx);
+
+ // Predefine blocks
+ let start_block = bcx.create_block();
+ let block_map: IndexVec<BasicBlock, Block> =
+ (0..mir.basic_blocks.len()).map(|_| bcx.create_block()).collect();
+
+ // Make FunctionCx
+ let target_config = module.target_config();
+ let pointer_type = target_config.pointer_type();
+ let clif_comments = crate::pretty_clif::CommentWriter::new(tcx, instance);
+
+ let func_debug_cx = if let Some(debug_context) = &mut cx.debug_context {
+ Some(debug_context.define_function(tcx, &symbol_name, mir.span))
+ } else {
+ None
+ };
+
+ let mut fx = FunctionCx {
+ cx,
+ module,
+ tcx,
+ target_config,
+ pointer_type,
+ constants_cx: ConstantCx::new(),
+ func_debug_cx,
+
+ instance,
+ symbol_name,
+ mir,
+ fn_abi: Some(RevealAllLayoutCx(tcx).fn_abi_of_instance(instance, ty::List::empty())),
+
+ bcx,
+ block_map,
+ local_map: IndexVec::with_capacity(mir.local_decls.len()),
+ caller_location: None, // set by `codegen_fn_prelude`
+
+ clif_comments,
+ last_source_file: None,
+ next_ssa_var: 0,
+ };
+
+ tcx.sess.time("codegen clif ir", || codegen_fn_body(&mut fx, start_block));
++ fx.bcx.seal_all_blocks();
++ fx.bcx.finalize();
+
+ // Recover all necessary data from fx, before accessing func will prevent future access to it.
+ let symbol_name = fx.symbol_name;
+ let clif_comments = fx.clif_comments;
+ let func_debug_cx = fx.func_debug_cx;
+
+ fx.constants_cx.finalize(fx.tcx, &mut *fx.module);
+
+ if cx.should_write_ir {
+ crate::pretty_clif::write_clif_file(
+ tcx.output_filenames(()),
+ &symbol_name,
+ "unopt",
+ module.isa(),
+ &func,
+ &clif_comments,
+ );
+ }
+
+ // Verify function
+ verify_func(tcx, &clif_comments, &func);
+
+ CodegenedFunction { symbol_name, func_id, func, clif_comments, func_debug_cx }
+}
+
+pub(crate) fn compile_fn(
+ cx: &mut crate::CodegenCx,
+ cached_context: &mut Context,
+ module: &mut dyn Module,
+ codegened_func: CodegenedFunction,
+) {
+ let clif_comments = codegened_func.clif_comments;
+
+ // Store function in context
+ let context = cached_context;
+ context.clear();
+ context.func = codegened_func.func;
+
+ // If the return block is not reachable, then the SSA builder may have inserted an `iconst.i128`
+ // instruction, which doesn't have an encoding.
+ context.compute_cfg();
+ context.compute_domtree();
+ context.eliminate_unreachable_code(module.isa()).unwrap();
+ context.dce(module.isa()).unwrap();
+ // Some Cranelift optimizations expect the domtree to not yet be computed and as such don't
+ // invalidate it when it would change.
+ context.domtree.clear();
+
+ #[cfg(any())] // This is never true
+ let _clif_guard = {
+ use std::fmt::Write;
+
+ let func_clone = context.func.clone();
+ let clif_comments_clone = clif_comments.clone();
+ let mut clif = String::new();
+ for flag in module.isa().flags().iter() {
+ writeln!(clif, "set {}", flag).unwrap();
+ }
+ write!(clif, "target {}", module.isa().triple().architecture.to_string()).unwrap();
+ for isa_flag in module.isa().isa_flags().iter() {
+ write!(clif, " {}", isa_flag).unwrap();
+ }
+ writeln!(clif, "\n").unwrap();
+ crate::PrintOnPanic(move || {
+ let mut clif = clif.clone();
+ ::cranelift_codegen::write::decorate_function(
+ &mut &clif_comments_clone,
+ &mut clif,
+ &func_clone,
+ )
+ .unwrap();
+ clif
+ })
+ };
+
+ // Define function
+ cx.profiler.verbose_generic_activity("define function").run(|| {
+ context.want_disasm = cx.should_write_ir;
+ module.define_function(codegened_func.func_id, context).unwrap();
+ });
+
+ if cx.should_write_ir {
+ // Write optimized function to file for debugging
+ crate::pretty_clif::write_clif_file(
+ &cx.output_filenames,
+ &codegened_func.symbol_name,
+ "opt",
+ module.isa(),
+ &context.func,
+ &clif_comments,
+ );
+
+ if let Some(disasm) = &context.compiled_code().unwrap().disasm {
+ crate::pretty_clif::write_ir_file(
+ &cx.output_filenames,
+ &format!("{}.vcode", codegened_func.symbol_name),
+ |file| file.write_all(disasm.as_bytes()),
+ )
+ }
+ }
+
+ // Define debuginfo for function
+ let isa = module.isa();
+ let debug_context = &mut cx.debug_context;
+ let unwind_context = &mut cx.unwind_context;
+ cx.profiler.verbose_generic_activity("generate debug info").run(|| {
+ if let Some(debug_context) = debug_context {
+ codegened_func.func_debug_cx.unwrap().finalize(
+ debug_context,
+ codegened_func.func_id,
+ context,
+ );
+ }
+ unwind_context.add_function(codegened_func.func_id, &context, isa);
+ });
+}
+
+pub(crate) fn verify_func(
+ tcx: TyCtxt<'_>,
+ writer: &crate::pretty_clif::CommentWriter,
+ func: &Function,
+) {
+ tcx.sess.time("verify clif ir", || {
+ let flags = cranelift_codegen::settings::Flags::new(cranelift_codegen::settings::builder());
+ match cranelift_codegen::verify_function(&func, &flags) {
+ Ok(_) => {}
+ Err(err) => {
+ tcx.sess.err(&format!("{:?}", err));
+ let pretty_error = cranelift_codegen::print_errors::pretty_verifier_error(
+ &func,
+ Some(Box::new(writer)),
+ err,
+ );
+ tcx.sess.fatal(&format!("cranelift verify error:\n{}", pretty_error));
+ }
+ }
+ });
+}
+
+fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
+ if !crate::constant::check_constants(fx) {
+ fx.bcx.append_block_params_for_function_params(fx.block_map[START_BLOCK]);
+ fx.bcx.switch_to_block(fx.block_map[START_BLOCK]);
+ // compilation should have been aborted
+ fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
+ return;
+ }
+
+ let arg_uninhabited = fx
+ .mir
+ .args_iter()
+ .any(|arg| fx.layout_of(fx.monomorphize(fx.mir.local_decls[arg].ty)).abi.is_uninhabited());
+ if arg_uninhabited {
+ fx.bcx.append_block_params_for_function_params(fx.block_map[START_BLOCK]);
+ fx.bcx.switch_to_block(fx.block_map[START_BLOCK]);
+ fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
+ return;
+ }
+ fx.tcx.sess.time("codegen prelude", || crate::abi::codegen_fn_prelude(fx, start_block));
+
+ for (bb, bb_data) in fx.mir.basic_blocks.iter_enumerated() {
+ let block = fx.get_block(bb);
+ fx.bcx.switch_to_block(block);
+
+ if bb_data.is_cleanup {
+ // Unwinding after panicking is not supported
+ continue;
+
+ // FIXME Once unwinding is supported and Cranelift supports marking blocks as cold, do
+ // so for cleanup blocks.
+ }
+
+ fx.bcx.ins().nop();
+ for stmt in &bb_data.statements {
+ fx.set_debug_loc(stmt.source_info);
+ codegen_stmt(fx, block, stmt);
+ }
+
+ if fx.clif_comments.enabled() {
+ let mut terminator_head = "\n".to_string();
+ with_no_trimmed_paths!({
+ bb_data.terminator().kind.fmt_head(&mut terminator_head).unwrap();
+ });
+ let inst = fx.bcx.func.layout.last_inst(block).unwrap();
+ fx.add_comment(inst, terminator_head);
+ }
+
+ let source_info = bb_data.terminator().source_info;
+ fx.set_debug_loc(source_info);
+
++ let _print_guard =
++ crate::PrintOnPanic(|| format!("terminator {:?}", bb_data.terminator().kind));
++
+ match &bb_data.terminator().kind {
+ TerminatorKind::Goto { target } => {
+ if let TerminatorKind::Return = fx.mir[*target].terminator().kind {
+ let mut can_immediately_return = true;
+ for stmt in &fx.mir[*target].statements {
+ if let StatementKind::StorageDead(_) = stmt.kind {
+ } else {
+ // FIXME Can sometimes happen, see rust-lang/rust#70531
+ can_immediately_return = false;
+ break;
+ }
+ }
+
+ if can_immediately_return {
+ crate::abi::codegen_return(fx);
+ continue;
+ }
+ }
+
+ let block = fx.get_block(*target);
+ fx.bcx.ins().jump(block, &[]);
+ }
+ TerminatorKind::Return => {
+ crate::abi::codegen_return(fx);
+ }
+ TerminatorKind::Assert { cond, expected, msg, target, cleanup: _ } => {
+ if !fx.tcx.sess.overflow_checks() {
+ if let mir::AssertKind::OverflowNeg(_) = *msg {
+ let target = fx.get_block(*target);
+ fx.bcx.ins().jump(target, &[]);
+ continue;
+ }
+ }
+ let cond = codegen_operand(fx, cond).load_scalar(fx);
+
+ let target = fx.get_block(*target);
+ let failure = fx.bcx.create_block();
+ fx.bcx.set_cold_block(failure);
+
+ if *expected {
+ fx.bcx.ins().brz(cond, failure, &[]);
+ } else {
+ fx.bcx.ins().brnz(cond, failure, &[]);
+ };
+ fx.bcx.ins().jump(target, &[]);
+
+ fx.bcx.switch_to_block(failure);
+ fx.bcx.ins().nop();
+
+ match msg {
+ AssertKind::BoundsCheck { ref len, ref index } => {
+ let len = codegen_operand(fx, len).load_scalar(fx);
+ let index = codegen_operand(fx, index).load_scalar(fx);
+ let location = fx.get_caller_location(source_info).load_scalar(fx);
+
+ codegen_panic_inner(
+ fx,
+ rustc_hir::LangItem::PanicBoundsCheck,
+ &[index, len, location],
+ source_info.span,
+ );
+ }
+ _ => {
+ let msg_str = msg.description();
+ codegen_panic(fx, msg_str, source_info);
+ }
+ }
+ }
+
+ TerminatorKind::SwitchInt { discr, targets } => {
+ let discr = codegen_operand(fx, discr);
+ let switch_ty = discr.layout().ty;
+ let discr = discr.load_scalar(fx);
+
+ let use_bool_opt = switch_ty.kind() == fx.tcx.types.bool.kind()
+ || (targets.iter().count() == 1 && targets.iter().next().unwrap().0 == 0);
+ if use_bool_opt {
+ assert_eq!(targets.iter().count(), 1);
+ let (then_value, then_block) = targets.iter().next().unwrap();
+ let then_block = fx.get_block(then_block);
+ let else_block = fx.get_block(targets.otherwise());
+ let test_zero = match then_value {
+ 0 => true,
+ 1 => false,
+ _ => unreachable!("{:?}", targets),
+ };
+
+ let (discr, is_inverted) =
+ crate::optimize::peephole::maybe_unwrap_bool_not(&mut fx.bcx, discr);
+ let test_zero = if is_inverted { !test_zero } else { test_zero };
+ if let Some(taken) = crate::optimize::peephole::maybe_known_branch_taken(
+ &fx.bcx, discr, test_zero,
+ ) {
+ if taken {
+ fx.bcx.ins().jump(then_block, &[]);
+ } else {
+ fx.bcx.ins().jump(else_block, &[]);
+ }
+ } else {
+ if test_zero {
+ fx.bcx.ins().brz(discr, then_block, &[]);
+ fx.bcx.ins().jump(else_block, &[]);
+ } else {
+ fx.bcx.ins().brnz(discr, then_block, &[]);
+ fx.bcx.ins().jump(else_block, &[]);
+ }
+ }
+ } else {
+ let mut switch = ::cranelift_frontend::Switch::new();
+ for (value, block) in targets.iter() {
+ let block = fx.get_block(block);
+ switch.set_entry(value, block);
+ }
+ let otherwise_block = fx.get_block(targets.otherwise());
+ switch.emit(&mut fx.bcx, discr, otherwise_block);
+ }
+ }
+ TerminatorKind::Call {
+ func,
+ args,
+ destination,
+ target,
+ fn_span,
+ cleanup: _,
+ from_hir_call: _,
+ } => {
+ fx.tcx.sess.time("codegen call", || {
+ crate::abi::codegen_terminator_call(
+ fx,
+ mir::SourceInfo { span: *fn_span, ..source_info },
+ func,
+ args,
+ *destination,
+ *target,
+ )
+ });
+ }
+ TerminatorKind::InlineAsm {
+ template,
+ operands,
+ options,
+ destination,
+ line_spans: _,
+ cleanup: _,
+ } => {
+ if options.contains(InlineAsmOptions::MAY_UNWIND) {
+ fx.tcx.sess.span_fatal(
+ source_info.span,
+ "cranelift doesn't support unwinding from inline assembly.",
+ );
+ }
+
+ crate::inline_asm::codegen_inline_asm(
+ fx,
+ source_info.span,
+ template,
+ operands,
+ *options,
+ *destination,
+ );
+ }
-
- fx.bcx.seal_all_blocks();
- fx.bcx.finalize();
++ TerminatorKind::Abort => {
++ codegen_panic_cannot_unwind(fx, source_info);
++ }
++ TerminatorKind::Resume => {
+ // FIXME implement unwinding
+ fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
+ }
+ TerminatorKind::Unreachable => {
+ fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
+ }
+ TerminatorKind::Yield { .. }
+ | TerminatorKind::FalseEdge { .. }
+ | TerminatorKind::FalseUnwind { .. }
+ | TerminatorKind::DropAndReplace { .. }
+ | TerminatorKind::GeneratorDrop => {
+ bug!("shouldn't exist at codegen {:?}", bb_data.terminator());
+ }
+ TerminatorKind::Drop { place, target, unwind: _ } => {
+ let drop_place = codegen_place(fx, *place);
+ crate::abi::codegen_drop(fx, source_info, drop_place);
+
+ let target_block = fx.get_block(*target);
+ fx.bcx.ins().jump(target_block, &[]);
+ }
+ };
+ }
- pub(crate) fn codegen_panic_inner<'tcx>(
+}
+
+fn codegen_stmt<'tcx>(
+ fx: &mut FunctionCx<'_, '_, 'tcx>,
+ #[allow(unused_variables)] cur_block: Block,
+ stmt: &Statement<'tcx>,
+) {
+ let _print_guard = crate::PrintOnPanic(|| format!("stmt {:?}", stmt));
+
+ fx.set_debug_loc(stmt.source_info);
+
+ #[cfg(any())] // This is never true
+ match &stmt.kind {
+ StatementKind::StorageLive(..) | StatementKind::StorageDead(..) => {} // Those are not very useful
+ _ => {
+ if fx.clif_comments.enabled() {
+ let inst = fx.bcx.func.layout.last_inst(cur_block).unwrap();
+ fx.add_comment(inst, format!("{:?}", stmt));
+ }
+ }
+ }
+
+ match &stmt.kind {
+ StatementKind::SetDiscriminant { place, variant_index } => {
+ let place = codegen_place(fx, **place);
+ crate::discriminant::codegen_set_discriminant(fx, place, *variant_index);
+ }
+ StatementKind::Assign(to_place_and_rval) => {
+ let lval = codegen_place(fx, to_place_and_rval.0);
+ let dest_layout = lval.layout();
+ match to_place_and_rval.1 {
+ Rvalue::Use(ref operand) => {
+ let val = codegen_operand(fx, operand);
+ lval.write_cvalue(fx, val);
+ }
+ Rvalue::CopyForDeref(place) => {
+ let cplace = codegen_place(fx, place);
+ let val = cplace.to_cvalue(fx);
+ lval.write_cvalue(fx, val)
+ }
+ Rvalue::Ref(_, _, place) | Rvalue::AddressOf(_, place) => {
+ let place = codegen_place(fx, place);
+ let ref_ = place.place_ref(fx, lval.layout());
+ lval.write_cvalue(fx, ref_);
+ }
+ Rvalue::ThreadLocalRef(def_id) => {
+ let val = crate::constant::codegen_tls_ref(fx, def_id, lval.layout());
+ lval.write_cvalue(fx, val);
+ }
+ Rvalue::BinaryOp(bin_op, ref lhs_rhs) => {
+ let lhs = codegen_operand(fx, &lhs_rhs.0);
+ let rhs = codegen_operand(fx, &lhs_rhs.1);
+
+ let res = crate::num::codegen_binop(fx, bin_op, lhs, rhs);
+ lval.write_cvalue(fx, res);
+ }
+ Rvalue::CheckedBinaryOp(bin_op, ref lhs_rhs) => {
+ let lhs = codegen_operand(fx, &lhs_rhs.0);
+ let rhs = codegen_operand(fx, &lhs_rhs.1);
+
+ let res = if !fx.tcx.sess.overflow_checks() {
+ let val =
+ crate::num::codegen_int_binop(fx, bin_op, lhs, rhs).load_scalar(fx);
+ let is_overflow = fx.bcx.ins().iconst(types::I8, 0);
+ CValue::by_val_pair(val, is_overflow, lval.layout())
+ } else {
+ crate::num::codegen_checked_int_binop(fx, bin_op, lhs, rhs)
+ };
+
+ lval.write_cvalue(fx, res);
+ }
+ Rvalue::UnaryOp(un_op, ref operand) => {
+ let operand = codegen_operand(fx, operand);
+ let layout = operand.layout();
+ let val = operand.load_scalar(fx);
+ let res = match un_op {
+ UnOp::Not => match layout.ty.kind() {
+ ty::Bool => {
+ let res = fx.bcx.ins().icmp_imm(IntCC::Equal, val, 0);
+ CValue::by_val(res, layout)
+ }
+ ty::Uint(_) | ty::Int(_) => {
+ CValue::by_val(fx.bcx.ins().bnot(val), layout)
+ }
+ _ => unreachable!("un op Not for {:?}", layout.ty),
+ },
+ UnOp::Neg => match layout.ty.kind() {
+ ty::Int(_) => CValue::by_val(fx.bcx.ins().ineg(val), layout),
+ ty::Float(_) => CValue::by_val(fx.bcx.ins().fneg(val), layout),
+ _ => unreachable!("un op Neg for {:?}", layout.ty),
+ },
+ };
+ lval.write_cvalue(fx, res);
+ }
+ Rvalue::Cast(
+ CastKind::Pointer(PointerCast::ReifyFnPointer),
+ ref operand,
+ to_ty,
+ ) => {
+ let from_ty = fx.monomorphize(operand.ty(&fx.mir.local_decls, fx.tcx));
+ let to_layout = fx.layout_of(fx.monomorphize(to_ty));
+ match *from_ty.kind() {
+ ty::FnDef(def_id, substs) => {
+ let func_ref = fx.get_function_ref(
+ Instance::resolve_for_fn_ptr(
+ fx.tcx,
+ ParamEnv::reveal_all(),
+ def_id,
+ substs,
+ )
+ .unwrap()
+ .polymorphize(fx.tcx),
+ );
+ let func_addr = fx.bcx.ins().func_addr(fx.pointer_type, func_ref);
+ lval.write_cvalue(fx, CValue::by_val(func_addr, to_layout));
+ }
+ _ => bug!("Trying to ReifyFnPointer on non FnDef {:?}", from_ty),
+ }
+ }
+ Rvalue::Cast(
+ CastKind::Pointer(PointerCast::UnsafeFnPointer),
+ ref operand,
+ to_ty,
+ )
+ | Rvalue::Cast(
+ CastKind::Pointer(PointerCast::MutToConstPointer),
+ ref operand,
+ to_ty,
+ )
+ | Rvalue::Cast(
+ CastKind::Pointer(PointerCast::ArrayToPointer),
+ ref operand,
+ to_ty,
+ ) => {
+ let to_layout = fx.layout_of(fx.monomorphize(to_ty));
+ let operand = codegen_operand(fx, operand);
+ lval.write_cvalue(fx, operand.cast_pointer_to(to_layout));
+ }
+ Rvalue::Cast(
+ CastKind::IntToInt
+ | CastKind::FloatToFloat
+ | CastKind::FloatToInt
+ | CastKind::IntToFloat
+ | CastKind::FnPtrToPtr
+ | CastKind::PtrToPtr
+ | CastKind::PointerExposeAddress
+ | CastKind::PointerFromExposedAddress,
+ ref operand,
+ to_ty,
+ ) => {
+ let operand = codegen_operand(fx, operand);
+ let from_ty = operand.layout().ty;
+ let to_ty = fx.monomorphize(to_ty);
+
+ fn is_fat_ptr<'tcx>(fx: &FunctionCx<'_, '_, 'tcx>, ty: Ty<'tcx>) -> bool {
+ ty.builtin_deref(true)
+ .map(|ty::TypeAndMut { ty: pointee_ty, mutbl: _ }| {
+ has_ptr_meta(fx.tcx, pointee_ty)
+ })
+ .unwrap_or(false)
+ }
+
+ if is_fat_ptr(fx, from_ty) {
+ if is_fat_ptr(fx, to_ty) {
+ // fat-ptr -> fat-ptr
+ lval.write_cvalue(fx, operand.cast_pointer_to(dest_layout));
+ } else {
+ // fat-ptr -> thin-ptr
+ let (ptr, _extra) = operand.load_scalar_pair(fx);
+ lval.write_cvalue(fx, CValue::by_val(ptr, dest_layout))
+ }
+ } else {
+ let to_clif_ty = fx.clif_type(to_ty).unwrap();
+ let from = operand.load_scalar(fx);
+
+ let res = clif_int_or_float_cast(
+ fx,
+ from,
+ type_sign(from_ty),
+ to_clif_ty,
+ type_sign(to_ty),
+ );
+ lval.write_cvalue(fx, CValue::by_val(res, dest_layout));
+ }
+ }
+ Rvalue::Cast(
+ CastKind::Pointer(PointerCast::ClosureFnPointer(_)),
+ ref operand,
+ _to_ty,
+ ) => {
+ let operand = codegen_operand(fx, operand);
+ match *operand.layout().ty.kind() {
+ ty::Closure(def_id, substs) => {
+ let instance = Instance::resolve_closure(
+ fx.tcx,
+ def_id,
+ substs,
+ ty::ClosureKind::FnOnce,
+ )
+ .expect("failed to normalize and resolve closure during codegen")
+ .polymorphize(fx.tcx);
+ let func_ref = fx.get_function_ref(instance);
+ let func_addr = fx.bcx.ins().func_addr(fx.pointer_type, func_ref);
+ lval.write_cvalue(fx, CValue::by_val(func_addr, lval.layout()));
+ }
+ _ => bug!("{} cannot be cast to a fn ptr", operand.layout().ty),
+ }
+ }
+ Rvalue::Cast(CastKind::Pointer(PointerCast::Unsize), ref operand, _to_ty) => {
+ let operand = codegen_operand(fx, operand);
+ operand.unsize_value(fx, lval);
+ }
+ Rvalue::Cast(CastKind::DynStar, ref operand, _) => {
+ let operand = codegen_operand(fx, operand);
+ operand.coerce_dyn_star(fx, lval);
+ }
+ Rvalue::Discriminant(place) => {
+ let place = codegen_place(fx, place);
+ let value = place.to_cvalue(fx);
+ crate::discriminant::codegen_get_discriminant(fx, lval, value, dest_layout);
+ }
+ Rvalue::Repeat(ref operand, times) => {
+ let operand = codegen_operand(fx, operand);
+ let times = fx
+ .monomorphize(times)
+ .eval(fx.tcx, ParamEnv::reveal_all())
+ .kind()
+ .try_to_bits(fx.tcx.data_layout.pointer_size)
+ .unwrap();
+ if operand.layout().size.bytes() == 0 {
+ // Do nothing for ZST's
+ } else if fx.clif_type(operand.layout().ty) == Some(types::I8) {
+ let times = fx.bcx.ins().iconst(fx.pointer_type, times as i64);
+ // FIXME use emit_small_memset where possible
+ let addr = lval.to_ptr().get_addr(fx);
+ let val = operand.load_scalar(fx);
+ fx.bcx.call_memset(fx.target_config, addr, val, times);
+ } else {
+ let loop_block = fx.bcx.create_block();
+ let loop_block2 = fx.bcx.create_block();
+ let done_block = fx.bcx.create_block();
+ let index = fx.bcx.append_block_param(loop_block, fx.pointer_type);
+ let zero = fx.bcx.ins().iconst(fx.pointer_type, 0);
+ fx.bcx.ins().jump(loop_block, &[zero]);
+
+ fx.bcx.switch_to_block(loop_block);
+ let done = fx.bcx.ins().icmp_imm(IntCC::Equal, index, times as i64);
+ fx.bcx.ins().brnz(done, done_block, &[]);
+ fx.bcx.ins().jump(loop_block2, &[]);
+
+ fx.bcx.switch_to_block(loop_block2);
+ let to = lval.place_index(fx, index);
+ to.write_cvalue(fx, operand);
+ let index = fx.bcx.ins().iadd_imm(index, 1);
+ fx.bcx.ins().jump(loop_block, &[index]);
+
+ fx.bcx.switch_to_block(done_block);
+ fx.bcx.ins().nop();
+ }
+ }
+ Rvalue::Len(place) => {
+ let place = codegen_place(fx, place);
+ let usize_layout = fx.layout_of(fx.tcx.types.usize);
+ let len = codegen_array_len(fx, place);
+ lval.write_cvalue(fx, CValue::by_val(len, usize_layout));
+ }
+ Rvalue::ShallowInitBox(ref operand, content_ty) => {
+ let content_ty = fx.monomorphize(content_ty);
+ let box_layout = fx.layout_of(fx.tcx.mk_box(content_ty));
+ let operand = codegen_operand(fx, operand);
+ let operand = operand.load_scalar(fx);
+ lval.write_cvalue(fx, CValue::by_val(operand, box_layout));
+ }
+ Rvalue::NullaryOp(null_op, ty) => {
+ assert!(lval.layout().ty.is_sized(fx.tcx, ParamEnv::reveal_all()));
+ let layout = fx.layout_of(fx.monomorphize(ty));
+ let val = match null_op {
+ NullOp::SizeOf => layout.size.bytes(),
+ NullOp::AlignOf => layout.align.abi.bytes(),
+ };
+ let val = CValue::const_val(fx, fx.layout_of(fx.tcx.types.usize), val.into());
+ lval.write_cvalue(fx, val);
+ }
+ Rvalue::Aggregate(ref kind, ref operands) => match kind.as_ref() {
+ AggregateKind::Array(_ty) => {
+ for (i, operand) in operands.iter().enumerate() {
+ let operand = codegen_operand(fx, operand);
+ let index = fx.bcx.ins().iconst(fx.pointer_type, i as i64);
+ let to = lval.place_index(fx, index);
+ to.write_cvalue(fx, operand);
+ }
+ }
+ _ => unreachable!("shouldn't exist at codegen {:?}", to_place_and_rval.1),
+ },
+ }
+ }
+ StatementKind::StorageLive(_)
+ | StatementKind::StorageDead(_)
+ | StatementKind::Deinit(_)
+ | StatementKind::Nop
+ | StatementKind::FakeRead(..)
+ | StatementKind::Retag { .. }
+ | StatementKind::AscribeUserType(..) => {}
+
+ StatementKind::Coverage { .. } => fx.tcx.sess.fatal("-Zcoverage is unimplemented"),
+ StatementKind::Intrinsic(ref intrinsic) => match &**intrinsic {
+ // We ignore `assume` intrinsics, they are only useful for optimizations
+ NonDivergingIntrinsic::Assume(_) => {}
+ NonDivergingIntrinsic::CopyNonOverlapping(mir::CopyNonOverlapping {
+ src,
+ dst,
+ count,
+ }) => {
+ let dst = codegen_operand(fx, dst);
+ let pointee = dst
+ .layout()
+ .pointee_info_at(fx, rustc_target::abi::Size::ZERO)
+ .expect("Expected pointer");
+ let dst = dst.load_scalar(fx);
+ let src = codegen_operand(fx, src).load_scalar(fx);
+ let count = codegen_operand(fx, count).load_scalar(fx);
+ let elem_size: u64 = pointee.size.bytes();
+ let bytes = if elem_size != 1 {
+ fx.bcx.ins().imul_imm(count, elem_size as i64)
+ } else {
+ count
+ };
+ fx.bcx.call_memcpy(fx.target_config, dst, src, bytes);
+ }
+ },
+ }
+}
+
+fn codegen_array_len<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, place: CPlace<'tcx>) -> Value {
+ match *place.layout().ty.kind() {
+ ty::Array(_elem_ty, len) => {
+ let len = fx.monomorphize(len).eval_usize(fx.tcx, ParamEnv::reveal_all()) as i64;
+ fx.bcx.ins().iconst(fx.pointer_type, len)
+ }
+ ty::Slice(_elem_ty) => {
+ place.to_ptr_maybe_unsized().1.expect("Length metadata for slice place")
+ }
+ _ => bug!("Rvalue::Len({:?})", place),
+ }
+}
+
+pub(crate) fn codegen_place<'tcx>(
+ fx: &mut FunctionCx<'_, '_, 'tcx>,
+ place: Place<'tcx>,
+) -> CPlace<'tcx> {
+ let mut cplace = fx.get_local_place(place.local);
+
+ for elem in place.projection {
+ match elem {
+ PlaceElem::Deref => {
+ cplace = cplace.place_deref(fx);
+ }
+ PlaceElem::OpaqueCast(ty) => cplace = cplace.place_opaque_cast(fx, ty),
+ PlaceElem::Field(field, _ty) => {
+ cplace = cplace.place_field(fx, field);
+ }
+ PlaceElem::Index(local) => {
+ let index = fx.get_local_place(local).to_cvalue(fx).load_scalar(fx);
+ cplace = cplace.place_index(fx, index);
+ }
+ PlaceElem::ConstantIndex { offset, min_length: _, from_end } => {
+ let offset: u64 = offset;
+ let index = if !from_end {
+ fx.bcx.ins().iconst(fx.pointer_type, offset as i64)
+ } else {
+ let len = codegen_array_len(fx, cplace);
+ fx.bcx.ins().iadd_imm(len, -(offset as i64))
+ };
+ cplace = cplace.place_index(fx, index);
+ }
+ PlaceElem::Subslice { from, to, from_end } => {
+ // These indices are generated by slice patterns.
+ // slice[from:-to] in Python terms.
+
+ let from: u64 = from;
+ let to: u64 = to;
+
+ match cplace.layout().ty.kind() {
+ ty::Array(elem_ty, _len) => {
+ assert!(!from_end, "array subslices are never `from_end`");
+ let elem_layout = fx.layout_of(*elem_ty);
+ let ptr = cplace.to_ptr();
+ cplace = CPlace::for_ptr(
+ ptr.offset_i64(fx, elem_layout.size.bytes() as i64 * (from as i64)),
+ fx.layout_of(fx.tcx.mk_array(*elem_ty, to - from)),
+ );
+ }
+ ty::Slice(elem_ty) => {
+ assert!(from_end, "slice subslices should be `from_end`");
+ let elem_layout = fx.layout_of(*elem_ty);
+ let (ptr, len) = cplace.to_ptr_maybe_unsized();
+ let len = len.unwrap();
+ cplace = CPlace::for_ptr_with_extra(
+ ptr.offset_i64(fx, elem_layout.size.bytes() as i64 * (from as i64)),
+ fx.bcx.ins().iadd_imm(len, -(from as i64 + to as i64)),
+ cplace.layout(),
+ );
+ }
+ _ => unreachable!(),
+ }
+ }
+ PlaceElem::Downcast(_adt_def, variant) => {
+ cplace = cplace.downcast_variant(fx, variant);
+ }
+ }
+ }
+
+ cplace
+}
+
+pub(crate) fn codegen_operand<'tcx>(
+ fx: &mut FunctionCx<'_, '_, 'tcx>,
+ operand: &Operand<'tcx>,
+) -> CValue<'tcx> {
+ match operand {
+ Operand::Move(place) | Operand::Copy(place) => {
+ let cplace = codegen_place(fx, *place);
+ cplace.to_cvalue(fx)
+ }
+ Operand::Constant(const_) => crate::constant::codegen_constant_operand(fx, const_),
+ }
+}
+
+pub(crate) fn codegen_panic<'tcx>(
+ fx: &mut FunctionCx<'_, '_, 'tcx>,
+ msg_str: &str,
+ source_info: mir::SourceInfo,
+) {
+ let location = fx.get_caller_location(source_info).load_scalar(fx);
+
+ let msg_ptr = fx.anonymous_str(msg_str);
+ let msg_len = fx.bcx.ins().iconst(fx.pointer_type, i64::try_from(msg_str.len()).unwrap());
+ let args = [msg_ptr, msg_len, location];
+
+ codegen_panic_inner(fx, rustc_hir::LangItem::Panic, &args, source_info.span);
+}
+
- vec![
- AbiParam::new(fx.pointer_type),
- AbiParam::new(fx.pointer_type),
- AbiParam::new(fx.pointer_type),
- ],
++pub(crate) fn codegen_panic_nounwind<'tcx>(
++ fx: &mut FunctionCx<'_, '_, 'tcx>,
++ msg_str: &str,
++ source_info: mir::SourceInfo,
++) {
++ let msg_ptr = fx.anonymous_str(msg_str);
++ let msg_len = fx.bcx.ins().iconst(fx.pointer_type, i64::try_from(msg_str.len()).unwrap());
++ let args = [msg_ptr, msg_len];
++
++ codegen_panic_inner(fx, rustc_hir::LangItem::PanicNounwind, &args, source_info.span);
++}
++
++pub(crate) fn codegen_panic_cannot_unwind<'tcx>(
++ fx: &mut FunctionCx<'_, '_, 'tcx>,
++ source_info: mir::SourceInfo,
++) {
++ let args = [];
++
++ codegen_panic_inner(fx, rustc_hir::LangItem::PanicCannotUnwind, &args, source_info.span);
++}
++
++fn codegen_panic_inner<'tcx>(
+ fx: &mut FunctionCx<'_, '_, 'tcx>,
+ lang_item: rustc_hir::LangItem,
+ args: &[Value],
+ span: Span,
+) {
+ let def_id = fx
+ .tcx
+ .lang_items()
+ .require(lang_item)
+ .unwrap_or_else(|e| fx.tcx.sess.span_fatal(span, e.to_string()));
+
+ let instance = Instance::mono(fx.tcx, def_id).polymorphize(fx.tcx);
+ let symbol_name = fx.tcx.symbol_name(instance).name;
+
+ fx.lib_call(
+ &*symbol_name,
++ args.iter().map(|&arg| AbiParam::new(fx.bcx.func.dfg.value_type(arg))).collect(),
+ vec![],
+ args,
+ );
+
+ fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
+}
--- /dev/null
+use cranelift_codegen::isa::TargetFrontendConfig;
+use gimli::write::FileId;
+
+use rustc_data_structures::sync::Lrc;
+use rustc_index::vec::IndexVec;
+use rustc_middle::ty::layout::{
+ FnAbiError, FnAbiOfHelpers, FnAbiRequest, LayoutError, LayoutOfHelpers,
+};
+use rustc_span::SourceFile;
+use rustc_target::abi::call::FnAbi;
+use rustc_target::abi::{Integer, Primitive};
+use rustc_target::spec::{HasTargetSpec, Target};
+
+use crate::constant::ConstantCx;
+use crate::debuginfo::FunctionDebugContext;
+use crate::prelude::*;
+
+pub(crate) fn pointer_ty(tcx: TyCtxt<'_>) -> types::Type {
+ match tcx.data_layout.pointer_size.bits() {
+ 16 => types::I16,
+ 32 => types::I32,
+ 64 => types::I64,
+ bits => bug!("ptr_sized_integer: unknown pointer bit size {}", bits),
+ }
+}
+
+pub(crate) fn scalar_to_clif_type(tcx: TyCtxt<'_>, scalar: Scalar) -> Type {
+ match scalar.primitive() {
+ Primitive::Int(int, _sign) => match int {
+ Integer::I8 => types::I8,
+ Integer::I16 => types::I16,
+ Integer::I32 => types::I32,
+ Integer::I64 => types::I64,
+ Integer::I128 => types::I128,
+ },
+ Primitive::F32 => types::F32,
+ Primitive::F64 => types::F64,
+ Primitive::Pointer => pointer_ty(tcx),
+ }
+}
+
+fn clif_type_from_ty<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> Option<types::Type> {
+ Some(match ty.kind() {
+ ty::Bool => types::I8,
+ ty::Uint(size) => match size {
+ UintTy::U8 => types::I8,
+ UintTy::U16 => types::I16,
+ UintTy::U32 => types::I32,
+ UintTy::U64 => types::I64,
+ UintTy::U128 => types::I128,
+ UintTy::Usize => pointer_ty(tcx),
+ },
+ ty::Int(size) => match size {
+ IntTy::I8 => types::I8,
+ IntTy::I16 => types::I16,
+ IntTy::I32 => types::I32,
+ IntTy::I64 => types::I64,
+ IntTy::I128 => types::I128,
+ IntTy::Isize => pointer_ty(tcx),
+ },
+ ty::Char => types::I32,
+ ty::Float(size) => match size {
+ FloatTy::F32 => types::F32,
+ FloatTy::F64 => types::F64,
+ },
+ ty::FnPtr(_) => pointer_ty(tcx),
+ ty::RawPtr(TypeAndMut { ty: pointee_ty, mutbl: _ }) | ty::Ref(_, pointee_ty, _) => {
+ if has_ptr_meta(tcx, *pointee_ty) {
+ return None;
+ } else {
+ pointer_ty(tcx)
+ }
+ }
+ ty::Adt(adt_def, _) if adt_def.repr().simd() => {
+ let (element, count) = match &tcx.layout_of(ParamEnv::reveal_all().and(ty)).unwrap().abi
+ {
+ Abi::Vector { element, count } => (element.clone(), *count),
+ _ => unreachable!(),
+ };
+
+ match scalar_to_clif_type(tcx, element).by(u32::try_from(count).unwrap()) {
+ // Cranelift currently only implements icmp for 128bit vectors.
+ Some(vector_ty) if vector_ty.bits() == 128 => vector_ty,
+ _ => return None,
+ }
+ }
+ ty::Param(_) => bug!("ty param {:?}", ty),
+ _ => return None,
+ })
+}
+
+fn clif_pair_type_from_ty<'tcx>(
+ tcx: TyCtxt<'tcx>,
+ ty: Ty<'tcx>,
+) -> Option<(types::Type, types::Type)> {
+ Some(match ty.kind() {
+ ty::Tuple(types) if types.len() == 2 => {
+ let a = clif_type_from_ty(tcx, types[0])?;
+ let b = clif_type_from_ty(tcx, types[1])?;
+ if a.is_vector() || b.is_vector() {
+ return None;
+ }
+ (a, b)
+ }
+ ty::RawPtr(TypeAndMut { ty: pointee_ty, mutbl: _ }) | ty::Ref(_, pointee_ty, _) => {
+ if has_ptr_meta(tcx, *pointee_ty) {
+ (pointer_ty(tcx), pointer_ty(tcx))
+ } else {
+ return None;
+ }
+ }
+ _ => return None,
+ })
+}
+
+/// Is a pointer to this type a fat ptr?
+pub(crate) fn has_ptr_meta<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> bool {
+ let ptr_ty = tcx.mk_ptr(TypeAndMut { ty, mutbl: rustc_hir::Mutability::Not });
+ match &tcx.layout_of(ParamEnv::reveal_all().and(ptr_ty)).unwrap().abi {
+ Abi::Scalar(_) => false,
+ Abi::ScalarPair(_, _) => true,
+ abi => unreachable!("Abi of ptr to {:?} is {:?}???", ty, abi),
+ }
+}
+
+pub(crate) fn codegen_icmp_imm(
+ fx: &mut FunctionCx<'_, '_, '_>,
+ intcc: IntCC,
+ lhs: Value,
+ rhs: i128,
+) -> Value {
+ let lhs_ty = fx.bcx.func.dfg.value_type(lhs);
+ if lhs_ty == types::I128 {
+ // FIXME legalize `icmp_imm.i128` in Cranelift
+
+ let (lhs_lsb, lhs_msb) = fx.bcx.ins().isplit(lhs);
+ let (rhs_lsb, rhs_msb) = (rhs as u128 as u64 as i64, (rhs as u128 >> 64) as u64 as i64);
+
+ match intcc {
+ IntCC::Equal => {
+ let lsb_eq = fx.bcx.ins().icmp_imm(IntCC::Equal, lhs_lsb, rhs_lsb);
+ let msb_eq = fx.bcx.ins().icmp_imm(IntCC::Equal, lhs_msb, rhs_msb);
+ fx.bcx.ins().band(lsb_eq, msb_eq)
+ }
+ IntCC::NotEqual => {
+ let lsb_ne = fx.bcx.ins().icmp_imm(IntCC::NotEqual, lhs_lsb, rhs_lsb);
+ let msb_ne = fx.bcx.ins().icmp_imm(IntCC::NotEqual, lhs_msb, rhs_msb);
+ fx.bcx.ins().bor(lsb_ne, msb_ne)
+ }
+ _ => {
+ // if msb_eq {
+ // lsb_cc
+ // } else {
+ // msb_cc
+ // }
+
+ let msb_eq = fx.bcx.ins().icmp_imm(IntCC::Equal, lhs_msb, rhs_msb);
+ let lsb_cc = fx.bcx.ins().icmp_imm(intcc, lhs_lsb, rhs_lsb);
+ let msb_cc = fx.bcx.ins().icmp_imm(intcc, lhs_msb, rhs_msb);
+
+ fx.bcx.ins().select(msb_eq, lsb_cc, msb_cc)
+ }
+ }
+ } else {
+ let rhs = rhs as i64; // Truncates on purpose in case rhs is actually an unsigned value
+ fx.bcx.ins().icmp_imm(intcc, lhs, rhs)
+ }
+}
+
++pub(crate) fn codegen_bitcast(fx: &mut FunctionCx<'_, '_, '_>, dst_ty: Type, val: Value) -> Value {
++ let mut flags = MemFlags::new();
++ flags.set_endianness(match fx.tcx.data_layout.endian {
++ rustc_target::abi::Endian::Big => cranelift_codegen::ir::Endianness::Big,
++ rustc_target::abi::Endian::Little => cranelift_codegen::ir::Endianness::Little,
++ });
++ fx.bcx.ins().bitcast(dst_ty, flags, val)
++}
++
+pub(crate) fn type_zero_value(bcx: &mut FunctionBuilder<'_>, ty: Type) -> Value {
+ if ty == types::I128 {
+ let zero = bcx.ins().iconst(types::I64, 0);
+ bcx.ins().iconcat(zero, zero)
+ } else {
+ bcx.ins().iconst(ty, 0)
+ }
+}
+
+pub(crate) fn type_min_max_value(
+ bcx: &mut FunctionBuilder<'_>,
+ ty: Type,
+ signed: bool,
+) -> (Value, Value) {
+ assert!(ty.is_int());
+
+ if ty == types::I128 {
+ if signed {
+ let min = i128::MIN as u128;
+ let min_lsb = bcx.ins().iconst(types::I64, min as u64 as i64);
+ let min_msb = bcx.ins().iconst(types::I64, (min >> 64) as u64 as i64);
+ let min = bcx.ins().iconcat(min_lsb, min_msb);
+
+ let max = i128::MAX as u128;
+ let max_lsb = bcx.ins().iconst(types::I64, max as u64 as i64);
+ let max_msb = bcx.ins().iconst(types::I64, (max >> 64) as u64 as i64);
+ let max = bcx.ins().iconcat(max_lsb, max_msb);
+
+ return (min, max);
+ } else {
+ let min_half = bcx.ins().iconst(types::I64, 0);
+ let min = bcx.ins().iconcat(min_half, min_half);
+
+ let max_half = bcx.ins().iconst(types::I64, u64::MAX as i64);
+ let max = bcx.ins().iconcat(max_half, max_half);
+
+ return (min, max);
+ }
+ }
+
+ let min = match (ty, signed) {
+ (types::I8, false) | (types::I16, false) | (types::I32, false) | (types::I64, false) => {
+ 0i64
+ }
+ (types::I8, true) => i64::from(i8::MIN),
+ (types::I16, true) => i64::from(i16::MIN),
+ (types::I32, true) => i64::from(i32::MIN),
+ (types::I64, true) => i64::MIN,
+ _ => unreachable!(),
+ };
+
+ let max = match (ty, signed) {
+ (types::I8, false) => i64::from(u8::MAX),
+ (types::I16, false) => i64::from(u16::MAX),
+ (types::I32, false) => i64::from(u32::MAX),
+ (types::I64, false) => u64::MAX as i64,
+ (types::I8, true) => i64::from(i8::MAX),
+ (types::I16, true) => i64::from(i16::MAX),
+ (types::I32, true) => i64::from(i32::MAX),
+ (types::I64, true) => i64::MAX,
+ _ => unreachable!(),
+ };
+
+ let (min, max) = (bcx.ins().iconst(ty, min), bcx.ins().iconst(ty, max));
+
+ (min, max)
+}
+
+pub(crate) fn type_sign(ty: Ty<'_>) -> bool {
+ match ty.kind() {
+ ty::Ref(..) | ty::RawPtr(..) | ty::FnPtr(..) | ty::Char | ty::Uint(..) | ty::Bool => false,
+ ty::Int(..) => true,
+ ty::Float(..) => false, // `signed` is unused for floats
+ _ => panic!("{}", ty),
+ }
+}
+
+pub(crate) struct FunctionCx<'m, 'clif, 'tcx: 'm> {
+ pub(crate) cx: &'clif mut crate::CodegenCx,
+ pub(crate) module: &'m mut dyn Module,
+ pub(crate) tcx: TyCtxt<'tcx>,
+ pub(crate) target_config: TargetFrontendConfig, // Cached from module
+ pub(crate) pointer_type: Type, // Cached from module
+ pub(crate) constants_cx: ConstantCx,
+ pub(crate) func_debug_cx: Option<FunctionDebugContext>,
+
+ pub(crate) instance: Instance<'tcx>,
+ pub(crate) symbol_name: String,
+ pub(crate) mir: &'tcx Body<'tcx>,
+ pub(crate) fn_abi: Option<&'tcx FnAbi<'tcx, Ty<'tcx>>>,
+
+ pub(crate) bcx: FunctionBuilder<'clif>,
+ pub(crate) block_map: IndexVec<BasicBlock, Block>,
+ pub(crate) local_map: IndexVec<Local, CPlace<'tcx>>,
+
+ /// When `#[track_caller]` is used, the implicit caller location is stored in this variable.
+ pub(crate) caller_location: Option<CValue<'tcx>>,
+
+ pub(crate) clif_comments: crate::pretty_clif::CommentWriter,
+
+ /// Last accessed source file and it's debuginfo file id.
+ ///
+ /// For optimization purposes only
+ pub(crate) last_source_file: Option<(Lrc<SourceFile>, FileId)>,
+
+ /// This should only be accessed by `CPlace::new_var`.
+ pub(crate) next_ssa_var: u32,
+}
+
+impl<'tcx> LayoutOfHelpers<'tcx> for FunctionCx<'_, '_, 'tcx> {
+ type LayoutOfResult = TyAndLayout<'tcx>;
+
+ #[inline]
+ fn handle_layout_err(&self, err: LayoutError<'tcx>, span: Span, ty: Ty<'tcx>) -> ! {
+ RevealAllLayoutCx(self.tcx).handle_layout_err(err, span, ty)
+ }
+}
+
+impl<'tcx> FnAbiOfHelpers<'tcx> for FunctionCx<'_, '_, 'tcx> {
+ type FnAbiOfResult = &'tcx FnAbi<'tcx, Ty<'tcx>>;
+
+ #[inline]
+ fn handle_fn_abi_err(
+ &self,
+ err: FnAbiError<'tcx>,
+ span: Span,
+ fn_abi_request: FnAbiRequest<'tcx>,
+ ) -> ! {
+ RevealAllLayoutCx(self.tcx).handle_fn_abi_err(err, span, fn_abi_request)
+ }
+}
+
+impl<'tcx> layout::HasTyCtxt<'tcx> for FunctionCx<'_, '_, 'tcx> {
+ fn tcx<'b>(&'b self) -> TyCtxt<'tcx> {
+ self.tcx
+ }
+}
+
+impl<'tcx> rustc_target::abi::HasDataLayout for FunctionCx<'_, '_, 'tcx> {
+ fn data_layout(&self) -> &rustc_target::abi::TargetDataLayout {
+ &self.tcx.data_layout
+ }
+}
+
+impl<'tcx> layout::HasParamEnv<'tcx> for FunctionCx<'_, '_, 'tcx> {
+ fn param_env(&self) -> ParamEnv<'tcx> {
+ ParamEnv::reveal_all()
+ }
+}
+
+impl<'tcx> HasTargetSpec for FunctionCx<'_, '_, 'tcx> {
+ fn target_spec(&self) -> &Target {
+ &self.tcx.sess.target
+ }
+}
+
+impl<'tcx> FunctionCx<'_, '_, 'tcx> {
+ pub(crate) fn monomorphize<T>(&self, value: T) -> T
+ where
+ T: TypeFoldable<'tcx> + Copy,
+ {
+ self.instance.subst_mir_and_normalize_erasing_regions(
+ self.tcx,
+ ty::ParamEnv::reveal_all(),
+ value,
+ )
+ }
+
+ pub(crate) fn clif_type(&self, ty: Ty<'tcx>) -> Option<Type> {
+ clif_type_from_ty(self.tcx, ty)
+ }
+
+ pub(crate) fn clif_pair_type(&self, ty: Ty<'tcx>) -> Option<(Type, Type)> {
+ clif_pair_type_from_ty(self.tcx, ty)
+ }
+
+ pub(crate) fn get_block(&self, bb: BasicBlock) -> Block {
+ *self.block_map.get(bb).unwrap()
+ }
+
+ pub(crate) fn get_local_place(&mut self, local: Local) -> CPlace<'tcx> {
+ *self.local_map.get(local).unwrap_or_else(|| {
+ panic!("Local {:?} doesn't exist", local);
+ })
+ }
+
+ pub(crate) fn set_debug_loc(&mut self, source_info: mir::SourceInfo) {
+ if let Some(debug_context) = &mut self.cx.debug_context {
+ let (file, line, column) =
+ DebugContext::get_span_loc(self.tcx, self.mir.span, source_info.span);
+
+ // add_source_file is very slow.
+ // Optimize for the common case of the current file not being changed.
+ let mut cached_file_id = None;
+ if let Some((ref last_source_file, last_file_id)) = self.last_source_file {
+ // If the allocations are not equal, the files may still be equal, but that
+ // doesn't matter, as this is just an optimization.
+ if rustc_data_structures::sync::Lrc::ptr_eq(last_source_file, &file) {
+ cached_file_id = Some(last_file_id);
+ }
+ }
+
+ let file_id = if let Some(file_id) = cached_file_id {
+ file_id
+ } else {
+ debug_context.add_source_file(&file)
+ };
+
+ let source_loc =
+ self.func_debug_cx.as_mut().unwrap().add_dbg_loc(file_id, line, column);
+ self.bcx.set_srcloc(source_loc);
+ }
+ }
+
+ // Note: must be kept in sync with get_caller_location from cg_ssa
+ pub(crate) fn get_caller_location(&mut self, mut source_info: mir::SourceInfo) -> CValue<'tcx> {
+ let span_to_caller_location = |fx: &mut FunctionCx<'_, '_, 'tcx>, span: Span| {
+ let topmost = span.ctxt().outer_expn().expansion_cause().unwrap_or(span);
+ let caller = fx.tcx.sess.source_map().lookup_char_pos(topmost.lo());
+ let const_loc = fx.tcx.const_caller_location((
+ rustc_span::symbol::Symbol::intern(
+ &caller.file.name.prefer_remapped().to_string_lossy(),
+ ),
+ caller.line as u32,
+ caller.col_display as u32 + 1,
+ ));
+ crate::constant::codegen_const_value(fx, const_loc, fx.tcx.caller_location_ty())
+ };
+
+ // Walk up the `SourceScope`s, in case some of them are from MIR inlining.
+ // If so, the starting `source_info.span` is in the innermost inlined
+ // function, and will be replaced with outer callsite spans as long
+ // as the inlined functions were `#[track_caller]`.
+ loop {
+ let scope_data = &self.mir.source_scopes[source_info.scope];
+
+ if let Some((callee, callsite_span)) = scope_data.inlined {
+ // Stop inside the most nested non-`#[track_caller]` function,
+ // before ever reaching its caller (which is irrelevant).
+ if !callee.def.requires_caller_location(self.tcx) {
+ return span_to_caller_location(self, source_info.span);
+ }
+ source_info.span = callsite_span;
+ }
+
+ // Skip past all of the parents with `inlined: None`.
+ match scope_data.inlined_parent_scope {
+ Some(parent) => source_info.scope = parent,
+ None => break,
+ }
+ }
+
+ // No inlined `SourceScope`s, or all of them were `#[track_caller]`.
+ self.caller_location.unwrap_or_else(|| span_to_caller_location(self, source_info.span))
+ }
+
+ pub(crate) fn anonymous_str(&mut self, msg: &str) -> Value {
+ let mut data_ctx = DataContext::new();
+ data_ctx.define(msg.as_bytes().to_vec().into_boxed_slice());
+ let msg_id = self.module.declare_anonymous_data(false, false).unwrap();
+
+ // Ignore DuplicateDefinition error, as the data will be the same
+ let _ = self.module.define_data(msg_id, &data_ctx);
+
+ let local_msg_id = self.module.declare_data_in_func(msg_id, self.bcx.func);
+ if self.clif_comments.enabled() {
+ self.add_comment(local_msg_id, msg);
+ }
+ self.bcx.ins().global_value(self.pointer_type, local_msg_id)
+ }
+}
+
+pub(crate) struct RevealAllLayoutCx<'tcx>(pub(crate) TyCtxt<'tcx>);
+
+impl<'tcx> LayoutOfHelpers<'tcx> for RevealAllLayoutCx<'tcx> {
+ type LayoutOfResult = TyAndLayout<'tcx>;
+
+ #[inline]
+ fn handle_layout_err(&self, err: LayoutError<'tcx>, span: Span, ty: Ty<'tcx>) -> ! {
+ if let layout::LayoutError::SizeOverflow(_) = err {
+ self.0.sess.span_fatal(span, &err.to_string())
+ } else {
+ span_bug!(span, "failed to get layout for `{}`: {}", ty, err)
+ }
+ }
+}
+
+impl<'tcx> FnAbiOfHelpers<'tcx> for RevealAllLayoutCx<'tcx> {
+ type FnAbiOfResult = &'tcx FnAbi<'tcx, Ty<'tcx>>;
+
+ #[inline]
+ fn handle_fn_abi_err(
+ &self,
+ err: FnAbiError<'tcx>,
+ span: Span,
+ fn_abi_request: FnAbiRequest<'tcx>,
+ ) -> ! {
+ if let FnAbiError::Layout(LayoutError::SizeOverflow(_)) = err {
+ self.0.sess.span_fatal(span, &err.to_string())
+ } else {
+ match fn_abi_request {
+ FnAbiRequest::OfFnPtr { sig, extra_args } => {
+ span_bug!(
+ span,
+ "`fn_abi_of_fn_ptr({}, {:?})` failed: {}",
+ sig,
+ extra_args,
+ err
+ );
+ }
+ FnAbiRequest::OfInstance { instance, extra_args } => {
+ span_bug!(
+ span,
+ "`fn_abi_of_instance({}, {:?})` failed: {}",
+ instance,
+ extra_args,
+ err
+ );
+ }
+ }
+ }
+ }
+}
+
+impl<'tcx> layout::HasTyCtxt<'tcx> for RevealAllLayoutCx<'tcx> {
+ fn tcx<'b>(&'b self) -> TyCtxt<'tcx> {
+ self.0
+ }
+}
+
+impl<'tcx> rustc_target::abi::HasDataLayout for RevealAllLayoutCx<'tcx> {
+ fn data_layout(&self) -> &rustc_target::abi::TargetDataLayout {
+ &self.0.data_layout
+ }
+}
+
+impl<'tcx> layout::HasParamEnv<'tcx> for RevealAllLayoutCx<'tcx> {
+ fn param_env(&self) -> ParamEnv<'tcx> {
+ ParamEnv::reveal_all()
+ }
+}
+
+impl<'tcx> HasTargetSpec for RevealAllLayoutCx<'tcx> {
+ fn target_spec(&self) -> &Target {
+ &self.0.sess.target
+ }
+}
--- /dev/null
- let producer = format!(
- "cg_clif (rustc {}, cranelift {})",
- rustc_interface::util::rustc_version_str().unwrap_or("unknown version"),
- cranelift_codegen::VERSION,
- );
+//! Handling of everything related to debuginfo.
+
+mod emit;
+mod line_info;
+mod object;
+mod unwind;
+
+use crate::prelude::*;
+
+use cranelift_codegen::ir::Endianness;
+use cranelift_codegen::isa::TargetIsa;
+
+use gimli::write::{
+ Address, AttributeValue, DwarfUnit, FileId, LineProgram, LineString, Range, RangeList,
+ UnitEntryId,
+};
+use gimli::{Encoding, Format, LineEncoding, RunTimeEndian};
+use indexmap::IndexSet;
+
+pub(crate) use emit::{DebugReloc, DebugRelocName};
+pub(crate) use unwind::UnwindContext;
+
++pub(crate) fn producer() -> String {
++ format!(
++ "cg_clif (rustc {}, cranelift {})",
++ rustc_interface::util::rustc_version_str().unwrap_or("unknown version"),
++ cranelift_codegen::VERSION,
++ )
++}
++
+pub(crate) struct DebugContext {
+ endian: RunTimeEndian,
+
+ dwarf: DwarfUnit,
+ unit_range_list: RangeList,
+}
+
+pub(crate) struct FunctionDebugContext {
+ entry_id: UnitEntryId,
+ function_source_loc: (FileId, u64, u64),
+ source_loc_set: indexmap::IndexSet<(FileId, u64, u64)>,
+}
+
+impl DebugContext {
+ pub(crate) fn new(tcx: TyCtxt<'_>, isa: &dyn TargetIsa) -> Self {
+ let encoding = Encoding {
+ format: Format::Dwarf32,
+ // FIXME this should be configurable
+ // macOS doesn't seem to support DWARF > 3
+ // 5 version is required for md5 file hash
+ version: if tcx.sess.target.is_like_osx {
+ 3
+ } else {
+ // FIXME change to version 5 once the gdb and lldb shipping with the latest debian
+ // support it.
+ 4
+ },
+ address_size: isa.frontend_config().pointer_bytes(),
+ };
+
+ let endian = match isa.endianness() {
+ Endianness::Little => RunTimeEndian::Little,
+ Endianness::Big => RunTimeEndian::Big,
+ };
+
+ let mut dwarf = DwarfUnit::new(encoding);
+
++ let producer = producer();
+ let comp_dir = tcx
+ .sess
+ .opts
+ .working_dir
+ .to_string_lossy(FileNameDisplayPreference::Remapped)
+ .into_owned();
+ let (name, file_info) = match tcx.sess.local_crate_source_file() {
+ Some(path) => {
+ let name = path.to_string_lossy().into_owned();
+ (name, None)
+ }
+ None => (tcx.crate_name(LOCAL_CRATE).to_string(), None),
+ };
+
+ let mut line_program = LineProgram::new(
+ encoding,
+ LineEncoding::default(),
+ LineString::new(comp_dir.as_bytes(), encoding, &mut dwarf.line_strings),
+ LineString::new(name.as_bytes(), encoding, &mut dwarf.line_strings),
+ file_info,
+ );
+ line_program.file_has_md5 = file_info.is_some();
+
+ dwarf.unit.line_program = line_program;
+
+ {
+ let name = dwarf.strings.add(name);
+ let comp_dir = dwarf.strings.add(comp_dir);
+
+ let root = dwarf.unit.root();
+ let root = dwarf.unit.get_mut(root);
+ root.set(gimli::DW_AT_producer, AttributeValue::StringRef(dwarf.strings.add(producer)));
+ root.set(gimli::DW_AT_language, AttributeValue::Language(gimli::DW_LANG_Rust));
+ root.set(gimli::DW_AT_name, AttributeValue::StringRef(name));
+ root.set(gimli::DW_AT_comp_dir, AttributeValue::StringRef(comp_dir));
+ root.set(gimli::DW_AT_low_pc, AttributeValue::Address(Address::Constant(0)));
+ }
+
+ DebugContext { endian, dwarf, unit_range_list: RangeList(Vec::new()) }
+ }
+
+ pub(crate) fn define_function(
+ &mut self,
+ tcx: TyCtxt<'_>,
+ name: &str,
+ function_span: Span,
+ ) -> FunctionDebugContext {
+ let (file, line, column) = DebugContext::get_span_loc(tcx, function_span, function_span);
+
+ let file_id = self.add_source_file(&file);
+
+ // FIXME: add to appropriate scope instead of root
+ let scope = self.dwarf.unit.root();
+
+ let entry_id = self.dwarf.unit.add(scope, gimli::DW_TAG_subprogram);
+ let entry = self.dwarf.unit.get_mut(entry_id);
+ let name_id = self.dwarf.strings.add(name);
+ // Gdb requires DW_AT_name. Otherwise the DW_TAG_subprogram is skipped.
+ entry.set(gimli::DW_AT_name, AttributeValue::StringRef(name_id));
+ entry.set(gimli::DW_AT_linkage_name, AttributeValue::StringRef(name_id));
+
+ entry.set(gimli::DW_AT_decl_file, AttributeValue::FileIndex(Some(file_id)));
+ entry.set(gimli::DW_AT_decl_line, AttributeValue::Udata(line));
+ entry.set(gimli::DW_AT_decl_column, AttributeValue::Udata(column));
+
+ FunctionDebugContext {
+ entry_id,
+ function_source_loc: (file_id, line, column),
+ source_loc_set: IndexSet::new(),
+ }
+ }
+}
+
+impl FunctionDebugContext {
+ pub(crate) fn finalize(
+ mut self,
+ debug_context: &mut DebugContext,
+ func_id: FuncId,
+ context: &Context,
+ ) {
+ let symbol = func_id.as_u32() as usize;
+
+ let end = self.create_debug_lines(debug_context, symbol, context);
+
+ debug_context.unit_range_list.0.push(Range::StartLength {
+ begin: Address::Symbol { symbol, addend: 0 },
+ length: u64::from(end),
+ });
+
+ let func_entry = debug_context.dwarf.unit.get_mut(self.entry_id);
+ // Gdb requires both DW_AT_low_pc and DW_AT_high_pc. Otherwise the DW_TAG_subprogram is skipped.
+ func_entry.set(
+ gimli::DW_AT_low_pc,
+ AttributeValue::Address(Address::Symbol { symbol, addend: 0 }),
+ );
+ // Using Udata for DW_AT_high_pc requires at least DWARF4
+ func_entry.set(gimli::DW_AT_high_pc, AttributeValue::Udata(u64::from(end)));
+ }
+}
--- /dev/null
- object: cranelift_object::object::write::Object<'_>,
+//! The AOT driver uses [`cranelift_object`] to write object files suitable for linking into a
+//! standalone executable.
+
+use std::fs::File;
+use std::path::PathBuf;
+use std::sync::Arc;
+use std::thread::JoinHandle;
+
+use rustc_codegen_ssa::back::metadata::create_compressed_metadata_file;
+use rustc_codegen_ssa::{CodegenResults, CompiledModule, CrateInfo, ModuleKind};
+use rustc_data_structures::profiling::SelfProfilerRef;
+use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
+use rustc_metadata::EncodedMetadata;
+use rustc_middle::dep_graph::{WorkProduct, WorkProductId};
+use rustc_middle::mir::mono::{CodegenUnit, MonoItem};
+use rustc_session::cgu_reuse_tracker::CguReuse;
+use rustc_session::config::{DebugInfo, OutputFilenames, OutputType};
+use rustc_session::Session;
+
+use cranelift_object::{ObjectBuilder, ObjectModule};
+
+use crate::concurrency_limiter::{ConcurrencyLimiter, ConcurrencyLimiterToken};
+use crate::global_asm::GlobalAsmConfig;
+use crate::{prelude::*, BackendConfig};
+
+struct ModuleCodegenResult {
+ module_regular: CompiledModule,
+ module_global_asm: Option<CompiledModule>,
+ existing_work_product: Option<(WorkProductId, WorkProduct)>,
+}
+
+enum OngoingModuleCodegen {
+ Sync(Result<ModuleCodegenResult, String>),
+ Async(JoinHandle<Result<ModuleCodegenResult, String>>),
+}
+
+impl<HCX> HashStable<HCX> for OngoingModuleCodegen {
+ fn hash_stable(&self, _: &mut HCX, _: &mut StableHasher) {
+ // do nothing
+ }
+}
+
+pub(crate) struct OngoingCodegen {
+ modules: Vec<OngoingModuleCodegen>,
+ allocator_module: Option<CompiledModule>,
+ metadata_module: Option<CompiledModule>,
+ metadata: EncodedMetadata,
+ crate_info: CrateInfo,
+ concurrency_limiter: ConcurrencyLimiter,
+}
+
+impl OngoingCodegen {
+ pub(crate) fn join(
+ self,
+ sess: &Session,
+ backend_config: &BackendConfig,
+ ) -> (CodegenResults, FxHashMap<WorkProductId, WorkProduct>) {
+ let mut work_products = FxHashMap::default();
+ let mut modules = vec![];
+
+ for module_codegen in self.modules {
+ let module_codegen_result = match module_codegen {
+ OngoingModuleCodegen::Sync(module_codegen_result) => module_codegen_result,
+ OngoingModuleCodegen::Async(join_handle) => match join_handle.join() {
+ Ok(module_codegen_result) => module_codegen_result,
+ Err(panic) => std::panic::resume_unwind(panic),
+ },
+ };
+
+ let module_codegen_result = match module_codegen_result {
+ Ok(module_codegen_result) => module_codegen_result,
+ Err(err) => sess.fatal(&err),
+ };
+ let ModuleCodegenResult { module_regular, module_global_asm, existing_work_product } =
+ module_codegen_result;
+
+ if let Some((work_product_id, work_product)) = existing_work_product {
+ work_products.insert(work_product_id, work_product);
+ } else {
+ let work_product = if backend_config.disable_incr_cache {
+ None
+ } else if let Some(module_global_asm) = &module_global_asm {
+ rustc_incremental::copy_cgu_workproduct_to_incr_comp_cache_dir(
+ sess,
+ &module_regular.name,
+ &[
+ ("o", &module_regular.object.as_ref().unwrap()),
+ ("asm.o", &module_global_asm.object.as_ref().unwrap()),
+ ],
+ )
+ } else {
+ rustc_incremental::copy_cgu_workproduct_to_incr_comp_cache_dir(
+ sess,
+ &module_regular.name,
+ &[("o", &module_regular.object.as_ref().unwrap())],
+ )
+ };
+ if let Some((work_product_id, work_product)) = work_product {
+ work_products.insert(work_product_id, work_product);
+ }
+ }
+
+ modules.push(module_regular);
+ if let Some(module_global_asm) = module_global_asm {
+ modules.push(module_global_asm);
+ }
+ }
+
+ self.concurrency_limiter.finished();
+
++ sess.abort_if_errors();
++
+ (
+ CodegenResults {
+ modules,
+ allocator_module: self.allocator_module,
+ metadata_module: self.metadata_module,
+ metadata: self.metadata,
+ crate_info: self.crate_info,
+ },
+ work_products,
+ )
+ }
+}
+
+fn make_module(sess: &Session, backend_config: &BackendConfig, name: String) -> ObjectModule {
+ let isa = crate::build_isa(sess, backend_config);
+
+ let mut builder =
+ ObjectBuilder::new(isa, name + ".o", cranelift_module::default_libcall_names()).unwrap();
+ // Unlike cg_llvm, cg_clif defaults to disabling -Zfunction-sections. For cg_llvm binary size
+ // is important, while cg_clif cares more about compilation times. Enabling -Zfunction-sections
+ // can easily double the amount of time necessary to perform linking.
+ builder.per_function_section(sess.opts.unstable_opts.function_sections.unwrap_or(false));
+ ObjectModule::new(builder)
+}
+
+fn emit_cgu(
+ output_filenames: &OutputFilenames,
+ prof: &SelfProfilerRef,
+ name: String,
+ module: ObjectModule,
+ debug: Option<DebugContext>,
+ unwind_context: UnwindContext,
+ global_asm_object_file: Option<PathBuf>,
+) -> Result<ModuleCodegenResult, String> {
+ let mut product = module.finish();
+
+ if let Some(mut debug) = debug {
+ debug.emit(&mut product);
+ }
+
+ unwind_context.emit(&mut product);
+
+ let module_regular =
+ emit_module(output_filenames, prof, product.object, ModuleKind::Regular, name.clone())?;
+
+ Ok(ModuleCodegenResult {
+ module_regular,
+ module_global_asm: global_asm_object_file.map(|global_asm_object_file| CompiledModule {
+ name: format!("{name}.asm"),
+ kind: ModuleKind::Regular,
+ object: Some(global_asm_object_file),
+ dwarf_object: None,
+ bytecode: None,
+ }),
+ existing_work_product: None,
+ })
+}
+
+fn emit_module(
+ output_filenames: &OutputFilenames,
+ prof: &SelfProfilerRef,
- tcx.sess.abort_if_errors();
-
++ mut object: cranelift_object::object::write::Object<'_>,
+ kind: ModuleKind,
+ name: String,
+) -> Result<CompiledModule, String> {
++ if object.format() == cranelift_object::object::BinaryFormat::Elf {
++ let comment_section = object.add_section(
++ Vec::new(),
++ b".comment".to_vec(),
++ cranelift_object::object::SectionKind::OtherString,
++ );
++ let mut producer = vec![0];
++ producer.extend(crate::debuginfo::producer().as_bytes());
++ producer.push(0);
++ object.set_section_data(comment_section, producer, 1);
++ }
++
+ let tmp_file = output_filenames.temp_path(OutputType::Object, Some(&name));
+ let mut file = match File::create(&tmp_file) {
+ Ok(file) => file,
+ Err(err) => return Err(format!("error creating object file: {}", err)),
+ };
+
+ if let Err(err) = object.write_stream(&mut file) {
+ return Err(format!("error writing object file: {}", err));
+ }
+
+ prof.artifact_size("object_file", &*name, file.metadata().unwrap().len());
+
+ Ok(CompiledModule { name, kind, object: Some(tmp_file), dwarf_object: None, bytecode: None })
+}
+
+fn reuse_workproduct_for_cgu(
+ tcx: TyCtxt<'_>,
+ cgu: &CodegenUnit<'_>,
+) -> Result<ModuleCodegenResult, String> {
+ let work_product = cgu.previous_work_product(tcx);
+ let obj_out_regular =
+ tcx.output_filenames(()).temp_path(OutputType::Object, Some(cgu.name().as_str()));
+ let source_file_regular = rustc_incremental::in_incr_comp_dir_sess(
+ &tcx.sess,
+ &work_product.saved_files.get("o").expect("no saved object file in work product"),
+ );
+
+ if let Err(err) = rustc_fs_util::link_or_copy(&source_file_regular, &obj_out_regular) {
+ return Err(format!(
+ "unable to copy {} to {}: {}",
+ source_file_regular.display(),
+ obj_out_regular.display(),
+ err
+ ));
+ }
+ let obj_out_global_asm =
+ crate::global_asm::add_file_stem_postfix(obj_out_regular.clone(), ".asm");
+ let has_global_asm = if let Some(asm_o) = work_product.saved_files.get("asm.o") {
+ let source_file_global_asm = rustc_incremental::in_incr_comp_dir_sess(&tcx.sess, asm_o);
+ if let Err(err) = rustc_fs_util::link_or_copy(&source_file_global_asm, &obj_out_global_asm)
+ {
+ return Err(format!(
+ "unable to copy {} to {}: {}",
+ source_file_regular.display(),
+ obj_out_regular.display(),
+ err
+ ));
+ }
+ true
+ } else {
+ false
+ };
+
+ Ok(ModuleCodegenResult {
+ module_regular: CompiledModule {
+ name: cgu.name().to_string(),
+ kind: ModuleKind::Regular,
+ object: Some(obj_out_regular),
+ dwarf_object: None,
+ bytecode: None,
+ },
+ module_global_asm: if has_global_asm {
+ Some(CompiledModule {
+ name: cgu.name().to_string(),
+ kind: ModuleKind::Regular,
+ object: Some(obj_out_global_asm),
+ dwarf_object: None,
+ bytecode: None,
+ })
+ } else {
+ None
+ },
+ existing_work_product: Some((cgu.work_product_id(), work_product)),
+ })
+}
+
+fn module_codegen(
+ tcx: TyCtxt<'_>,
+ (backend_config, global_asm_config, cgu_name, token): (
+ BackendConfig,
+ Arc<GlobalAsmConfig>,
+ rustc_span::Symbol,
+ ConcurrencyLimiterToken,
+ ),
+) -> OngoingModuleCodegen {
+ let (cgu_name, mut cx, mut module, codegened_functions) = tcx.sess.time("codegen cgu", || {
+ let cgu = tcx.codegen_unit(cgu_name);
+ let mono_items = cgu.items_in_deterministic_order(tcx);
+
+ let mut module = make_module(tcx.sess, &backend_config, cgu_name.as_str().to_string());
+
+ let mut cx = crate::CodegenCx::new(
+ tcx,
+ backend_config.clone(),
+ module.isa(),
+ tcx.sess.opts.debuginfo != DebugInfo::None,
+ cgu_name,
+ );
+ super::predefine_mono_items(tcx, &mut module, &mono_items);
+ let mut codegened_functions = vec![];
+ for (mono_item, _) in mono_items {
+ match mono_item {
+ MonoItem::Fn(inst) => {
+ tcx.sess.time("codegen fn", || {
+ let codegened_function = crate::base::codegen_fn(
+ tcx,
+ &mut cx,
+ Function::new(),
+ &mut module,
+ inst,
+ );
+ codegened_functions.push(codegened_function);
+ });
+ }
+ MonoItem::Static(def_id) => {
+ crate::constant::codegen_static(tcx, &mut module, def_id)
+ }
+ MonoItem::GlobalAsm(item_id) => {
+ crate::global_asm::codegen_global_asm_item(tcx, &mut cx.global_asm, item_id);
+ }
+ }
+ }
+ crate::main_shim::maybe_create_entry_wrapper(
+ tcx,
+ &mut module,
+ &mut cx.unwind_context,
+ false,
+ cgu.is_primary(),
+ );
+
+ let cgu_name = cgu.name().as_str().to_owned();
+
+ (cgu_name, cx, module, codegened_functions)
+ });
+
+ OngoingModuleCodegen::Async(std::thread::spawn(move || {
+ cx.profiler.clone().verbose_generic_activity("compile functions").run(|| {
+ let mut cached_context = Context::new();
+ for codegened_func in codegened_functions {
+ crate::base::compile_fn(&mut cx, &mut cached_context, &mut module, codegened_func);
+ }
+ });
+
+ let global_asm_object_file =
+ cx.profiler.verbose_generic_activity("compile assembly").run(|| {
+ crate::global_asm::compile_global_asm(&global_asm_config, &cgu_name, &cx.global_asm)
+ })?;
+
+ let codegen_result = cx.profiler.verbose_generic_activity("write object file").run(|| {
+ emit_cgu(
+ &global_asm_config.output_filenames,
+ &cx.profiler,
+ cgu_name,
+ module,
+ cx.debug_context,
+ cx.unwind_context,
+ global_asm_object_file,
+ )
+ });
+ std::mem::drop(token);
+ codegen_result
+ }))
+}
+
+pub(crate) fn run_aot(
+ tcx: TyCtxt<'_>,
+ backend_config: BackendConfig,
+ metadata: EncodedMetadata,
+ need_metadata_module: bool,
+) -> Box<OngoingCodegen> {
+ let cgus = if tcx.sess.opts.output_types.should_codegen() {
+ tcx.collect_and_partition_mono_items(()).1
+ } else {
+ // If only `--emit metadata` is used, we shouldn't perform any codegen.
+ // Also `tcx.collect_and_partition_mono_items` may panic in that case.
+ &[]
+ };
+
+ if tcx.dep_graph.is_fully_enabled() {
+ for cgu in &*cgus {
+ tcx.ensure().codegen_unit(cgu.name());
+ }
+ }
+
+ let global_asm_config = Arc::new(crate::global_asm::GlobalAsmConfig::new(tcx));
+
+ let mut concurrency_limiter = ConcurrencyLimiter::new(tcx.sess, cgus.len());
+
+ let modules = super::time(tcx, backend_config.display_cg_time, "codegen mono items", || {
+ cgus.iter()
+ .map(|cgu| {
+ let cgu_reuse = if backend_config.disable_incr_cache {
+ CguReuse::No
+ } else {
+ determine_cgu_reuse(tcx, cgu)
+ };
+ tcx.sess.cgu_reuse_tracker.set_actual_reuse(cgu.name().as_str(), cgu_reuse);
+
+ match cgu_reuse {
+ CguReuse::No => {
+ let dep_node = cgu.codegen_dep_node(tcx);
+ tcx.dep_graph
+ .with_task(
+ dep_node,
+ tcx,
+ (
+ backend_config.clone(),
+ global_asm_config.clone(),
+ cgu.name(),
+ concurrency_limiter.acquire(),
+ ),
+ module_codegen,
+ Some(rustc_middle::dep_graph::hash_result),
+ )
+ .0
+ }
+ CguReuse::PreLto => unreachable!(),
+ CguReuse::PostLto => {
+ concurrency_limiter.job_already_done();
+ OngoingModuleCodegen::Sync(reuse_workproduct_for_cgu(tcx, &*cgu))
+ }
+ }
+ })
+ .collect::<Vec<_>>()
+ });
+
+ let mut allocator_module = make_module(tcx.sess, &backend_config, "allocator_shim".to_string());
+ let mut allocator_unwind_context = UnwindContext::new(allocator_module.isa(), true);
+ let created_alloc_shim =
+ crate::allocator::codegen(tcx, &mut allocator_module, &mut allocator_unwind_context);
+
+ let allocator_module = if created_alloc_shim {
+ let mut product = allocator_module.finish();
+ allocator_unwind_context.emit(&mut product);
+
+ match emit_module(
+ tcx.output_filenames(()),
+ &tcx.sess.prof,
+ product.object,
+ ModuleKind::Allocator,
+ "allocator_shim".to_owned(),
+ ) {
+ Ok(allocator_module) => Some(allocator_module),
+ Err(err) => tcx.sess.fatal(err),
+ }
+ } else {
+ None
+ };
+
+ let metadata_module = if need_metadata_module {
+ let _timer = tcx.prof.generic_activity("codegen crate metadata");
+ let (metadata_cgu_name, tmp_file) = tcx.sess.time("write compressed metadata", || {
+ use rustc_middle::mir::mono::CodegenUnitNameBuilder;
+
+ let cgu_name_builder = &mut CodegenUnitNameBuilder::new(tcx);
+ let metadata_cgu_name = cgu_name_builder
+ .build_cgu_name(LOCAL_CRATE, &["crate"], Some("metadata"))
+ .as_str()
+ .to_string();
+
+ let tmp_file =
+ tcx.output_filenames(()).temp_path(OutputType::Metadata, Some(&metadata_cgu_name));
+
+ let symbol_name = rustc_middle::middle::exported_symbols::metadata_symbol_name(tcx);
+ let obj = create_compressed_metadata_file(tcx.sess, &metadata, &symbol_name);
+
+ if let Err(err) = std::fs::write(&tmp_file, obj) {
+ tcx.sess.fatal(&format!("error writing metadata object file: {}", err));
+ }
+
+ (metadata_cgu_name, tmp_file)
+ });
+
+ Some(CompiledModule {
+ name: metadata_cgu_name,
+ kind: ModuleKind::Metadata,
+ object: Some(tmp_file),
+ dwarf_object: None,
+ bytecode: None,
+ })
+ } else {
+ None
+ };
+
+ // FIXME handle `-Ctarget-cpu=native`
+ let target_cpu = match tcx.sess.opts.cg.target_cpu {
+ Some(ref name) => name,
+ None => tcx.sess.target.cpu.as_ref(),
+ }
+ .to_owned();
+
+ Box::new(OngoingCodegen {
+ modules,
+ allocator_module,
+ metadata_module,
+ metadata,
+ crate_info: CrateInfo::new(tcx, target_cpu),
+ concurrency_limiter,
+ })
+}
+
+// Adapted from https://github.com/rust-lang/rust/blob/303d8aff6092709edd4dbd35b1c88e9aa40bf6d8/src/librustc_codegen_ssa/base.rs#L922-L953
+fn determine_cgu_reuse<'tcx>(tcx: TyCtxt<'tcx>, cgu: &CodegenUnit<'tcx>) -> CguReuse {
+ if !tcx.dep_graph.is_fully_enabled() {
+ return CguReuse::No;
+ }
+
+ let work_product_id = &cgu.work_product_id();
+ if tcx.dep_graph.previous_work_product(work_product_id).is_none() {
+ // We don't have anything cached for this CGU. This can happen
+ // if the CGU did not exist in the previous session.
+ return CguReuse::No;
+ }
+
+ // Try to mark the CGU as green. If it we can do so, it means that nothing
+ // affecting the LLVM module has changed and we can re-use a cached version.
+ // If we compile with any kind of LTO, this means we can re-use the bitcode
+ // of the Pre-LTO stage (possibly also the Post-LTO version but we'll only
+ // know that later). If we are not doing LTO, there is only one optimized
+ // version of each module, so we re-use that.
+ let dep_node = cgu.codegen_dep_node(tcx);
+ assert!(
+ !tcx.dep_graph.dep_node_exists(&dep_node),
+ "CompileCodegenUnit dep-node for CGU `{}` already exists before marking.",
+ cgu.name()
+ );
+
+ if tcx.try_mark_green(&dep_node) { CguReuse::PostLto } else { CguReuse::No }
+}
--- /dev/null
- types::F32 => fx.bcx.ins().bitcast(types::I32, a_lane),
- types::F64 => fx.bcx.ins().bitcast(types::I64, a_lane),
+//! Emulate x86 LLVM intrinsics
+
+use crate::intrinsics::*;
+use crate::prelude::*;
+
+use rustc_middle::ty::subst::SubstsRef;
+
+pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>(
+ fx: &mut FunctionCx<'_, '_, 'tcx>,
+ intrinsic: &str,
+ _substs: SubstsRef<'tcx>,
+ args: &[mir::Operand<'tcx>],
+ ret: CPlace<'tcx>,
+ target: Option<BasicBlock>,
+) {
+ match intrinsic {
+ "llvm.x86.sse2.pause" | "llvm.aarch64.isb" => {
+ // Spin loop hint
+ }
+
+ // Used by `_mm_movemask_epi8` and `_mm256_movemask_epi8`
+ "llvm.x86.sse2.pmovmskb.128" | "llvm.x86.avx2.pmovmskb" | "llvm.x86.sse2.movmsk.pd" => {
+ intrinsic_args!(fx, args => (a); intrinsic);
+
+ let (lane_count, lane_ty) = a.layout().ty.simd_size_and_type(fx.tcx);
+ let lane_ty = fx.clif_type(lane_ty).unwrap();
+ assert!(lane_count <= 32);
+
+ let mut res = fx.bcx.ins().iconst(types::I32, 0);
+
+ for lane in (0..lane_count).rev() {
+ let a_lane = a.value_lane(fx, lane).load_scalar(fx);
+
+ // cast float to int
+ let a_lane = match lane_ty {
++ types::F32 => codegen_bitcast(fx, types::I32, a_lane),
++ types::F64 => codegen_bitcast(fx, types::I64, a_lane),
+ _ => a_lane,
+ };
+
+ // extract sign bit of an int
+ let a_lane_sign = fx.bcx.ins().ushr_imm(a_lane, i64::from(lane_ty.bits() - 1));
+
+ // shift sign bit into result
+ let a_lane_sign = clif_intcast(fx, a_lane_sign, types::I32, false);
+ res = fx.bcx.ins().ishl_imm(res, 1);
+ res = fx.bcx.ins().bor(res, a_lane_sign);
+ }
+
+ let res = CValue::by_val(res, fx.layout_of(fx.tcx.types.i32));
+ ret.write_cvalue(fx, res);
+ }
+ "llvm.x86.sse2.cmp.ps" | "llvm.x86.sse2.cmp.pd" => {
+ let (x, y, kind) = match args {
+ [x, y, kind] => (x, y, kind),
+ _ => bug!("wrong number of args for intrinsic {intrinsic}"),
+ };
+ let x = codegen_operand(fx, x);
+ let y = codegen_operand(fx, y);
+ let kind = crate::constant::mir_operand_get_const_val(fx, kind)
+ .expect("llvm.x86.sse2.cmp.* kind not const");
+
+ let flt_cc = match kind
+ .try_to_bits(Size::from_bytes(1))
+ .unwrap_or_else(|| panic!("kind not scalar: {:?}", kind))
+ {
+ 0 => FloatCC::Equal,
+ 1 => FloatCC::LessThan,
+ 2 => FloatCC::LessThanOrEqual,
+ 7 => FloatCC::Ordered,
+ 3 => FloatCC::Unordered,
+ 4 => FloatCC::NotEqual,
+ 5 => FloatCC::UnorderedOrGreaterThanOrEqual,
+ 6 => FloatCC::UnorderedOrGreaterThan,
+ kind => unreachable!("kind {:?}", kind),
+ };
+
+ simd_pair_for_each_lane(fx, x, y, ret, &|fx, lane_ty, res_lane_ty, x_lane, y_lane| {
+ let res_lane = match lane_ty.kind() {
+ ty::Float(_) => fx.bcx.ins().fcmp(flt_cc, x_lane, y_lane),
+ _ => unreachable!("{:?}", lane_ty),
+ };
+ bool_to_zero_or_max_uint(fx, res_lane_ty, res_lane)
+ });
+ }
+ "llvm.x86.sse2.psrli.d" => {
+ let (a, imm8) = match args {
+ [a, imm8] => (a, imm8),
+ _ => bug!("wrong number of args for intrinsic {intrinsic}"),
+ };
+ let a = codegen_operand(fx, a);
+ let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8)
+ .expect("llvm.x86.sse2.psrli.d imm8 not const");
+
+ simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| match imm8
+ .try_to_bits(Size::from_bytes(4))
+ .unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8))
+ {
+ imm8 if imm8 < 32 => fx.bcx.ins().ushr_imm(lane, i64::from(imm8 as u8)),
+ _ => fx.bcx.ins().iconst(types::I32, 0),
+ });
+ }
+ "llvm.x86.sse2.pslli.d" => {
+ let (a, imm8) = match args {
+ [a, imm8] => (a, imm8),
+ _ => bug!("wrong number of args for intrinsic {intrinsic}"),
+ };
+ let a = codegen_operand(fx, a);
+ let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8)
+ .expect("llvm.x86.sse2.psrli.d imm8 not const");
+
+ simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| match imm8
+ .try_to_bits(Size::from_bytes(4))
+ .unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8))
+ {
+ imm8 if imm8 < 32 => fx.bcx.ins().ishl_imm(lane, i64::from(imm8 as u8)),
+ _ => fx.bcx.ins().iconst(types::I32, 0),
+ });
+ }
+ "llvm.x86.sse2.storeu.dq" => {
+ intrinsic_args!(fx, args => (mem_addr, a); intrinsic);
+ let mem_addr = mem_addr.load_scalar(fx);
+
+ // FIXME correctly handle the unalignment
+ let dest = CPlace::for_ptr(Pointer::new(mem_addr), a.layout());
+ dest.write_cvalue(fx, a);
+ }
+ "llvm.x86.addcarry.64" => {
+ intrinsic_args!(fx, args => (c_in, a, b); intrinsic);
+ let c_in = c_in.load_scalar(fx);
+
+ llvm_add_sub(fx, BinOp::Add, ret, c_in, a, b);
+ }
+ "llvm.x86.subborrow.64" => {
+ intrinsic_args!(fx, args => (b_in, a, b); intrinsic);
+ let b_in = b_in.load_scalar(fx);
+
+ llvm_add_sub(fx, BinOp::Sub, ret, b_in, a, b);
+ }
+ _ => {
+ fx.tcx.sess.warn(&format!(
+ "unsupported x86 llvm intrinsic {}; replacing with trap",
+ intrinsic
+ ));
+ crate::trap::trap_unimplemented(fx, intrinsic);
+ return;
+ }
+ }
+
+ let dest = target.expect("all llvm intrinsics used by stdlib should return");
+ let ret_block = fx.get_block(dest);
+ fx.bcx.ins().jump(ret_block, &[]);
+}
+
+// llvm.x86.avx2.vperm2i128
+// llvm.x86.ssse3.pshuf.b.128
+// llvm.x86.avx2.pshuf.b
+// llvm.x86.avx2.psrli.w
+// llvm.x86.sse2.psrli.w
+
+fn llvm_add_sub<'tcx>(
+ fx: &mut FunctionCx<'_, '_, 'tcx>,
+ bin_op: BinOp,
+ ret: CPlace<'tcx>,
+ cb_in: Value,
+ a: CValue<'tcx>,
+ b: CValue<'tcx>,
+) {
+ assert_eq!(
+ a.layout().ty,
+ fx.tcx.types.u64,
+ "llvm.x86.addcarry.64/llvm.x86.subborrow.64 second operand must be u64"
+ );
+ assert_eq!(
+ b.layout().ty,
+ fx.tcx.types.u64,
+ "llvm.x86.addcarry.64/llvm.x86.subborrow.64 third operand must be u64"
+ );
+
+ // c + carry -> c + first intermediate carry or borrow respectively
+ let int0 = crate::num::codegen_checked_int_binop(fx, bin_op, a, b);
+ let c = int0.value_field(fx, mir::Field::new(0));
+ let cb0 = int0.value_field(fx, mir::Field::new(1)).load_scalar(fx);
+
+ // c + carry -> c + second intermediate carry or borrow respectively
+ let cb_in_as_u64 = fx.bcx.ins().uextend(types::I64, cb_in);
+ let cb_in_as_u64 = CValue::by_val(cb_in_as_u64, fx.layout_of(fx.tcx.types.u64));
+ let int1 = crate::num::codegen_checked_int_binop(fx, bin_op, c, cb_in_as_u64);
+ let (c, cb1) = int1.load_scalar_pair(fx);
+
+ // carry0 | carry1 -> carry or borrow respectively
+ let cb_out = fx.bcx.ins().bor(cb0, cb1);
+
+ let layout = fx.layout_of(fx.tcx.mk_tup([fx.tcx.types.u8, fx.tcx.types.u64].iter()));
+ let val = CValue::by_val_pair(cb_out, c, layout);
+ ret.write_cvalue(fx, val);
+}
--- /dev/null
- res = fx.bcx.ins().bitcast(ty, res);
+//! Codegen of intrinsics. This includes `extern "rust-intrinsic"`, `extern "platform-intrinsic"`
+//! and LLVM intrinsics that have symbol names starting with `llvm.`.
+
+macro_rules! intrinsic_args {
+ ($fx:expr, $args:expr => ($($arg:tt),*); $intrinsic:expr) => {
+ #[allow(unused_parens)]
+ let ($($arg),*) = if let [$($arg),*] = $args {
+ ($(codegen_operand($fx, $arg)),*)
+ } else {
+ $crate::intrinsics::bug_on_incorrect_arg_count($intrinsic);
+ };
+ }
+}
+
+mod cpuid;
+mod llvm;
+mod llvm_aarch64;
+mod llvm_x86;
+mod simd;
+
+pub(crate) use cpuid::codegen_cpuid_call;
+pub(crate) use llvm::codegen_llvm_intrinsic_call;
+
+use rustc_middle::ty::print::with_no_trimmed_paths;
+use rustc_middle::ty::subst::SubstsRef;
+use rustc_span::symbol::{kw, sym, Symbol};
+
+use crate::prelude::*;
+use cranelift_codegen::ir::AtomicRmwOp;
+
+fn bug_on_incorrect_arg_count(intrinsic: impl std::fmt::Display) -> ! {
+ bug!("wrong number of args for intrinsic {}", intrinsic);
+}
+
+fn report_atomic_type_validation_error<'tcx>(
+ fx: &mut FunctionCx<'_, '_, 'tcx>,
+ intrinsic: Symbol,
+ span: Span,
+ ty: Ty<'tcx>,
+) {
+ fx.tcx.sess.span_err(
+ span,
+ &format!(
+ "`{}` intrinsic: expected basic integer or raw pointer type, found `{:?}`",
+ intrinsic, ty
+ ),
+ );
+ // Prevent verifier error
+ fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
+}
+
+pub(crate) fn clif_vector_type<'tcx>(tcx: TyCtxt<'tcx>, layout: TyAndLayout<'tcx>) -> Option<Type> {
+ let (element, count) = match layout.abi {
+ Abi::Vector { element, count } => (element, count),
+ _ => unreachable!(),
+ };
+
+ match scalar_to_clif_type(tcx, element).by(u32::try_from(count).unwrap()) {
+ // Cranelift currently only implements icmp for 128bit vectors.
+ Some(vector_ty) if vector_ty.bits() == 128 => Some(vector_ty),
+ _ => None,
+ }
+}
+
+fn simd_for_each_lane<'tcx>(
+ fx: &mut FunctionCx<'_, '_, 'tcx>,
+ val: CValue<'tcx>,
+ ret: CPlace<'tcx>,
+ f: &dyn Fn(&mut FunctionCx<'_, '_, 'tcx>, Ty<'tcx>, Ty<'tcx>, Value) -> Value,
+) {
+ let layout = val.layout();
+
+ let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx);
+ let lane_layout = fx.layout_of(lane_ty);
+ let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx);
+ let ret_lane_layout = fx.layout_of(ret_lane_ty);
+ assert_eq!(lane_count, ret_lane_count);
+
+ for lane_idx in 0..lane_count {
+ let lane = val.value_lane(fx, lane_idx).load_scalar(fx);
+
+ let res_lane = f(fx, lane_layout.ty, ret_lane_layout.ty, lane);
+ let res_lane = CValue::by_val(res_lane, ret_lane_layout);
+
+ ret.place_lane(fx, lane_idx).write_cvalue(fx, res_lane);
+ }
+}
+
+fn simd_pair_for_each_lane_typed<'tcx>(
+ fx: &mut FunctionCx<'_, '_, 'tcx>,
+ x: CValue<'tcx>,
+ y: CValue<'tcx>,
+ ret: CPlace<'tcx>,
+ f: &dyn Fn(&mut FunctionCx<'_, '_, 'tcx>, CValue<'tcx>, CValue<'tcx>) -> CValue<'tcx>,
+) {
+ assert_eq!(x.layout(), y.layout());
+ let layout = x.layout();
+
+ let (lane_count, _lane_ty) = layout.ty.simd_size_and_type(fx.tcx);
+ let (ret_lane_count, _ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx);
+ assert_eq!(lane_count, ret_lane_count);
+
+ for lane_idx in 0..lane_count {
+ let x_lane = x.value_lane(fx, lane_idx);
+ let y_lane = y.value_lane(fx, lane_idx);
+
+ let res_lane = f(fx, x_lane, y_lane);
+
+ ret.place_lane(fx, lane_idx).write_cvalue(fx, res_lane);
+ }
+}
+
+fn simd_pair_for_each_lane<'tcx>(
+ fx: &mut FunctionCx<'_, '_, 'tcx>,
+ x: CValue<'tcx>,
+ y: CValue<'tcx>,
+ ret: CPlace<'tcx>,
+ f: &dyn Fn(&mut FunctionCx<'_, '_, 'tcx>, Ty<'tcx>, Ty<'tcx>, Value, Value) -> Value,
+) {
+ assert_eq!(x.layout(), y.layout());
+ let layout = x.layout();
+
+ let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx);
+ let lane_layout = fx.layout_of(lane_ty);
+ let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx);
+ let ret_lane_layout = fx.layout_of(ret_lane_ty);
+ assert_eq!(lane_count, ret_lane_count);
+
+ for lane_idx in 0..lane_count {
+ let x_lane = x.value_lane(fx, lane_idx).load_scalar(fx);
+ let y_lane = y.value_lane(fx, lane_idx).load_scalar(fx);
+
+ let res_lane = f(fx, lane_layout.ty, ret_lane_layout.ty, x_lane, y_lane);
+ let res_lane = CValue::by_val(res_lane, ret_lane_layout);
+
+ ret.place_lane(fx, lane_idx).write_cvalue(fx, res_lane);
+ }
+}
+
+fn simd_reduce<'tcx>(
+ fx: &mut FunctionCx<'_, '_, 'tcx>,
+ val: CValue<'tcx>,
+ acc: Option<Value>,
+ ret: CPlace<'tcx>,
+ f: &dyn Fn(&mut FunctionCx<'_, '_, 'tcx>, Ty<'tcx>, Value, Value) -> Value,
+) {
+ let (lane_count, lane_ty) = val.layout().ty.simd_size_and_type(fx.tcx);
+ let lane_layout = fx.layout_of(lane_ty);
+ assert_eq!(lane_layout, ret.layout());
+
+ let (mut res_val, start_lane) =
+ if let Some(acc) = acc { (acc, 0) } else { (val.value_lane(fx, 0).load_scalar(fx), 1) };
+ for lane_idx in start_lane..lane_count {
+ let lane = val.value_lane(fx, lane_idx).load_scalar(fx);
+ res_val = f(fx, lane_layout.ty, res_val, lane);
+ }
+ let res = CValue::by_val(res_val, lane_layout);
+ ret.write_cvalue(fx, res);
+}
+
+// FIXME move all uses to `simd_reduce`
+fn simd_reduce_bool<'tcx>(
+ fx: &mut FunctionCx<'_, '_, 'tcx>,
+ val: CValue<'tcx>,
+ ret: CPlace<'tcx>,
+ f: &dyn Fn(&mut FunctionCx<'_, '_, 'tcx>, Value, Value) -> Value,
+) {
+ let (lane_count, _lane_ty) = val.layout().ty.simd_size_and_type(fx.tcx);
+ assert!(ret.layout().ty.is_bool());
+
+ let res_val = val.value_lane(fx, 0).load_scalar(fx);
+ let mut res_val = fx.bcx.ins().band_imm(res_val, 1); // mask to boolean
+ for lane_idx in 1..lane_count {
+ let lane = val.value_lane(fx, lane_idx).load_scalar(fx);
+ let lane = fx.bcx.ins().band_imm(lane, 1); // mask to boolean
+ res_val = f(fx, res_val, lane);
+ }
+ let res_val = if fx.bcx.func.dfg.value_type(res_val) != types::I8 {
+ fx.bcx.ins().ireduce(types::I8, res_val)
+ } else {
+ res_val
+ };
+ let res = CValue::by_val(res_val, ret.layout());
+ ret.write_cvalue(fx, res);
+}
+
+fn bool_to_zero_or_max_uint<'tcx>(
+ fx: &mut FunctionCx<'_, '_, 'tcx>,
+ ty: Ty<'tcx>,
+ val: Value,
+) -> Value {
+ let ty = fx.clif_type(ty).unwrap();
+
+ let int_ty = match ty {
+ types::F32 => types::I32,
+ types::F64 => types::I64,
+ ty => ty,
+ };
+
+ let mut res = fx.bcx.ins().bmask(int_ty, val);
+
+ if ty.is_float() {
- let ret_block = fx.get_block(target);
- fx.bcx.ins().jump(ret_block, &[]);
++ res = codegen_bitcast(fx, ty, res);
+ }
+
+ res
+}
+
+pub(crate) fn codegen_intrinsic_call<'tcx>(
+ fx: &mut FunctionCx<'_, '_, 'tcx>,
+ instance: Instance<'tcx>,
+ args: &[mir::Operand<'tcx>],
+ destination: CPlace<'tcx>,
+ target: Option<BasicBlock>,
+ source_info: mir::SourceInfo,
+) {
+ let intrinsic = fx.tcx.item_name(instance.def_id());
+ let substs = instance.substs;
+
+ let target = if let Some(target) = target {
+ target
+ } else {
+ // Insert non returning intrinsics here
+ match intrinsic {
+ sym::abort => {
+ fx.bcx.ins().trap(TrapCode::User(0));
+ }
+ sym::transmute => {
+ crate::base::codegen_panic(fx, "Transmuting to uninhabited type.", source_info);
+ }
+ _ => unimplemented!("unsupported intrinsic {}", intrinsic),
+ }
+ return;
+ };
+
+ if intrinsic.as_str().starts_with("simd_") {
+ self::simd::codegen_simd_intrinsic_call(
+ fx,
+ intrinsic,
+ substs,
+ args,
+ destination,
++ target,
+ source_info.span,
+ );
- crate::base::codegen_panic(
+ } else if codegen_float_intrinsic_call(fx, intrinsic, args, destination) {
+ let ret_block = fx.get_block(target);
+ fx.bcx.ins().jump(ret_block, &[]);
+ } else {
+ codegen_regular_intrinsic_call(
+ fx,
+ instance,
+ intrinsic,
+ substs,
+ args,
+ destination,
+ Some(target),
+ source_info,
+ );
+ }
+}
+
+fn codegen_float_intrinsic_call<'tcx>(
+ fx: &mut FunctionCx<'_, '_, 'tcx>,
+ intrinsic: Symbol,
+ args: &[mir::Operand<'tcx>],
+ ret: CPlace<'tcx>,
+) -> bool {
+ let (name, arg_count, ty) = match intrinsic {
+ sym::expf32 => ("expf", 1, fx.tcx.types.f32),
+ sym::expf64 => ("exp", 1, fx.tcx.types.f64),
+ sym::exp2f32 => ("exp2f", 1, fx.tcx.types.f32),
+ sym::exp2f64 => ("exp2", 1, fx.tcx.types.f64),
+ sym::sqrtf32 => ("sqrtf", 1, fx.tcx.types.f32),
+ sym::sqrtf64 => ("sqrt", 1, fx.tcx.types.f64),
+ sym::powif32 => ("__powisf2", 2, fx.tcx.types.f32), // compiler-builtins
+ sym::powif64 => ("__powidf2", 2, fx.tcx.types.f64), // compiler-builtins
+ sym::powf32 => ("powf", 2, fx.tcx.types.f32),
+ sym::powf64 => ("pow", 2, fx.tcx.types.f64),
+ sym::logf32 => ("logf", 1, fx.tcx.types.f32),
+ sym::logf64 => ("log", 1, fx.tcx.types.f64),
+ sym::log2f32 => ("log2f", 1, fx.tcx.types.f32),
+ sym::log2f64 => ("log2", 1, fx.tcx.types.f64),
+ sym::log10f32 => ("log10f", 1, fx.tcx.types.f32),
+ sym::log10f64 => ("log10", 1, fx.tcx.types.f64),
+ sym::fabsf32 => ("fabsf", 1, fx.tcx.types.f32),
+ sym::fabsf64 => ("fabs", 1, fx.tcx.types.f64),
+ sym::fmaf32 => ("fmaf", 3, fx.tcx.types.f32),
+ sym::fmaf64 => ("fma", 3, fx.tcx.types.f64),
+ sym::copysignf32 => ("copysignf", 2, fx.tcx.types.f32),
+ sym::copysignf64 => ("copysign", 2, fx.tcx.types.f64),
+ sym::floorf32 => ("floorf", 1, fx.tcx.types.f32),
+ sym::floorf64 => ("floor", 1, fx.tcx.types.f64),
+ sym::ceilf32 => ("ceilf", 1, fx.tcx.types.f32),
+ sym::ceilf64 => ("ceil", 1, fx.tcx.types.f64),
+ sym::truncf32 => ("truncf", 1, fx.tcx.types.f32),
+ sym::truncf64 => ("trunc", 1, fx.tcx.types.f64),
+ sym::roundf32 => ("roundf", 1, fx.tcx.types.f32),
+ sym::roundf64 => ("round", 1, fx.tcx.types.f64),
+ sym::sinf32 => ("sinf", 1, fx.tcx.types.f32),
+ sym::sinf64 => ("sin", 1, fx.tcx.types.f64),
+ sym::cosf32 => ("cosf", 1, fx.tcx.types.f32),
+ sym::cosf64 => ("cos", 1, fx.tcx.types.f64),
+ _ => return false,
+ };
+
+ if args.len() != arg_count {
+ bug!("wrong number of args for intrinsic {:?}", intrinsic);
+ }
+
+ let (a, b, c);
+ let args = match args {
+ [x] => {
+ a = [codegen_operand(fx, x)];
+ &a as &[_]
+ }
+ [x, y] => {
+ b = [codegen_operand(fx, x), codegen_operand(fx, y)];
+ &b
+ }
+ [x, y, z] => {
+ c = [codegen_operand(fx, x), codegen_operand(fx, y), codegen_operand(fx, z)];
+ &c
+ }
+ _ => unreachable!(),
+ };
+
+ let layout = fx.layout_of(ty);
+ let res = match intrinsic {
+ sym::fmaf32 | sym::fmaf64 => {
+ let a = args[0].load_scalar(fx);
+ let b = args[1].load_scalar(fx);
+ let c = args[2].load_scalar(fx);
+ CValue::by_val(fx.bcx.ins().fma(a, b, c), layout)
+ }
+ sym::copysignf32 | sym::copysignf64 => {
+ let a = args[0].load_scalar(fx);
+ let b = args[1].load_scalar(fx);
+ CValue::by_val(fx.bcx.ins().fcopysign(a, b), layout)
+ }
+ sym::fabsf32
+ | sym::fabsf64
+ | sym::floorf32
+ | sym::floorf64
+ | sym::ceilf32
+ | sym::ceilf64
+ | sym::truncf32
+ | sym::truncf64 => {
+ let a = args[0].load_scalar(fx);
+
+ let val = match intrinsic {
+ sym::fabsf32 | sym::fabsf64 => fx.bcx.ins().fabs(a),
+ sym::floorf32 | sym::floorf64 => fx.bcx.ins().floor(a),
+ sym::ceilf32 | sym::ceilf64 => fx.bcx.ins().ceil(a),
+ sym::truncf32 | sym::truncf64 => fx.bcx.ins().trunc(a),
+ _ => unreachable!(),
+ };
+
+ CValue::by_val(val, layout)
+ }
+ // These intrinsics aren't supported natively by Cranelift.
+ // Lower them to a libcall.
+ _ => fx.easy_call(name, &args, ty),
+ };
+
+ ret.write_cvalue(fx, res);
+
+ true
+}
+
+fn codegen_regular_intrinsic_call<'tcx>(
+ fx: &mut FunctionCx<'_, '_, 'tcx>,
+ instance: Instance<'tcx>,
+ intrinsic: Symbol,
+ substs: SubstsRef<'tcx>,
+ args: &[mir::Operand<'tcx>],
+ ret: CPlace<'tcx>,
+ destination: Option<BasicBlock>,
+ source_info: mir::SourceInfo,
+) {
+ let usize_layout = fx.layout_of(fx.tcx.types.usize);
+
+ match intrinsic {
+ sym::likely | sym::unlikely => {
+ intrinsic_args!(fx, args => (a); intrinsic);
+
+ ret.write_cvalue(fx, a);
+ }
+ sym::breakpoint => {
+ intrinsic_args!(fx, args => (); intrinsic);
+
+ fx.bcx.ins().debugtrap();
+ }
+ sym::copy | sym::copy_nonoverlapping => {
+ intrinsic_args!(fx, args => (src, dst, count); intrinsic);
+ let src = src.load_scalar(fx);
+ let dst = dst.load_scalar(fx);
+ let count = count.load_scalar(fx);
+
+ let elem_ty = substs.type_at(0);
+ let elem_size: u64 = fx.layout_of(elem_ty).size.bytes();
+ assert_eq!(args.len(), 3);
+ let byte_amount =
+ if elem_size != 1 { fx.bcx.ins().imul_imm(count, elem_size as i64) } else { count };
+
+ if intrinsic == sym::copy_nonoverlapping {
+ // FIXME emit_small_memcpy
+ fx.bcx.call_memcpy(fx.target_config, dst, src, byte_amount);
+ } else {
+ // FIXME emit_small_memmove
+ fx.bcx.call_memmove(fx.target_config, dst, src, byte_amount);
+ }
+ }
+ sym::volatile_copy_memory | sym::volatile_copy_nonoverlapping_memory => {
+ // NOTE: the volatile variants have src and dst swapped
+ intrinsic_args!(fx, args => (dst, src, count); intrinsic);
+ let dst = dst.load_scalar(fx);
+ let src = src.load_scalar(fx);
+ let count = count.load_scalar(fx);
+
+ let elem_ty = substs.type_at(0);
+ let elem_size: u64 = fx.layout_of(elem_ty).size.bytes();
+ assert_eq!(args.len(), 3);
+ let byte_amount =
+ if elem_size != 1 { fx.bcx.ins().imul_imm(count, elem_size as i64) } else { count };
+
+ // FIXME make the copy actually volatile when using emit_small_mem{cpy,move}
+ if intrinsic == sym::volatile_copy_nonoverlapping_memory {
+ // FIXME emit_small_memcpy
+ fx.bcx.call_memcpy(fx.target_config, dst, src, byte_amount);
+ } else {
+ // FIXME emit_small_memmove
+ fx.bcx.call_memmove(fx.target_config, dst, src, byte_amount);
+ }
+ }
+ sym::size_of_val => {
+ intrinsic_args!(fx, args => (ptr); intrinsic);
+
+ let layout = fx.layout_of(substs.type_at(0));
+ // Note: Can't use is_unsized here as truly unsized types need to take the fixed size
+ // branch
+ let size = if let Abi::ScalarPair(_, _) = ptr.layout().abi {
+ let (_ptr, info) = ptr.load_scalar_pair(fx);
+ let (size, _align) = crate::unsize::size_and_align_of_dst(fx, layout, info);
+ size
+ } else {
+ fx.bcx.ins().iconst(fx.pointer_type, layout.size.bytes() as i64)
+ };
+ ret.write_cvalue(fx, CValue::by_val(size, usize_layout));
+ }
+ sym::min_align_of_val => {
+ intrinsic_args!(fx, args => (ptr); intrinsic);
+
+ let layout = fx.layout_of(substs.type_at(0));
+ // Note: Can't use is_unsized here as truly unsized types need to take the fixed size
+ // branch
+ let align = if let Abi::ScalarPair(_, _) = ptr.layout().abi {
+ let (_ptr, info) = ptr.load_scalar_pair(fx);
+ let (_size, align) = crate::unsize::size_and_align_of_dst(fx, layout, info);
+ align
+ } else {
+ fx.bcx.ins().iconst(fx.pointer_type, layout.align.abi.bytes() as i64)
+ };
+ ret.write_cvalue(fx, CValue::by_val(align, usize_layout));
+ }
+
+ sym::vtable_size => {
+ intrinsic_args!(fx, args => (vtable); intrinsic);
+ let vtable = vtable.load_scalar(fx);
+
+ let size = crate::vtable::size_of_obj(fx, vtable);
+ ret.write_cvalue(fx, CValue::by_val(size, usize_layout));
+ }
+
+ sym::vtable_align => {
+ intrinsic_args!(fx, args => (vtable); intrinsic);
+ let vtable = vtable.load_scalar(fx);
+
+ let align = crate::vtable::min_align_of_obj(fx, vtable);
+ ret.write_cvalue(fx, CValue::by_val(align, usize_layout));
+ }
+
+ sym::unchecked_add
+ | sym::unchecked_sub
+ | sym::unchecked_mul
+ | sym::unchecked_div
+ | sym::exact_div
+ | sym::unchecked_rem
+ | sym::unchecked_shl
+ | sym::unchecked_shr => {
+ intrinsic_args!(fx, args => (x, y); intrinsic);
+
+ // FIXME trap on overflow
+ let bin_op = match intrinsic {
+ sym::unchecked_add => BinOp::Add,
+ sym::unchecked_sub => BinOp::Sub,
+ sym::unchecked_mul => BinOp::Mul,
+ sym::unchecked_div | sym::exact_div => BinOp::Div,
+ sym::unchecked_rem => BinOp::Rem,
+ sym::unchecked_shl => BinOp::Shl,
+ sym::unchecked_shr => BinOp::Shr,
+ _ => unreachable!(),
+ };
+ let res = crate::num::codegen_int_binop(fx, bin_op, x, y);
+ ret.write_cvalue(fx, res);
+ }
+ sym::add_with_overflow | sym::sub_with_overflow | sym::mul_with_overflow => {
+ intrinsic_args!(fx, args => (x, y); intrinsic);
+
+ assert_eq!(x.layout().ty, y.layout().ty);
+ let bin_op = match intrinsic {
+ sym::add_with_overflow => BinOp::Add,
+ sym::sub_with_overflow => BinOp::Sub,
+ sym::mul_with_overflow => BinOp::Mul,
+ _ => unreachable!(),
+ };
+
+ let res = crate::num::codegen_checked_int_binop(fx, bin_op, x, y);
+ ret.write_cvalue(fx, res);
+ }
+ sym::saturating_add | sym::saturating_sub => {
+ intrinsic_args!(fx, args => (lhs, rhs); intrinsic);
+
+ assert_eq!(lhs.layout().ty, rhs.layout().ty);
+ let bin_op = match intrinsic {
+ sym::saturating_add => BinOp::Add,
+ sym::saturating_sub => BinOp::Sub,
+ _ => unreachable!(),
+ };
+
+ let res = crate::num::codegen_saturating_int_binop(fx, bin_op, lhs, rhs);
+ ret.write_cvalue(fx, res);
+ }
+ sym::rotate_left => {
+ intrinsic_args!(fx, args => (x, y); intrinsic);
+ let y = y.load_scalar(fx);
+
+ let layout = x.layout();
+ let x = x.load_scalar(fx);
+ let res = fx.bcx.ins().rotl(x, y);
+ ret.write_cvalue(fx, CValue::by_val(res, layout));
+ }
+ sym::rotate_right => {
+ intrinsic_args!(fx, args => (x, y); intrinsic);
+ let y = y.load_scalar(fx);
+
+ let layout = x.layout();
+ let x = x.load_scalar(fx);
+ let res = fx.bcx.ins().rotr(x, y);
+ ret.write_cvalue(fx, CValue::by_val(res, layout));
+ }
+
+ // The only difference between offset and arith_offset is regarding UB. Because Cranelift
+ // doesn't have UB both are codegen'ed the same way
+ sym::offset | sym::arith_offset => {
+ intrinsic_args!(fx, args => (base, offset); intrinsic);
+ let offset = offset.load_scalar(fx);
+
+ let pointee_ty = base.layout().ty.builtin_deref(true).unwrap().ty;
+ let pointee_size = fx.layout_of(pointee_ty).size.bytes();
+ let ptr_diff = if pointee_size != 1 {
+ fx.bcx.ins().imul_imm(offset, pointee_size as i64)
+ } else {
+ offset
+ };
+ let base_val = base.load_scalar(fx);
+ let res = fx.bcx.ins().iadd(base_val, ptr_diff);
+ ret.write_cvalue(fx, CValue::by_val(res, base.layout()));
+ }
+
+ sym::ptr_mask => {
+ intrinsic_args!(fx, args => (ptr, mask); intrinsic);
+ let ptr = ptr.load_scalar(fx);
+ let mask = mask.load_scalar(fx);
+ fx.bcx.ins().band(ptr, mask);
+ }
+
+ sym::transmute => {
+ intrinsic_args!(fx, args => (from); intrinsic);
+
+ ret.write_cvalue_transmute(fx, from);
+ }
+ sym::write_bytes | sym::volatile_set_memory => {
+ intrinsic_args!(fx, args => (dst, val, count); intrinsic);
+ let val = val.load_scalar(fx);
+ let count = count.load_scalar(fx);
+
+ let pointee_ty = dst.layout().ty.builtin_deref(true).unwrap().ty;
+ let pointee_size = fx.layout_of(pointee_ty).size.bytes();
+ let count = if pointee_size != 1 {
+ fx.bcx.ins().imul_imm(count, pointee_size as i64)
+ } else {
+ count
+ };
+ let dst_ptr = dst.load_scalar(fx);
+ // FIXME make the memset actually volatile when switching to emit_small_memset
+ // FIXME use emit_small_memset
+ fx.bcx.call_memset(fx.target_config, dst_ptr, val, count);
+ }
+ sym::ctlz | sym::ctlz_nonzero => {
+ intrinsic_args!(fx, args => (arg); intrinsic);
+ let val = arg.load_scalar(fx);
+
+ // FIXME trap on `ctlz_nonzero` with zero arg.
+ let res = fx.bcx.ins().clz(val);
+ let res = CValue::by_val(res, arg.layout());
+ ret.write_cvalue(fx, res);
+ }
+ sym::cttz | sym::cttz_nonzero => {
+ intrinsic_args!(fx, args => (arg); intrinsic);
+ let val = arg.load_scalar(fx);
+
+ // FIXME trap on `cttz_nonzero` with zero arg.
+ let res = fx.bcx.ins().ctz(val);
+ let res = CValue::by_val(res, arg.layout());
+ ret.write_cvalue(fx, res);
+ }
+ sym::ctpop => {
+ intrinsic_args!(fx, args => (arg); intrinsic);
+ let val = arg.load_scalar(fx);
+
+ let res = fx.bcx.ins().popcnt(val);
+ let res = CValue::by_val(res, arg.layout());
+ ret.write_cvalue(fx, res);
+ }
+ sym::bitreverse => {
+ intrinsic_args!(fx, args => (arg); intrinsic);
+ let val = arg.load_scalar(fx);
+
+ let res = fx.bcx.ins().bitrev(val);
+ let res = CValue::by_val(res, arg.layout());
+ ret.write_cvalue(fx, res);
+ }
+ sym::bswap => {
+ intrinsic_args!(fx, args => (arg); intrinsic);
+ let val = arg.load_scalar(fx);
+
+ let res = if fx.bcx.func.dfg.value_type(val) == types::I8 {
+ val
+ } else {
+ fx.bcx.ins().bswap(val)
+ };
+ let res = CValue::by_val(res, arg.layout());
+ ret.write_cvalue(fx, res);
+ }
+ sym::assert_inhabited | sym::assert_zero_valid | sym::assert_mem_uninitialized_valid => {
+ intrinsic_args!(fx, args => (); intrinsic);
+
+ let layout = fx.layout_of(substs.type_at(0));
+ if layout.abi.is_uninhabited() {
+ with_no_trimmed_paths!({
- crate::base::codegen_panic(
++ crate::base::codegen_panic_nounwind(
+ fx,
+ &format!("attempted to instantiate uninhabited type `{}`", layout.ty),
+ source_info,
+ )
+ });
+ return;
+ }
+
+ if intrinsic == sym::assert_zero_valid && !fx.tcx.permits_zero_init(layout) {
+ with_no_trimmed_paths!({
- crate::base::codegen_panic(
++ crate::base::codegen_panic_nounwind(
+ fx,
+ &format!(
+ "attempted to zero-initialize type `{}`, which is invalid",
+ layout.ty
+ ),
+ source_info,
+ );
+ });
+ return;
+ }
+
+ if intrinsic == sym::assert_mem_uninitialized_valid
+ && !fx.tcx.permits_uninit_init(layout)
+ {
+ with_no_trimmed_paths!({
++ crate::base::codegen_panic_nounwind(
+ fx,
+ &format!(
+ "attempted to leave type `{}` uninitialized, which is invalid",
+ layout.ty
+ ),
+ source_info,
+ )
+ });
+ return;
+ }
+ }
+
+ sym::volatile_load | sym::unaligned_volatile_load => {
+ intrinsic_args!(fx, args => (ptr); intrinsic);
+
+ // Cranelift treats loads as volatile by default
+ // FIXME correctly handle unaligned_volatile_load
+ let inner_layout = fx.layout_of(ptr.layout().ty.builtin_deref(true).unwrap().ty);
+ let val = CValue::by_ref(Pointer::new(ptr.load_scalar(fx)), inner_layout);
+ ret.write_cvalue(fx, val);
+ }
+ sym::volatile_store | sym::unaligned_volatile_store => {
+ intrinsic_args!(fx, args => (ptr, val); intrinsic);
+ let ptr = ptr.load_scalar(fx);
+
+ // Cranelift treats stores as volatile by default
+ // FIXME correctly handle unaligned_volatile_store
+ let dest = CPlace::for_ptr(Pointer::new(ptr), val.layout());
+ dest.write_cvalue(fx, val);
+ }
+
+ sym::pref_align_of
+ | sym::needs_drop
+ | sym::type_id
+ | sym::type_name
+ | sym::variant_count => {
+ intrinsic_args!(fx, args => (); intrinsic);
+
+ let const_val =
+ fx.tcx.const_eval_instance(ParamEnv::reveal_all(), instance, None).unwrap();
+ let val = crate::constant::codegen_const_value(fx, const_val, ret.layout().ty);
+ ret.write_cvalue(fx, val);
+ }
+
+ sym::ptr_offset_from | sym::ptr_offset_from_unsigned => {
+ intrinsic_args!(fx, args => (ptr, base); intrinsic);
+ let ptr = ptr.load_scalar(fx);
+ let base = base.load_scalar(fx);
+ let ty = substs.type_at(0);
+
+ let pointee_size: u64 = fx.layout_of(ty).size.bytes();
+ let diff_bytes = fx.bcx.ins().isub(ptr, base);
+ // FIXME this can be an exact division.
+ let val = if intrinsic == sym::ptr_offset_from_unsigned {
+ let usize_layout = fx.layout_of(fx.tcx.types.usize);
+ // Because diff_bytes ULE isize::MAX, this would be fine as signed,
+ // but unsigned is slightly easier to codegen, so might as well.
+ CValue::by_val(fx.bcx.ins().udiv_imm(diff_bytes, pointee_size as i64), usize_layout)
+ } else {
+ let isize_layout = fx.layout_of(fx.tcx.types.isize);
+ CValue::by_val(fx.bcx.ins().sdiv_imm(diff_bytes, pointee_size as i64), isize_layout)
+ };
+ ret.write_cvalue(fx, val);
+ }
+
+ sym::ptr_guaranteed_cmp => {
+ intrinsic_args!(fx, args => (a, b); intrinsic);
+
+ let val = crate::num::codegen_ptr_binop(fx, BinOp::Eq, a, b).load_scalar(fx);
+ ret.write_cvalue(fx, CValue::by_val(val, fx.layout_of(fx.tcx.types.u8)));
+ }
+
+ sym::caller_location => {
+ intrinsic_args!(fx, args => (); intrinsic);
+
+ let caller_location = fx.get_caller_location(source_info);
+ ret.write_cvalue(fx, caller_location);
+ }
+
+ _ if intrinsic.as_str().starts_with("atomic_fence") => {
+ intrinsic_args!(fx, args => (); intrinsic);
+
+ fx.bcx.ins().fence();
+ }
+ _ if intrinsic.as_str().starts_with("atomic_singlethreadfence") => {
+ intrinsic_args!(fx, args => (); intrinsic);
+
+ // FIXME use a compiler fence once Cranelift supports it
+ fx.bcx.ins().fence();
+ }
+ _ if intrinsic.as_str().starts_with("atomic_load") => {
+ intrinsic_args!(fx, args => (ptr); intrinsic);
+ let ptr = ptr.load_scalar(fx);
+
+ let ty = substs.type_at(0);
+ match ty.kind() {
+ ty::Uint(UintTy::U128) | ty::Int(IntTy::I128) => {
+ // FIXME implement 128bit atomics
+ if fx.tcx.is_compiler_builtins(LOCAL_CRATE) {
+ // special case for compiler-builtins to avoid having to patch it
+ crate::trap::trap_unimplemented(fx, "128bit atomics not yet supported");
+ return;
+ } else {
+ fx.tcx
+ .sess
+ .span_fatal(source_info.span, "128bit atomics not yet supported");
+ }
+ }
+ ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
+ _ => {
+ report_atomic_type_validation_error(fx, intrinsic, source_info.span, ty);
+ return;
+ }
+ }
+ let clif_ty = fx.clif_type(ty).unwrap();
+
+ let val = fx.bcx.ins().atomic_load(clif_ty, MemFlags::trusted(), ptr);
+
+ let val = CValue::by_val(val, fx.layout_of(ty));
+ ret.write_cvalue(fx, val);
+ }
+ _ if intrinsic.as_str().starts_with("atomic_store") => {
+ intrinsic_args!(fx, args => (ptr, val); intrinsic);
+ let ptr = ptr.load_scalar(fx);
+
+ let ty = substs.type_at(0);
+ match ty.kind() {
+ ty::Uint(UintTy::U128) | ty::Int(IntTy::I128) => {
+ // FIXME implement 128bit atomics
+ if fx.tcx.is_compiler_builtins(LOCAL_CRATE) {
+ // special case for compiler-builtins to avoid having to patch it
+ crate::trap::trap_unimplemented(fx, "128bit atomics not yet supported");
+ return;
+ } else {
+ fx.tcx
+ .sess
+ .span_fatal(source_info.span, "128bit atomics not yet supported");
+ }
+ }
+ ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
+ _ => {
+ report_atomic_type_validation_error(fx, intrinsic, source_info.span, ty);
+ return;
+ }
+ }
+
+ let val = val.load_scalar(fx);
+
+ fx.bcx.ins().atomic_store(MemFlags::trusted(), val, ptr);
+ }
+ _ if intrinsic.as_str().starts_with("atomic_xchg") => {
+ intrinsic_args!(fx, args => (ptr, new); intrinsic);
+ let ptr = ptr.load_scalar(fx);
+
+ let layout = new.layout();
+ match layout.ty.kind() {
+ ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
+ _ => {
+ report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
+ return;
+ }
+ }
+ let ty = fx.clif_type(layout.ty).unwrap();
+
+ let new = new.load_scalar(fx);
+
+ let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Xchg, ptr, new);
+
+ let old = CValue::by_val(old, layout);
+ ret.write_cvalue(fx, old);
+ }
+ _ if intrinsic.as_str().starts_with("atomic_cxchg") => {
+ // both atomic_cxchg_* and atomic_cxchgweak_*
+ intrinsic_args!(fx, args => (ptr, test_old, new); intrinsic);
+ let ptr = ptr.load_scalar(fx);
+
+ let layout = new.layout();
+ match layout.ty.kind() {
+ ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
+ _ => {
+ report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
+ return;
+ }
+ }
+
+ let test_old = test_old.load_scalar(fx);
+ let new = new.load_scalar(fx);
+
+ let old = fx.bcx.ins().atomic_cas(MemFlags::trusted(), ptr, test_old, new);
+ let is_eq = fx.bcx.ins().icmp(IntCC::Equal, old, test_old);
+
+ let ret_val = CValue::by_val_pair(old, is_eq, ret.layout());
+ ret.write_cvalue(fx, ret_val)
+ }
+
+ _ if intrinsic.as_str().starts_with("atomic_xadd") => {
+ intrinsic_args!(fx, args => (ptr, amount); intrinsic);
+ let ptr = ptr.load_scalar(fx);
+
+ let layout = amount.layout();
+ match layout.ty.kind() {
+ ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
+ _ => {
+ report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
+ return;
+ }
+ }
+ let ty = fx.clif_type(layout.ty).unwrap();
+
+ let amount = amount.load_scalar(fx);
+
+ let old =
+ fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Add, ptr, amount);
+
+ let old = CValue::by_val(old, layout);
+ ret.write_cvalue(fx, old);
+ }
+ _ if intrinsic.as_str().starts_with("atomic_xsub") => {
+ intrinsic_args!(fx, args => (ptr, amount); intrinsic);
+ let ptr = ptr.load_scalar(fx);
+
+ let layout = amount.layout();
+ match layout.ty.kind() {
+ ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
+ _ => {
+ report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
+ return;
+ }
+ }
+ let ty = fx.clif_type(layout.ty).unwrap();
+
+ let amount = amount.load_scalar(fx);
+
+ let old =
+ fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Sub, ptr, amount);
+
+ let old = CValue::by_val(old, layout);
+ ret.write_cvalue(fx, old);
+ }
+ _ if intrinsic.as_str().starts_with("atomic_and") => {
+ intrinsic_args!(fx, args => (ptr, src); intrinsic);
+ let ptr = ptr.load_scalar(fx);
+
+ let layout = src.layout();
+ match layout.ty.kind() {
+ ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
+ _ => {
+ report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
+ return;
+ }
+ }
+ let ty = fx.clif_type(layout.ty).unwrap();
+
+ let src = src.load_scalar(fx);
+
+ let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::And, ptr, src);
+
+ let old = CValue::by_val(old, layout);
+ ret.write_cvalue(fx, old);
+ }
+ _ if intrinsic.as_str().starts_with("atomic_or") => {
+ intrinsic_args!(fx, args => (ptr, src); intrinsic);
+ let ptr = ptr.load_scalar(fx);
+
+ let layout = src.layout();
+ match layout.ty.kind() {
+ ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
+ _ => {
+ report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
+ return;
+ }
+ }
+ let ty = fx.clif_type(layout.ty).unwrap();
+
+ let src = src.load_scalar(fx);
+
+ let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Or, ptr, src);
+
+ let old = CValue::by_val(old, layout);
+ ret.write_cvalue(fx, old);
+ }
+ _ if intrinsic.as_str().starts_with("atomic_xor") => {
+ intrinsic_args!(fx, args => (ptr, src); intrinsic);
+ let ptr = ptr.load_scalar(fx);
+
+ let layout = src.layout();
+ match layout.ty.kind() {
+ ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
+ _ => {
+ report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
+ return;
+ }
+ }
+ let ty = fx.clif_type(layout.ty).unwrap();
+
+ let src = src.load_scalar(fx);
+
+ let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Xor, ptr, src);
+
+ let old = CValue::by_val(old, layout);
+ ret.write_cvalue(fx, old);
+ }
+ _ if intrinsic.as_str().starts_with("atomic_nand") => {
+ intrinsic_args!(fx, args => (ptr, src); intrinsic);
+ let ptr = ptr.load_scalar(fx);
+
+ let layout = src.layout();
+ match layout.ty.kind() {
+ ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
+ _ => {
+ report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
+ return;
+ }
+ }
+ let ty = fx.clif_type(layout.ty).unwrap();
+
+ let src = src.load_scalar(fx);
+
+ let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Nand, ptr, src);
+
+ let old = CValue::by_val(old, layout);
+ ret.write_cvalue(fx, old);
+ }
+ _ if intrinsic.as_str().starts_with("atomic_max") => {
+ intrinsic_args!(fx, args => (ptr, src); intrinsic);
+ let ptr = ptr.load_scalar(fx);
+
+ let layout = src.layout();
+ match layout.ty.kind() {
+ ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
+ _ => {
+ report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
+ return;
+ }
+ }
+ let ty = fx.clif_type(layout.ty).unwrap();
+
+ let src = src.load_scalar(fx);
+
+ let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Smax, ptr, src);
+
+ let old = CValue::by_val(old, layout);
+ ret.write_cvalue(fx, old);
+ }
+ _ if intrinsic.as_str().starts_with("atomic_umax") => {
+ intrinsic_args!(fx, args => (ptr, src); intrinsic);
+ let ptr = ptr.load_scalar(fx);
+
+ let layout = src.layout();
+ match layout.ty.kind() {
+ ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
+ _ => {
+ report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
+ return;
+ }
+ }
+ let ty = fx.clif_type(layout.ty).unwrap();
+
+ let src = src.load_scalar(fx);
+
+ let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Umax, ptr, src);
+
+ let old = CValue::by_val(old, layout);
+ ret.write_cvalue(fx, old);
+ }
+ _ if intrinsic.as_str().starts_with("atomic_min") => {
+ intrinsic_args!(fx, args => (ptr, src); intrinsic);
+ let ptr = ptr.load_scalar(fx);
+
+ let layout = src.layout();
+ match layout.ty.kind() {
+ ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
+ _ => {
+ report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
+ return;
+ }
+ }
+ let ty = fx.clif_type(layout.ty).unwrap();
+
+ let src = src.load_scalar(fx);
+
+ let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Smin, ptr, src);
+
+ let old = CValue::by_val(old, layout);
+ ret.write_cvalue(fx, old);
+ }
+ _ if intrinsic.as_str().starts_with("atomic_umin") => {
+ intrinsic_args!(fx, args => (ptr, src); intrinsic);
+ let ptr = ptr.load_scalar(fx);
+
+ let layout = src.layout();
+ match layout.ty.kind() {
+ ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
+ _ => {
+ report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
+ return;
+ }
+ }
+ let ty = fx.clif_type(layout.ty).unwrap();
+
+ let src = src.load_scalar(fx);
+
+ let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Umin, ptr, src);
+
+ let old = CValue::by_val(old, layout);
+ ret.write_cvalue(fx, old);
+ }
+
+ sym::minnumf32 => {
+ intrinsic_args!(fx, args => (a, b); intrinsic);
+ let a = a.load_scalar(fx);
+ let b = b.load_scalar(fx);
+
+ let val = crate::num::codegen_float_min(fx, a, b);
+ let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f32));
+ ret.write_cvalue(fx, val);
+ }
+ sym::minnumf64 => {
+ intrinsic_args!(fx, args => (a, b); intrinsic);
+ let a = a.load_scalar(fx);
+ let b = b.load_scalar(fx);
+
+ let val = crate::num::codegen_float_min(fx, a, b);
+ let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f64));
+ ret.write_cvalue(fx, val);
+ }
+ sym::maxnumf32 => {
+ intrinsic_args!(fx, args => (a, b); intrinsic);
+ let a = a.load_scalar(fx);
+ let b = b.load_scalar(fx);
+
+ let val = crate::num::codegen_float_max(fx, a, b);
+ let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f32));
+ ret.write_cvalue(fx, val);
+ }
+ sym::maxnumf64 => {
+ intrinsic_args!(fx, args => (a, b); intrinsic);
+ let a = a.load_scalar(fx);
+ let b = b.load_scalar(fx);
+
+ let val = crate::num::codegen_float_max(fx, a, b);
+ let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f64));
+ ret.write_cvalue(fx, val);
+ }
+
+ kw::Try => {
+ intrinsic_args!(fx, args => (f, data, catch_fn); intrinsic);
+ let f = f.load_scalar(fx);
+ let data = data.load_scalar(fx);
+ let _catch_fn = catch_fn.load_scalar(fx);
+
+ // FIXME once unwinding is supported, change this to actually catch panics
+ let f_sig = fx.bcx.func.import_signature(Signature {
+ call_conv: fx.target_config.default_call_conv,
+ params: vec![AbiParam::new(pointer_ty(fx.tcx))],
+ returns: vec![],
+ });
+
+ fx.bcx.ins().call_indirect(f_sig, f, &[data]);
+
+ let layout = ret.layout();
+ let ret_val = CValue::const_val(fx, layout, ty::ScalarInt::null(layout.size));
+ ret.write_cvalue(fx, ret_val);
+ }
+
+ sym::fadd_fast | sym::fsub_fast | sym::fmul_fast | sym::fdiv_fast | sym::frem_fast => {
+ intrinsic_args!(fx, args => (x, y); intrinsic);
+
+ let res = crate::num::codegen_float_binop(
+ fx,
+ match intrinsic {
+ sym::fadd_fast => BinOp::Add,
+ sym::fsub_fast => BinOp::Sub,
+ sym::fmul_fast => BinOp::Mul,
+ sym::fdiv_fast => BinOp::Div,
+ sym::frem_fast => BinOp::Rem,
+ _ => unreachable!(),
+ },
+ x,
+ y,
+ );
+ ret.write_cvalue(fx, res);
+ }
+ sym::float_to_int_unchecked => {
+ intrinsic_args!(fx, args => (f); intrinsic);
+ let f = f.load_scalar(fx);
+
+ let res = crate::cast::clif_int_or_float_cast(
+ fx,
+ f,
+ false,
+ fx.clif_type(ret.layout().ty).unwrap(),
+ type_sign(ret.layout().ty),
+ );
+ ret.write_cvalue(fx, CValue::by_val(res, ret.layout()));
+ }
+
+ sym::raw_eq => {
+ intrinsic_args!(fx, args => (lhs_ref, rhs_ref); intrinsic);
+ let lhs_ref = lhs_ref.load_scalar(fx);
+ let rhs_ref = rhs_ref.load_scalar(fx);
+
+ let size = fx.layout_of(substs.type_at(0)).layout.size();
+ // FIXME add and use emit_small_memcmp
+ let is_eq_value = if size == Size::ZERO {
+ // No bytes means they're trivially equal
+ fx.bcx.ins().iconst(types::I8, 1)
+ } else if let Some(clty) = size.bits().try_into().ok().and_then(Type::int) {
+ // Can't use `trusted` for these loads; they could be unaligned.
+ let mut flags = MemFlags::new();
+ flags.set_notrap();
+ let lhs_val = fx.bcx.ins().load(clty, flags, lhs_ref, 0);
+ let rhs_val = fx.bcx.ins().load(clty, flags, rhs_ref, 0);
+ fx.bcx.ins().icmp(IntCC::Equal, lhs_val, rhs_val)
+ } else {
+ // Just call `memcmp` (like slices do in core) when the
+ // size is too large or it's not a power-of-two.
+ let signed_bytes = i64::try_from(size.bytes()).unwrap();
+ let bytes_val = fx.bcx.ins().iconst(fx.pointer_type, signed_bytes);
+ let params = vec![AbiParam::new(fx.pointer_type); 3];
+ let returns = vec![AbiParam::new(types::I32)];
+ let args = &[lhs_ref, rhs_ref, bytes_val];
+ let cmp = fx.lib_call("memcmp", params, returns, args)[0];
+ fx.bcx.ins().icmp_imm(IntCC::Equal, cmp, 0)
+ };
+ ret.write_cvalue(fx, CValue::by_val(is_eq_value, ret.layout()));
+ }
+
+ sym::const_allocate => {
+ intrinsic_args!(fx, args => (_size, _align); intrinsic);
+
+ // returns a null pointer at runtime.
+ let null = fx.bcx.ins().iconst(fx.pointer_type, 0);
+ ret.write_cvalue(fx, CValue::by_val(null, ret.layout()));
+ }
+
+ sym::const_deallocate => {
+ intrinsic_args!(fx, args => (_ptr, _size, _align); intrinsic);
+ // nop at runtime.
+ }
+
+ sym::black_box => {
+ intrinsic_args!(fx, args => (a); intrinsic);
+
+ // FIXME implement black_box semantics
+ ret.write_cvalue(fx, a);
+ }
+
+ // FIXME implement variadics in cranelift
+ sym::va_copy | sym::va_arg | sym::va_end => {
+ fx.tcx.sess.span_fatal(
+ source_info.span,
+ "Defining variadic functions is not yet supported by Cranelift",
+ );
+ }
+
+ _ => {
+ fx.tcx
+ .sess
+ .span_fatal(source_info.span, &format!("unsupported intrinsic {}", intrinsic));
+ }
+ }
+
+ let ret_block = fx.get_block(destination.unwrap());
+ fx.bcx.ins().jump(ret_block, &[]);
+}
--- /dev/null
- let dummy_block = fx.bcx.create_block();
+//! Codegen `extern "platform-intrinsic"` intrinsics.
+
+use rustc_middle::ty::subst::SubstsRef;
+use rustc_span::Symbol;
+use rustc_target::abi::Endian;
+
+use super::*;
+use crate::prelude::*;
+
+fn report_simd_type_validation_error(
+ fx: &mut FunctionCx<'_, '_, '_>,
+ intrinsic: Symbol,
+ span: Span,
+ ty: Ty<'_>,
+) {
+ fx.tcx.sess.span_err(span, &format!("invalid monomorphization of `{}` intrinsic: expected SIMD input type, found non-SIMD `{}`", intrinsic, ty));
+ // Prevent verifier error
+ fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
+}
+
+pub(super) fn codegen_simd_intrinsic_call<'tcx>(
+ fx: &mut FunctionCx<'_, '_, 'tcx>,
+ intrinsic: Symbol,
+ _substs: SubstsRef<'tcx>,
+ args: &[mir::Operand<'tcx>],
+ ret: CPlace<'tcx>,
++ target: BasicBlock,
+ span: Span,
+) {
+ match intrinsic {
+ sym::simd_as | sym::simd_cast => {
+ intrinsic_args!(fx, args => (a); intrinsic);
+
+ if !a.layout().ty.is_simd() {
+ report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
+ return;
+ }
+
+ simd_for_each_lane(fx, a, ret, &|fx, lane_ty, ret_lane_ty, lane| {
+ let ret_lane_clif_ty = fx.clif_type(ret_lane_ty).unwrap();
+
+ let from_signed = type_sign(lane_ty);
+ let to_signed = type_sign(ret_lane_ty);
+
+ clif_int_or_float_cast(fx, lane, from_signed, ret_lane_clif_ty, to_signed)
+ });
+ }
+
+ sym::simd_eq | sym::simd_ne | sym::simd_lt | sym::simd_le | sym::simd_gt | sym::simd_ge => {
+ intrinsic_args!(fx, args => (x, y); intrinsic);
+
+ if !x.layout().ty.is_simd() {
+ report_simd_type_validation_error(fx, intrinsic, span, x.layout().ty);
+ return;
+ }
+
+ // FIXME use vector instructions when possible
+ simd_pair_for_each_lane(fx, x, y, ret, &|fx, lane_ty, res_lane_ty, x_lane, y_lane| {
+ let res_lane = match (lane_ty.kind(), intrinsic) {
+ (ty::Uint(_), sym::simd_eq) => fx.bcx.ins().icmp(IntCC::Equal, x_lane, y_lane),
+ (ty::Uint(_), sym::simd_ne) => {
+ fx.bcx.ins().icmp(IntCC::NotEqual, x_lane, y_lane)
+ }
+ (ty::Uint(_), sym::simd_lt) => {
+ fx.bcx.ins().icmp(IntCC::UnsignedLessThan, x_lane, y_lane)
+ }
+ (ty::Uint(_), sym::simd_le) => {
+ fx.bcx.ins().icmp(IntCC::UnsignedLessThanOrEqual, x_lane, y_lane)
+ }
+ (ty::Uint(_), sym::simd_gt) => {
+ fx.bcx.ins().icmp(IntCC::UnsignedGreaterThan, x_lane, y_lane)
+ }
+ (ty::Uint(_), sym::simd_ge) => {
+ fx.bcx.ins().icmp(IntCC::UnsignedGreaterThanOrEqual, x_lane, y_lane)
+ }
+
+ (ty::Int(_), sym::simd_eq) => fx.bcx.ins().icmp(IntCC::Equal, x_lane, y_lane),
+ (ty::Int(_), sym::simd_ne) => {
+ fx.bcx.ins().icmp(IntCC::NotEqual, x_lane, y_lane)
+ }
+ (ty::Int(_), sym::simd_lt) => {
+ fx.bcx.ins().icmp(IntCC::SignedLessThan, x_lane, y_lane)
+ }
+ (ty::Int(_), sym::simd_le) => {
+ fx.bcx.ins().icmp(IntCC::SignedLessThanOrEqual, x_lane, y_lane)
+ }
+ (ty::Int(_), sym::simd_gt) => {
+ fx.bcx.ins().icmp(IntCC::SignedGreaterThan, x_lane, y_lane)
+ }
+ (ty::Int(_), sym::simd_ge) => {
+ fx.bcx.ins().icmp(IntCC::SignedGreaterThanOrEqual, x_lane, y_lane)
+ }
+
+ (ty::Float(_), sym::simd_eq) => {
+ fx.bcx.ins().fcmp(FloatCC::Equal, x_lane, y_lane)
+ }
+ (ty::Float(_), sym::simd_ne) => {
+ fx.bcx.ins().fcmp(FloatCC::NotEqual, x_lane, y_lane)
+ }
+ (ty::Float(_), sym::simd_lt) => {
+ fx.bcx.ins().fcmp(FloatCC::LessThan, x_lane, y_lane)
+ }
+ (ty::Float(_), sym::simd_le) => {
+ fx.bcx.ins().fcmp(FloatCC::LessThanOrEqual, x_lane, y_lane)
+ }
+ (ty::Float(_), sym::simd_gt) => {
+ fx.bcx.ins().fcmp(FloatCC::GreaterThan, x_lane, y_lane)
+ }
+ (ty::Float(_), sym::simd_ge) => {
+ fx.bcx.ins().fcmp(FloatCC::GreaterThanOrEqual, x_lane, y_lane)
+ }
+
+ _ => unreachable!(),
+ };
+
+ bool_to_zero_or_max_uint(fx, res_lane_ty, res_lane)
+ });
+ }
+
+ // simd_shuffle32<T, U>(x: T, y: T, idx: [u32; 32]) -> U
+ _ if intrinsic.as_str().starts_with("simd_shuffle") => {
+ let (x, y, idx) = match args {
+ [x, y, idx] => (x, y, idx),
+ _ => {
+ bug!("wrong number of args for intrinsic {intrinsic}");
+ }
+ };
+ let x = codegen_operand(fx, x);
+ let y = codegen_operand(fx, y);
+
+ if !x.layout().ty.is_simd() {
+ report_simd_type_validation_error(fx, intrinsic, span, x.layout().ty);
+ return;
+ }
+
+ // If this intrinsic is the older "simd_shuffleN" form, simply parse the integer.
+ // If there is no suffix, use the index array length.
+ let n: u16 = if intrinsic == sym::simd_shuffle {
+ // Make sure this is actually an array, since typeck only checks the length-suffixed
+ // version of this intrinsic.
+ let idx_ty = fx.monomorphize(idx.ty(fx.mir, fx.tcx));
+ match idx_ty.kind() {
+ ty::Array(ty, len) if matches!(ty.kind(), ty::Uint(ty::UintTy::U32)) => len
+ .try_eval_usize(fx.tcx, ty::ParamEnv::reveal_all())
+ .unwrap_or_else(|| {
+ span_bug!(span, "could not evaluate shuffle index array length")
+ })
+ .try_into()
+ .unwrap(),
+ _ => {
+ fx.tcx.sess.span_err(
+ span,
+ &format!(
+ "simd_shuffle index must be an array of `u32`, got `{}`",
+ idx_ty,
+ ),
+ );
+ // Prevent verifier error
+ fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
+ return;
+ }
+ }
+ } else {
+ // FIXME remove this case
+ intrinsic.as_str()["simd_shuffle".len()..].parse().unwrap()
+ };
+
+ assert_eq!(x.layout(), y.layout());
+ let layout = x.layout();
+
+ let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx);
+ let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx);
+
+ assert_eq!(lane_ty, ret_lane_ty);
+ assert_eq!(u64::from(n), ret_lane_count);
+
+ let total_len = lane_count * 2;
+
+ let indexes = {
+ use rustc_middle::mir::interpret::*;
+ let idx_const = crate::constant::mir_operand_get_const_val(fx, idx)
+ .expect("simd_shuffle* idx not const");
+
+ let idx_bytes = match idx_const {
+ ConstValue::ByRef { alloc, offset } => {
+ let size = Size::from_bytes(
+ 4 * ret_lane_count, /* size_of([u32; ret_lane_count]) */
+ );
+ alloc
+ .inner()
+ .get_bytes_strip_provenance(fx, alloc_range(offset, size))
+ .unwrap()
+ }
+ _ => unreachable!("{:?}", idx_const),
+ };
+
+ (0..ret_lane_count)
+ .map(|i| {
+ let i = usize::try_from(i).unwrap();
+ let idx = rustc_middle::mir::interpret::read_target_uint(
+ fx.tcx.data_layout.endian,
+ &idx_bytes[4 * i..4 * i + 4],
+ )
+ .expect("read_target_uint");
+ u16::try_from(idx).expect("try_from u32")
+ })
+ .collect::<Vec<u16>>()
+ };
+
+ for &idx in &indexes {
+ assert!(u64::from(idx) < total_len, "idx {} out of range 0..{}", idx, total_len);
+ }
+
+ for (out_idx, in_idx) in indexes.into_iter().enumerate() {
+ let in_lane = if u64::from(in_idx) < lane_count {
+ x.value_lane(fx, in_idx.into())
+ } else {
+ y.value_lane(fx, u64::from(in_idx) - lane_count)
+ };
+ let out_lane = ret.place_lane(fx, u64::try_from(out_idx).unwrap());
+ out_lane.write_cvalue(fx, in_lane);
+ }
+ }
+
+ sym::simd_insert => {
+ let (base, idx, val) = match args {
+ [base, idx, val] => (base, idx, val),
+ _ => {
+ bug!("wrong number of args for intrinsic {intrinsic}");
+ }
+ };
+ let base = codegen_operand(fx, base);
+ let val = codegen_operand(fx, val);
+
+ // FIXME validate
+ let idx_const = if let Some(idx_const) =
+ crate::constant::mir_operand_get_const_val(fx, idx)
+ {
+ idx_const
+ } else {
+ fx.tcx.sess.span_fatal(span, "Index argument for `simd_insert` is not a constant");
+ };
+
+ let idx = idx_const
+ .try_to_bits(Size::from_bytes(4 /* u32*/))
+ .unwrap_or_else(|| panic!("kind not scalar: {:?}", idx_const));
+ let (lane_count, _lane_ty) = base.layout().ty.simd_size_and_type(fx.tcx);
+ if idx >= lane_count.into() {
+ fx.tcx.sess.span_fatal(
+ fx.mir.span,
+ &format!("[simd_insert] idx {} >= lane_count {}", idx, lane_count),
+ );
+ }
+
+ ret.write_cvalue(fx, base);
+ let ret_lane = ret.place_field(fx, mir::Field::new(idx.try_into().unwrap()));
+ ret_lane.write_cvalue(fx, val);
+ }
+
+ sym::simd_extract => {
+ let (v, idx) = match args {
+ [v, idx] => (v, idx),
+ _ => {
+ bug!("wrong number of args for intrinsic {intrinsic}");
+ }
+ };
+ let v = codegen_operand(fx, v);
+
+ if !v.layout().ty.is_simd() {
+ report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
+ return;
+ }
+
+ let idx_const = if let Some(idx_const) =
+ crate::constant::mir_operand_get_const_val(fx, idx)
+ {
+ idx_const
+ } else {
+ fx.tcx.sess.span_warn(span, "Index argument for `simd_extract` is not a constant");
+ let trap_block = fx.bcx.create_block();
- fx.bcx.ins().jump(dummy_block, &[]);
+ let true_ = fx.bcx.ins().iconst(types::I8, 1);
+ fx.bcx.ins().brnz(true_, trap_block, &[]);
- fx.bcx.switch_to_block(dummy_block);
++ let ret_block = fx.get_block(target);
++ fx.bcx.ins().jump(ret_block, &[]);
+ fx.bcx.switch_to_block(trap_block);
+ crate::trap::trap_unimplemented(
+ fx,
+ "Index argument for `simd_extract` is not a constant",
+ );
- // simd_arith_offset
- // simd_scatter
- // simd_gather
+ return;
+ };
+
+ let idx = idx_const
+ .try_to_bits(Size::from_bytes(4 /* u32*/))
+ .unwrap_or_else(|| panic!("kind not scalar: {:?}", idx_const));
+ let (lane_count, _lane_ty) = v.layout().ty.simd_size_and_type(fx.tcx);
+ if idx >= lane_count.into() {
+ fx.tcx.sess.span_fatal(
+ fx.mir.span,
+ &format!("[simd_extract] idx {} >= lane_count {}", idx, lane_count),
+ );
+ }
+
+ let ret_lane = v.value_lane(fx, idx.try_into().unwrap());
+ ret.write_cvalue(fx, ret_lane);
+ }
+
+ sym::simd_neg => {
+ intrinsic_args!(fx, args => (a); intrinsic);
+
+ if !a.layout().ty.is_simd() {
+ report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
+ return;
+ }
+
+ simd_for_each_lane(
+ fx,
+ a,
+ ret,
+ &|fx, lane_ty, _ret_lane_ty, lane| match lane_ty.kind() {
+ ty::Int(_) => fx.bcx.ins().ineg(lane),
+ ty::Float(_) => fx.bcx.ins().fneg(lane),
+ _ => unreachable!(),
+ },
+ );
+ }
+
+ sym::simd_add
+ | sym::simd_sub
+ | sym::simd_mul
+ | sym::simd_div
+ | sym::simd_rem
+ | sym::simd_shl
+ | sym::simd_shr
+ | sym::simd_and
+ | sym::simd_or
+ | sym::simd_xor => {
+ intrinsic_args!(fx, args => (x, y); intrinsic);
+
+ // FIXME use vector instructions when possible
+ simd_pair_for_each_lane(fx, x, y, ret, &|fx, lane_ty, _ret_lane_ty, x_lane, y_lane| {
+ match (lane_ty.kind(), intrinsic) {
+ (ty::Uint(_), sym::simd_add) => fx.bcx.ins().iadd(x_lane, y_lane),
+ (ty::Uint(_), sym::simd_sub) => fx.bcx.ins().isub(x_lane, y_lane),
+ (ty::Uint(_), sym::simd_mul) => fx.bcx.ins().imul(x_lane, y_lane),
+ (ty::Uint(_), sym::simd_div) => fx.bcx.ins().udiv(x_lane, y_lane),
+ (ty::Uint(_), sym::simd_rem) => fx.bcx.ins().urem(x_lane, y_lane),
+
+ (ty::Int(_), sym::simd_add) => fx.bcx.ins().iadd(x_lane, y_lane),
+ (ty::Int(_), sym::simd_sub) => fx.bcx.ins().isub(x_lane, y_lane),
+ (ty::Int(_), sym::simd_mul) => fx.bcx.ins().imul(x_lane, y_lane),
+ (ty::Int(_), sym::simd_div) => fx.bcx.ins().sdiv(x_lane, y_lane),
+ (ty::Int(_), sym::simd_rem) => fx.bcx.ins().srem(x_lane, y_lane),
+
+ (ty::Float(_), sym::simd_add) => fx.bcx.ins().fadd(x_lane, y_lane),
+ (ty::Float(_), sym::simd_sub) => fx.bcx.ins().fsub(x_lane, y_lane),
+ (ty::Float(_), sym::simd_mul) => fx.bcx.ins().fmul(x_lane, y_lane),
+ (ty::Float(_), sym::simd_div) => fx.bcx.ins().fdiv(x_lane, y_lane),
+ (ty::Float(FloatTy::F32), sym::simd_rem) => fx.lib_call(
+ "fmodf",
+ vec![AbiParam::new(types::F32), AbiParam::new(types::F32)],
+ vec![AbiParam::new(types::F32)],
+ &[x_lane, y_lane],
+ )[0],
+ (ty::Float(FloatTy::F64), sym::simd_rem) => fx.lib_call(
+ "fmod",
+ vec![AbiParam::new(types::F64), AbiParam::new(types::F64)],
+ vec![AbiParam::new(types::F64)],
+ &[x_lane, y_lane],
+ )[0],
+
+ (ty::Uint(_), sym::simd_shl) => fx.bcx.ins().ishl(x_lane, y_lane),
+ (ty::Uint(_), sym::simd_shr) => fx.bcx.ins().ushr(x_lane, y_lane),
+ (ty::Uint(_), sym::simd_and) => fx.bcx.ins().band(x_lane, y_lane),
+ (ty::Uint(_), sym::simd_or) => fx.bcx.ins().bor(x_lane, y_lane),
+ (ty::Uint(_), sym::simd_xor) => fx.bcx.ins().bxor(x_lane, y_lane),
+
+ (ty::Int(_), sym::simd_shl) => fx.bcx.ins().ishl(x_lane, y_lane),
+ (ty::Int(_), sym::simd_shr) => fx.bcx.ins().sshr(x_lane, y_lane),
+ (ty::Int(_), sym::simd_and) => fx.bcx.ins().band(x_lane, y_lane),
+ (ty::Int(_), sym::simd_or) => fx.bcx.ins().bor(x_lane, y_lane),
+ (ty::Int(_), sym::simd_xor) => fx.bcx.ins().bxor(x_lane, y_lane),
+
+ _ => unreachable!(),
+ }
+ });
+ }
+
+ sym::simd_fma => {
+ intrinsic_args!(fx, args => (a, b, c); intrinsic);
+
+ if !a.layout().ty.is_simd() {
+ report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
+ return;
+ }
+ assert_eq!(a.layout(), b.layout());
+ assert_eq!(a.layout(), c.layout());
+ assert_eq!(a.layout(), ret.layout());
+
+ let layout = a.layout();
+ let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx);
+ let res_lane_layout = fx.layout_of(lane_ty);
+
+ for lane in 0..lane_count {
+ let a_lane = a.value_lane(fx, lane).load_scalar(fx);
+ let b_lane = b.value_lane(fx, lane).load_scalar(fx);
+ let c_lane = c.value_lane(fx, lane).load_scalar(fx);
+
+ let res_lane = fx.bcx.ins().fma(a_lane, b_lane, c_lane);
+ let res_lane = CValue::by_val(res_lane, res_lane_layout);
+
+ ret.place_lane(fx, lane).write_cvalue(fx, res_lane);
+ }
+ }
+
+ sym::simd_fmin | sym::simd_fmax => {
+ intrinsic_args!(fx, args => (x, y); intrinsic);
+
+ if !x.layout().ty.is_simd() {
+ report_simd_type_validation_error(fx, intrinsic, span, x.layout().ty);
+ return;
+ }
+
+ // FIXME use vector instructions when possible
+ simd_pair_for_each_lane(fx, x, y, ret, &|fx, lane_ty, _ret_lane_ty, x_lane, y_lane| {
+ match lane_ty.kind() {
+ ty::Float(_) => {}
+ _ => unreachable!("{:?}", lane_ty),
+ }
+ match intrinsic {
+ sym::simd_fmin => crate::num::codegen_float_min(fx, x_lane, y_lane),
+ sym::simd_fmax => crate::num::codegen_float_max(fx, x_lane, y_lane),
+ _ => unreachable!(),
+ }
+ });
+ }
+
+ sym::simd_round => {
+ intrinsic_args!(fx, args => (a); intrinsic);
+
+ if !a.layout().ty.is_simd() {
+ report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
+ return;
+ }
+
+ simd_for_each_lane(
+ fx,
+ a,
+ ret,
+ &|fx, lane_ty, _ret_lane_ty, lane| match lane_ty.kind() {
+ ty::Float(FloatTy::F32) => fx.lib_call(
+ "roundf",
+ vec![AbiParam::new(types::F32)],
+ vec![AbiParam::new(types::F32)],
+ &[lane],
+ )[0],
+ ty::Float(FloatTy::F64) => fx.lib_call(
+ "round",
+ vec![AbiParam::new(types::F64)],
+ vec![AbiParam::new(types::F64)],
+ &[lane],
+ )[0],
+ _ => unreachable!("{:?}", lane_ty),
+ },
+ );
+ }
+
+ sym::simd_fabs | sym::simd_fsqrt | sym::simd_ceil | sym::simd_floor | sym::simd_trunc => {
+ intrinsic_args!(fx, args => (a); intrinsic);
+
+ if !a.layout().ty.is_simd() {
+ report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
+ return;
+ }
+
+ simd_for_each_lane(fx, a, ret, &|fx, lane_ty, _ret_lane_ty, lane| {
+ match lane_ty.kind() {
+ ty::Float(_) => {}
+ _ => unreachable!("{:?}", lane_ty),
+ }
+ match intrinsic {
+ sym::simd_fabs => fx.bcx.ins().fabs(lane),
+ sym::simd_fsqrt => fx.bcx.ins().sqrt(lane),
+ sym::simd_ceil => fx.bcx.ins().ceil(lane),
+ sym::simd_floor => fx.bcx.ins().floor(lane),
+ sym::simd_trunc => fx.bcx.ins().trunc(lane),
+ _ => unreachable!(),
+ }
+ });
+ }
+
+ sym::simd_reduce_add_ordered | sym::simd_reduce_add_unordered => {
+ intrinsic_args!(fx, args => (v, acc); intrinsic);
+ let acc = acc.load_scalar(fx);
+
+ // FIXME there must be no acc param for integer vectors
+ if !v.layout().ty.is_simd() {
+ report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
+ return;
+ }
+
+ simd_reduce(fx, v, Some(acc), ret, &|fx, lane_ty, a, b| {
+ if lane_ty.is_floating_point() {
+ fx.bcx.ins().fadd(a, b)
+ } else {
+ fx.bcx.ins().iadd(a, b)
+ }
+ });
+ }
+
+ sym::simd_reduce_mul_ordered | sym::simd_reduce_mul_unordered => {
+ intrinsic_args!(fx, args => (v, acc); intrinsic);
+ let acc = acc.load_scalar(fx);
+
+ // FIXME there must be no acc param for integer vectors
+ if !v.layout().ty.is_simd() {
+ report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
+ return;
+ }
+
+ simd_reduce(fx, v, Some(acc), ret, &|fx, lane_ty, a, b| {
+ if lane_ty.is_floating_point() {
+ fx.bcx.ins().fmul(a, b)
+ } else {
+ fx.bcx.ins().imul(a, b)
+ }
+ });
+ }
+
+ sym::simd_reduce_all => {
+ intrinsic_args!(fx, args => (v); intrinsic);
+
+ if !v.layout().ty.is_simd() {
+ report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
+ return;
+ }
+
+ simd_reduce_bool(fx, v, ret, &|fx, a, b| fx.bcx.ins().band(a, b));
+ }
+
+ sym::simd_reduce_any => {
+ intrinsic_args!(fx, args => (v); intrinsic);
+
+ if !v.layout().ty.is_simd() {
+ report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
+ return;
+ }
+
+ simd_reduce_bool(fx, v, ret, &|fx, a, b| fx.bcx.ins().bor(a, b));
+ }
+
+ sym::simd_reduce_and => {
+ intrinsic_args!(fx, args => (v); intrinsic);
+
+ if !v.layout().ty.is_simd() {
+ report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
+ return;
+ }
+
+ simd_reduce(fx, v, None, ret, &|fx, _ty, a, b| fx.bcx.ins().band(a, b));
+ }
+
+ sym::simd_reduce_or => {
+ intrinsic_args!(fx, args => (v); intrinsic);
+
+ if !v.layout().ty.is_simd() {
+ report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
+ return;
+ }
+
+ simd_reduce(fx, v, None, ret, &|fx, _ty, a, b| fx.bcx.ins().bor(a, b));
+ }
+
+ sym::simd_reduce_xor => {
+ intrinsic_args!(fx, args => (v); intrinsic);
+
+ if !v.layout().ty.is_simd() {
+ report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
+ return;
+ }
+
+ simd_reduce(fx, v, None, ret, &|fx, _ty, a, b| fx.bcx.ins().bxor(a, b));
+ }
+
+ sym::simd_reduce_min => {
+ intrinsic_args!(fx, args => (v); intrinsic);
+
+ if !v.layout().ty.is_simd() {
+ report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
+ return;
+ }
+
+ simd_reduce(fx, v, None, ret, &|fx, ty, a, b| {
+ let lt = match ty.kind() {
+ ty::Int(_) => fx.bcx.ins().icmp(IntCC::SignedLessThan, a, b),
+ ty::Uint(_) => fx.bcx.ins().icmp(IntCC::UnsignedLessThan, a, b),
+ ty::Float(_) => return crate::num::codegen_float_min(fx, a, b),
+ _ => unreachable!(),
+ };
+ fx.bcx.ins().select(lt, a, b)
+ });
+ }
+
+ sym::simd_reduce_max => {
+ intrinsic_args!(fx, args => (v); intrinsic);
+
+ if !v.layout().ty.is_simd() {
+ report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
+ return;
+ }
+
+ simd_reduce(fx, v, None, ret, &|fx, ty, a, b| {
+ let gt = match ty.kind() {
+ ty::Int(_) => fx.bcx.ins().icmp(IntCC::SignedGreaterThan, a, b),
+ ty::Uint(_) => fx.bcx.ins().icmp(IntCC::UnsignedGreaterThan, a, b),
+ ty::Float(_) => return crate::num::codegen_float_max(fx, a, b),
+ _ => unreachable!(),
+ };
+ fx.bcx.ins().select(gt, a, b)
+ });
+ }
+
+ sym::simd_select => {
+ intrinsic_args!(fx, args => (m, a, b); intrinsic);
+
+ if !m.layout().ty.is_simd() {
+ report_simd_type_validation_error(fx, intrinsic, span, m.layout().ty);
+ return;
+ }
+ if !a.layout().ty.is_simd() {
+ report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
+ return;
+ }
+ assert_eq!(a.layout(), b.layout());
+
+ let (lane_count, lane_ty) = a.layout().ty.simd_size_and_type(fx.tcx);
+ let lane_layout = fx.layout_of(lane_ty);
+
+ for lane in 0..lane_count {
+ let m_lane = m.value_lane(fx, lane).load_scalar(fx);
+ let a_lane = a.value_lane(fx, lane).load_scalar(fx);
+ let b_lane = b.value_lane(fx, lane).load_scalar(fx);
+
+ let m_lane = fx.bcx.ins().icmp_imm(IntCC::Equal, m_lane, 0);
+ let res_lane =
+ CValue::by_val(fx.bcx.ins().select(m_lane, b_lane, a_lane), lane_layout);
+
+ ret.place_lane(fx, lane).write_cvalue(fx, res_lane);
+ }
+ }
+
+ sym::simd_select_bitmask => {
+ intrinsic_args!(fx, args => (m, a, b); intrinsic);
+
+ if !a.layout().ty.is_simd() {
+ report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
+ return;
+ }
+ assert_eq!(a.layout(), b.layout());
+
+ let (lane_count, lane_ty) = a.layout().ty.simd_size_and_type(fx.tcx);
+ let lane_layout = fx.layout_of(lane_ty);
+
+ let m = m.load_scalar(fx);
+
+ for lane in 0..lane_count {
+ let m_lane = fx.bcx.ins().ushr_imm(m, u64::from(lane) as i64);
+ let m_lane = fx.bcx.ins().band_imm(m_lane, 1);
+ let a_lane = a.value_lane(fx, lane).load_scalar(fx);
+ let b_lane = b.value_lane(fx, lane).load_scalar(fx);
+
+ let m_lane = fx.bcx.ins().icmp_imm(IntCC::Equal, m_lane, 0);
+ let res_lane =
+ CValue::by_val(fx.bcx.ins().select(m_lane, b_lane, a_lane), lane_layout);
+
+ ret.place_lane(fx, lane).write_cvalue(fx, res_lane);
+ }
+ }
+
+ sym::simd_bitmask => {
+ intrinsic_args!(fx, args => (a); intrinsic);
+
+ let (lane_count, lane_ty) = a.layout().ty.simd_size_and_type(fx.tcx);
+ let lane_clif_ty = fx.clif_type(lane_ty).unwrap();
+
+ // The `fn simd_bitmask(vector) -> unsigned integer` intrinsic takes a
+ // vector mask and returns the most significant bit (MSB) of each lane in the form
+ // of either:
+ // * an unsigned integer
+ // * an array of `u8`
+ // If the vector has less than 8 lanes, a u8 is returned with zeroed trailing bits.
+ //
+ // The bit order of the result depends on the byte endianness, LSB-first for little
+ // endian and MSB-first for big endian.
+ let expected_int_bits = lane_count.max(8);
+ let expected_bytes = expected_int_bits / 8 + ((expected_int_bits % 8 > 0) as u64);
+
+ match lane_ty.kind() {
+ ty::Int(_) | ty::Uint(_) => {}
+ _ => {
+ fx.tcx.sess.span_fatal(
+ span,
+ &format!(
+ "invalid monomorphization of `simd_bitmask` intrinsic: \
+ vector argument `{}`'s element type `{}`, expected integer element \
+ type",
+ a.layout().ty,
+ lane_ty
+ ),
+ );
+ }
+ }
+
+ let res_type =
+ Type::int_with_byte_size(u16::try_from(expected_bytes).unwrap()).unwrap();
+ let mut res = type_zero_value(&mut fx.bcx, res_type);
+
+ let lanes = match fx.tcx.sess.target.endian {
+ Endian::Big => Box::new(0..lane_count) as Box<dyn Iterator<Item = u64>>,
+ Endian::Little => Box::new((0..lane_count).rev()) as Box<dyn Iterator<Item = u64>>,
+ };
+ for lane in lanes {
+ let a_lane = a.value_lane(fx, lane).load_scalar(fx);
+
+ // extract sign bit of an int
+ let a_lane_sign = fx.bcx.ins().ushr_imm(a_lane, i64::from(lane_clif_ty.bits() - 1));
+
+ // shift sign bit into result
+ let a_lane_sign = clif_intcast(fx, a_lane_sign, res_type, false);
+ res = fx.bcx.ins().ishl_imm(res, 1);
+ res = fx.bcx.ins().bor(res, a_lane_sign);
+ }
+
+ match ret.layout().ty.kind() {
+ ty::Uint(i) if i.bit_width() == Some(expected_int_bits) => {}
+ ty::Array(elem, len)
+ if matches!(elem.kind(), ty::Uint(ty::UintTy::U8))
+ && len.try_eval_usize(fx.tcx, ty::ParamEnv::reveal_all())
+ == Some(expected_bytes) => {}
+ _ => {
+ fx.tcx.sess.span_fatal(
+ span,
+ &format!(
+ "invalid monomorphization of `simd_bitmask` intrinsic: \
+ cannot return `{}`, expected `u{}` or `[u8; {}]`",
+ ret.layout().ty,
+ expected_int_bits,
+ expected_bytes
+ ),
+ );
+ }
+ }
+
+ let res = CValue::by_val(res, ret.layout());
+ ret.write_cvalue(fx, res);
+ }
+
+ sym::simd_saturating_add | sym::simd_saturating_sub => {
+ intrinsic_args!(fx, args => (x, y); intrinsic);
+
+ let bin_op = match intrinsic {
+ sym::simd_saturating_add => BinOp::Add,
+ sym::simd_saturating_sub => BinOp::Sub,
+ _ => unreachable!(),
+ };
+
+ // FIXME use vector instructions when possible
+ simd_pair_for_each_lane_typed(fx, x, y, ret, &|fx, x_lane, y_lane| {
+ crate::num::codegen_saturating_int_binop(fx, bin_op, x_lane, y_lane)
+ });
+ }
+
- fx.tcx.sess.span_fatal(span, &format!("Unknown SIMD intrinsic {}", intrinsic));
++ sym::simd_expose_addr | sym::simd_from_exposed_addr | sym::simd_cast_ptr => {
++ intrinsic_args!(fx, args => (arg); intrinsic);
++ ret.write_cvalue_transmute(fx, arg);
++ }
++
++ sym::simd_arith_offset => {
++ intrinsic_args!(fx, args => (ptr, offset); intrinsic);
++
++ let (lane_count, ptr_lane_ty) = ptr.layout().ty.simd_size_and_type(fx.tcx);
++ let pointee_ty = ptr_lane_ty.builtin_deref(true).unwrap().ty;
++ let pointee_size = fx.layout_of(pointee_ty).size.bytes();
++ let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx);
++ let ret_lane_layout = fx.layout_of(ret_lane_ty);
++ assert_eq!(lane_count, ret_lane_count);
++
++ for lane_idx in 0..lane_count {
++ let ptr_lane = ptr.value_lane(fx, lane_idx).load_scalar(fx);
++ let offset_lane = offset.value_lane(fx, lane_idx).load_scalar(fx);
++
++ let ptr_diff = if pointee_size != 1 {
++ fx.bcx.ins().imul_imm(offset_lane, pointee_size as i64)
++ } else {
++ offset_lane
++ };
++ let res_lane = fx.bcx.ins().iadd(ptr_lane, ptr_diff);
++ let res_lane = CValue::by_val(res_lane, ret_lane_layout);
++
++ ret.place_lane(fx, lane_idx).write_cvalue(fx, res_lane);
++ }
++ }
++
++ sym::simd_gather => {
++ intrinsic_args!(fx, args => (val, ptr, mask); intrinsic);
++
++ let (val_lane_count, val_lane_ty) = val.layout().ty.simd_size_and_type(fx.tcx);
++ let (ptr_lane_count, _ptr_lane_ty) = ptr.layout().ty.simd_size_and_type(fx.tcx);
++ let (mask_lane_count, _mask_lane_ty) = mask.layout().ty.simd_size_and_type(fx.tcx);
++ let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx);
++ assert_eq!(val_lane_count, ptr_lane_count);
++ assert_eq!(val_lane_count, mask_lane_count);
++ assert_eq!(val_lane_count, ret_lane_count);
++
++ let lane_clif_ty = fx.clif_type(val_lane_ty).unwrap();
++ let ret_lane_layout = fx.layout_of(ret_lane_ty);
++
++ for lane_idx in 0..ptr_lane_count {
++ let val_lane = val.value_lane(fx, lane_idx).load_scalar(fx);
++ let ptr_lane = ptr.value_lane(fx, lane_idx).load_scalar(fx);
++ let mask_lane = mask.value_lane(fx, lane_idx).load_scalar(fx);
++
++ let if_enabled = fx.bcx.create_block();
++ let if_disabled = fx.bcx.create_block();
++ let next = fx.bcx.create_block();
++ let res_lane = fx.bcx.append_block_param(next, lane_clif_ty);
++
++ fx.bcx.ins().brnz(mask_lane, if_enabled, &[]);
++ fx.bcx.ins().jump(if_disabled, &[]);
++ fx.bcx.seal_block(if_enabled);
++ fx.bcx.seal_block(if_disabled);
++
++ fx.bcx.switch_to_block(if_enabled);
++ let res = fx.bcx.ins().load(lane_clif_ty, MemFlags::trusted(), ptr_lane, 0);
++ fx.bcx.ins().jump(next, &[res]);
++
++ fx.bcx.switch_to_block(if_disabled);
++ fx.bcx.ins().jump(next, &[val_lane]);
++
++ fx.bcx.seal_block(next);
++ fx.bcx.switch_to_block(next);
++
++ fx.bcx.ins().nop();
++
++ ret.place_lane(fx, lane_idx)
++ .write_cvalue(fx, CValue::by_val(res_lane, ret_lane_layout));
++ }
++ }
++
++ sym::simd_scatter => {
++ intrinsic_args!(fx, args => (val, ptr, mask); intrinsic);
++
++ let (val_lane_count, _val_lane_ty) = val.layout().ty.simd_size_and_type(fx.tcx);
++ let (ptr_lane_count, _ptr_lane_ty) = ptr.layout().ty.simd_size_and_type(fx.tcx);
++ let (mask_lane_count, _mask_lane_ty) = mask.layout().ty.simd_size_and_type(fx.tcx);
++ assert_eq!(val_lane_count, ptr_lane_count);
++ assert_eq!(val_lane_count, mask_lane_count);
++
++ for lane_idx in 0..ptr_lane_count {
++ let val_lane = val.value_lane(fx, lane_idx).load_scalar(fx);
++ let ptr_lane = ptr.value_lane(fx, lane_idx).load_scalar(fx);
++ let mask_lane = mask.value_lane(fx, lane_idx).load_scalar(fx);
++
++ let if_enabled = fx.bcx.create_block();
++ let next = fx.bcx.create_block();
++
++ fx.bcx.ins().brnz(mask_lane, if_enabled, &[]);
++ fx.bcx.ins().jump(next, &[]);
++ fx.bcx.seal_block(if_enabled);
++
++ fx.bcx.switch_to_block(if_enabled);
++ fx.bcx.ins().store(MemFlags::trusted(), val_lane, ptr_lane, 0);
++ fx.bcx.ins().jump(next, &[]);
++
++ fx.bcx.seal_block(next);
++ fx.bcx.switch_to_block(next);
++ }
++ }
++
+ _ => {
++ fx.tcx.sess.span_err(span, &format!("Unknown SIMD intrinsic {}", intrinsic));
++ // Prevent verifier error
++ fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
+ }
+ }
++ let ret_block = fx.get_block(target);
++ fx.bcx.ins().jump(ret_block, &[]);
+}
--- /dev/null
- if target_triple.architecture == target_lexicon::Architecture::X86_64 {
+#![feature(rustc_private)]
+// Note: please avoid adding other feature gates where possible
+#![warn(rust_2018_idioms)]
+#![warn(unused_lifetimes)]
+#![warn(unreachable_pub)]
+
+extern crate jobserver;
+#[macro_use]
+extern crate rustc_middle;
+extern crate rustc_ast;
+extern crate rustc_codegen_ssa;
+extern crate rustc_data_structures;
+extern crate rustc_errors;
+extern crate rustc_fs_util;
+extern crate rustc_hir;
+extern crate rustc_incremental;
+extern crate rustc_index;
+extern crate rustc_interface;
+extern crate rustc_metadata;
+extern crate rustc_session;
+extern crate rustc_span;
+extern crate rustc_target;
+
+// This prevents duplicating functions and statics that are already part of the host rustc process.
+#[allow(unused_extern_crates)]
+extern crate rustc_driver;
+
+use std::any::Any;
+use std::cell::{Cell, RefCell};
+use std::sync::Arc;
+
+use rustc_codegen_ssa::traits::CodegenBackend;
+use rustc_codegen_ssa::CodegenResults;
+use rustc_data_structures::profiling::SelfProfilerRef;
+use rustc_errors::ErrorGuaranteed;
+use rustc_metadata::EncodedMetadata;
+use rustc_middle::dep_graph::{WorkProduct, WorkProductId};
+use rustc_session::config::OutputFilenames;
+use rustc_session::Session;
+use rustc_span::Symbol;
+
+use cranelift_codegen::isa::TargetIsa;
+use cranelift_codegen::settings::{self, Configurable};
+
+pub use crate::config::*;
+use crate::prelude::*;
+
+mod abi;
+mod allocator;
+mod analyze;
+mod archive;
+mod base;
+mod cast;
+mod codegen_i128;
+mod common;
+mod compiler_builtins;
+mod concurrency_limiter;
+mod config;
+mod constant;
+mod debuginfo;
+mod discriminant;
+mod driver;
+mod global_asm;
+mod inline_asm;
+mod intrinsics;
+mod linkage;
+mod main_shim;
+mod num;
+mod optimize;
+mod pointer;
+mod pretty_clif;
+mod toolchain;
+mod trap;
+mod unsize;
+mod value_and_place;
+mod vtable;
+
+mod prelude {
+ pub(crate) use rustc_span::{FileNameDisplayPreference, Span};
+
+ pub(crate) use rustc_hir::def_id::{DefId, LOCAL_CRATE};
+ pub(crate) use rustc_middle::bug;
+ pub(crate) use rustc_middle::mir::{self, *};
+ pub(crate) use rustc_middle::ty::layout::{self, LayoutOf, TyAndLayout};
+ pub(crate) use rustc_middle::ty::{
+ self, FloatTy, Instance, InstanceDef, IntTy, ParamEnv, Ty, TyCtxt, TypeAndMut,
+ TypeFoldable, TypeVisitable, UintTy,
+ };
+ pub(crate) use rustc_target::abi::{Abi, Scalar, Size, VariantIdx};
+
+ pub(crate) use rustc_data_structures::fx::FxHashMap;
+
+ pub(crate) use rustc_index::vec::Idx;
+
+ pub(crate) use cranelift_codegen::ir::condcodes::{FloatCC, IntCC};
+ pub(crate) use cranelift_codegen::ir::function::Function;
+ pub(crate) use cranelift_codegen::ir::types;
+ pub(crate) use cranelift_codegen::ir::{
+ AbiParam, Block, FuncRef, Inst, InstBuilder, MemFlags, Signature, SourceLoc, StackSlot,
+ StackSlotData, StackSlotKind, TrapCode, Type, Value,
+ };
+ pub(crate) use cranelift_codegen::isa::{self, CallConv};
+ pub(crate) use cranelift_codegen::Context;
+ pub(crate) use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext, Variable};
+ pub(crate) use cranelift_module::{self, DataContext, FuncId, Linkage, Module};
+
+ pub(crate) use crate::abi::*;
+ pub(crate) use crate::base::{codegen_operand, codegen_place};
+ pub(crate) use crate::cast::*;
+ pub(crate) use crate::common::*;
+ pub(crate) use crate::debuginfo::{DebugContext, UnwindContext};
+ pub(crate) use crate::pointer::Pointer;
+ pub(crate) use crate::value_and_place::{CPlace, CPlaceInner, CValue};
+}
+
+struct PrintOnPanic<F: Fn() -> String>(F);
+impl<F: Fn() -> String> Drop for PrintOnPanic<F> {
+ fn drop(&mut self) {
+ if ::std::thread::panicking() {
+ println!("{}", (self.0)());
+ }
+ }
+}
+
+/// The codegen context holds any information shared between the codegen of individual functions
+/// inside a single codegen unit with the exception of the Cranelift [`Module`](cranelift_module::Module).
+struct CodegenCx {
+ profiler: SelfProfilerRef,
+ output_filenames: Arc<OutputFilenames>,
+ should_write_ir: bool,
+ global_asm: String,
+ inline_asm_index: Cell<usize>,
+ debug_context: Option<DebugContext>,
+ unwind_context: UnwindContext,
+ cgu_name: Symbol,
+}
+
+impl CodegenCx {
+ fn new(
+ tcx: TyCtxt<'_>,
+ backend_config: BackendConfig,
+ isa: &dyn TargetIsa,
+ debug_info: bool,
+ cgu_name: Symbol,
+ ) -> Self {
+ assert_eq!(pointer_ty(tcx), isa.pointer_type());
+
+ let unwind_context =
+ UnwindContext::new(isa, matches!(backend_config.codegen_mode, CodegenMode::Aot));
+ let debug_context = if debug_info && !tcx.sess.target.options.is_like_windows {
+ Some(DebugContext::new(tcx, isa))
+ } else {
+ None
+ };
+ CodegenCx {
+ profiler: tcx.prof.clone(),
+ output_filenames: tcx.output_filenames(()).clone(),
+ should_write_ir: crate::pretty_clif::should_write_ir(tcx),
+ global_asm: String::new(),
+ inline_asm_index: Cell::new(0),
+ debug_context,
+ unwind_context,
+ cgu_name,
+ }
+ }
+}
+
+pub struct CraneliftCodegenBackend {
+ pub config: RefCell<Option<BackendConfig>>,
+}
+
+impl CodegenBackend for CraneliftCodegenBackend {
+ fn init(&self, sess: &Session) {
+ use rustc_session::config::Lto;
+ match sess.lto() {
+ Lto::No | Lto::ThinLocal => {}
+ Lto::Thin | Lto::Fat => sess.warn("LTO is not supported. You may get a linker error."),
+ }
+
+ let mut config = self.config.borrow_mut();
+ if config.is_none() {
+ let new_config = BackendConfig::from_opts(&sess.opts.cg.llvm_args)
+ .unwrap_or_else(|err| sess.fatal(&err));
+ *config = Some(new_config);
+ }
+ }
+
+ fn target_features(&self, _sess: &Session, _allow_unstable: bool) -> Vec<rustc_span::Symbol> {
+ vec![]
+ }
+
+ fn print_version(&self) {
+ println!("Cranelift version: {}", cranelift_codegen::VERSION);
+ }
+
+ fn codegen_crate(
+ &self,
+ tcx: TyCtxt<'_>,
+ metadata: EncodedMetadata,
+ need_metadata_module: bool,
+ ) -> Box<dyn Any> {
+ tcx.sess.abort_if_errors();
+ let config = self.config.borrow().clone().unwrap();
+ match config.codegen_mode {
+ CodegenMode::Aot => driver::aot::run_aot(tcx, config, metadata, need_metadata_module),
+ CodegenMode::Jit | CodegenMode::JitLazy => {
+ #[cfg(feature = "jit")]
+ driver::jit::run_jit(tcx, config);
+
+ #[cfg(not(feature = "jit"))]
+ tcx.sess.fatal("jit support was disabled when compiling rustc_codegen_cranelift");
+ }
+ }
+ }
+
+ fn join_codegen(
+ &self,
+ ongoing_codegen: Box<dyn Any>,
+ sess: &Session,
+ _outputs: &OutputFilenames,
+ ) -> Result<(CodegenResults, FxHashMap<WorkProductId, WorkProduct>), ErrorGuaranteed> {
+ Ok(ongoing_codegen
+ .downcast::<driver::aot::OngoingCodegen>()
+ .unwrap()
+ .join(sess, self.config.borrow().as_ref().unwrap()))
+ }
+
+ fn link(
+ &self,
+ sess: &Session,
+ codegen_results: CodegenResults,
+ outputs: &OutputFilenames,
+ ) -> Result<(), ErrorGuaranteed> {
+ use rustc_codegen_ssa::back::link::link_binary;
+
+ link_binary(sess, &crate::archive::ArArchiveBuilderBuilder, &codegen_results, outputs)
+ }
+}
+
+fn target_triple(sess: &Session) -> target_lexicon::Triple {
+ match sess.target.llvm_target.parse() {
+ Ok(triple) => triple,
+ Err(err) => sess.fatal(&format!("target not recognized: {}", err)),
+ }
+}
+
+fn build_isa(sess: &Session, backend_config: &BackendConfig) -> Box<dyn isa::TargetIsa + 'static> {
+ use target_lexicon::BinaryFormat;
+
+ let target_triple = crate::target_triple(sess);
+
+ let mut flags_builder = settings::builder();
+ flags_builder.enable("is_pic").unwrap();
+ let enable_verifier = if backend_config.enable_verifier { "true" } else { "false" };
+ flags_builder.set("enable_verifier", enable_verifier).unwrap();
+ flags_builder.set("regalloc_checker", enable_verifier).unwrap();
+
+ let tls_model = match target_triple.binary_format {
+ BinaryFormat::Elf => "elf_gd",
+ BinaryFormat::Macho => "macho",
+ BinaryFormat::Coff => "coff",
+ _ => "none",
+ };
+ flags_builder.set("tls_model", tls_model).unwrap();
+
+ flags_builder.set("enable_simd", "true").unwrap();
+
+ flags_builder.set("enable_llvm_abi_extensions", "true").unwrap();
+
+ use rustc_session::config::OptLevel;
+ match sess.opts.optimize {
+ OptLevel::No => {
+ flags_builder.set("opt_level", "none").unwrap();
+ }
+ OptLevel::Less | OptLevel::Default => {}
+ OptLevel::Size | OptLevel::SizeMin | OptLevel::Aggressive => {
+ flags_builder.set("opt_level", "speed_and_size").unwrap();
+ }
+ }
+
- // __cranelift_probestack is not provided and inline stack probes are only supported on x86_64
++ if let target_lexicon::Architecture::Aarch64(_) | target_lexicon::Architecture::X86_64 =
++ target_triple.architecture
++ {
+ // Windows depends on stack probes to grow the committed part of the stack
+ flags_builder.enable("enable_probestack").unwrap();
+ flags_builder.set("probestack_strategy", "inline").unwrap();
+ } else {
++ // __cranelift_probestack is not provided and inline stack probes are only supported on AArch64 and x86_64
+ flags_builder.set("enable_probestack", "false").unwrap();
+ }
+
+ let flags = settings::Flags::new(flags_builder);
+
+ let isa_builder = match sess.opts.cg.target_cpu.as_deref() {
+ Some("native") => {
+ let builder = cranelift_native::builder_with_options(true).unwrap();
+ builder
+ }
+ Some(value) => {
+ let mut builder =
+ cranelift_codegen::isa::lookup(target_triple.clone()).unwrap_or_else(|err| {
+ sess.fatal(&format!("can't compile for {}: {}", target_triple, err));
+ });
+ if let Err(_) = builder.enable(value) {
+ sess.fatal("the specified target cpu isn't currently supported by Cranelift.");
+ }
+ builder
+ }
+ None => {
+ let mut builder =
+ cranelift_codegen::isa::lookup(target_triple.clone()).unwrap_or_else(|err| {
+ sess.fatal(&format!("can't compile for {}: {}", target_triple, err));
+ });
+ if target_triple.architecture == target_lexicon::Architecture::X86_64 {
+ // Don't use "haswell" as the default, as it implies `has_lzcnt`.
+ // macOS CI is still at Ivy Bridge EP, so `lzcnt` is interpreted as `bsr`.
+ builder.enable("nehalem").unwrap();
+ }
+ builder
+ }
+ };
+
+ match isa_builder.finish(flags) {
+ Ok(target_isa) => target_isa,
+ Err(err) => sess.fatal(&format!("failed to build TargetIsa: {}", err)),
+ }
+}
+
+/// This is the entrypoint for a hot plugged rustc_codegen_cranelift
+#[no_mangle]
+pub fn __rustc_codegen_backend() -> Box<dyn CodegenBackend> {
+ Box::new(CraneliftCodegenBackend { config: RefCell::new(None) })
+}
--- /dev/null
- let cmain_func_id = m.declare_function(entry_name, Linkage::Export, &cmain_sig).unwrap();
+use rustc_hir::LangItem;
+use rustc_middle::ty::subst::GenericArg;
+use rustc_middle::ty::AssocKind;
+use rustc_session::config::{sigpipe, EntryFnType};
+use rustc_span::symbol::Ident;
+
+use crate::prelude::*;
+
+/// Create the `main` function which will initialize the rust runtime and call
+/// users main function.
+pub(crate) fn maybe_create_entry_wrapper(
+ tcx: TyCtxt<'_>,
+ module: &mut impl Module,
+ unwind_context: &mut UnwindContext,
+ is_jit: bool,
+ is_primary_cgu: bool,
+) {
+ let (main_def_id, (is_main_fn, sigpipe)) = match tcx.entry_fn(()) {
+ Some((def_id, entry_ty)) => (
+ def_id,
+ match entry_ty {
+ EntryFnType::Main { sigpipe } => (true, sigpipe),
+ EntryFnType::Start => (false, sigpipe::DEFAULT),
+ },
+ ),
+ None => return,
+ };
+
+ if main_def_id.is_local() {
+ let instance = Instance::mono(tcx, main_def_id).polymorphize(tcx);
+ if !is_jit && module.get_name(&*tcx.symbol_name(instance).name).is_none() {
+ return;
+ }
+ } else if !is_primary_cgu {
+ return;
+ }
+
+ create_entry_fn(tcx, module, unwind_context, main_def_id, is_jit, is_main_fn, sigpipe);
+
+ fn create_entry_fn(
+ tcx: TyCtxt<'_>,
+ m: &mut impl Module,
+ unwind_context: &mut UnwindContext,
+ rust_main_def_id: DefId,
+ ignore_lang_start_wrapper: bool,
+ is_main_fn: bool,
+ sigpipe: u8,
+ ) {
+ let main_ret_ty = tcx.fn_sig(rust_main_def_id).output();
+ // Given that `main()` has no arguments,
+ // then its return type cannot have
+ // late-bound regions, since late-bound
+ // regions must appear in the argument
+ // listing.
+ let main_ret_ty = tcx.normalize_erasing_regions(
+ ty::ParamEnv::reveal_all(),
+ main_ret_ty.no_bound_vars().unwrap(),
+ );
+
+ let cmain_sig = Signature {
+ params: vec![
+ AbiParam::new(m.target_config().pointer_type()),
+ AbiParam::new(m.target_config().pointer_type()),
+ ],
+ returns: vec![AbiParam::new(m.target_config().pointer_type() /*isize*/)],
+ call_conv: crate::conv_to_call_conv(
++ tcx.sess,
+ tcx.sess.target.options.entry_abi,
+ m.target_config().default_call_conv,
+ ),
+ };
+
+ let entry_name = tcx.sess.target.options.entry_name.as_ref();
- m.define_function(cmain_func_id, &mut ctx).unwrap();
++ let cmain_func_id = match m.declare_function(entry_name, Linkage::Export, &cmain_sig) {
++ Ok(func_id) => func_id,
++ Err(err) => {
++ tcx.sess
++ .fatal(&format!("entry symbol `{entry_name}` declared multiple times: {err}"));
++ }
++ };
+
+ let instance = Instance::mono(tcx, rust_main_def_id).polymorphize(tcx);
+
+ let main_name = tcx.symbol_name(instance).name;
+ let main_sig = get_function_sig(tcx, m.target_config().default_call_conv, instance);
+ let main_func_id = m.declare_function(main_name, Linkage::Import, &main_sig).unwrap();
+
+ let mut ctx = Context::new();
+ ctx.func.signature = cmain_sig;
+ {
+ let mut func_ctx = FunctionBuilderContext::new();
+ let mut bcx = FunctionBuilder::new(&mut ctx.func, &mut func_ctx);
+
+ let block = bcx.create_block();
+ bcx.switch_to_block(block);
+ let arg_argc = bcx.append_block_param(block, m.target_config().pointer_type());
+ let arg_argv = bcx.append_block_param(block, m.target_config().pointer_type());
+ let arg_sigpipe = bcx.ins().iconst(types::I8, sigpipe as i64);
+
+ let main_func_ref = m.declare_func_in_func(main_func_id, &mut bcx.func);
+
+ let result = if is_main_fn && ignore_lang_start_wrapper {
+ // regular main fn, but ignoring #[lang = "start"] as we are running in the jit
+ // FIXME set program arguments somehow
+ let call_inst = bcx.ins().call(main_func_ref, &[]);
+ let call_results = bcx.func.dfg.inst_results(call_inst).to_owned();
+
+ let termination_trait = tcx.require_lang_item(LangItem::Termination, None);
+ let report = tcx
+ .associated_items(termination_trait)
+ .find_by_name_and_kind(
+ tcx,
+ Ident::from_str("report"),
+ AssocKind::Fn,
+ termination_trait,
+ )
+ .unwrap();
+ let report = Instance::resolve(
+ tcx,
+ ParamEnv::reveal_all(),
+ report.def_id,
+ tcx.mk_substs([GenericArg::from(main_ret_ty)].iter()),
+ )
+ .unwrap()
+ .unwrap()
+ .polymorphize(tcx);
+
+ let report_name = tcx.symbol_name(report).name;
+ let report_sig = get_function_sig(tcx, m.target_config().default_call_conv, report);
+ let report_func_id =
+ m.declare_function(report_name, Linkage::Import, &report_sig).unwrap();
+ let report_func_ref = m.declare_func_in_func(report_func_id, &mut bcx.func);
+
+ // FIXME do proper abi handling instead of expecting the pass mode to be identical
+ // for returns and arguments.
+ let report_call_inst = bcx.ins().call(report_func_ref, &call_results);
+ let res = bcx.func.dfg.inst_results(report_call_inst)[0];
+ match m.target_config().pointer_type() {
+ types::I32 => res,
+ types::I64 => bcx.ins().sextend(types::I64, res),
+ _ => unimplemented!("16bit systems are not yet supported"),
+ }
+ } else if is_main_fn {
+ let start_def_id = tcx.require_lang_item(LangItem::Start, None);
+ let start_instance = Instance::resolve(
+ tcx,
+ ParamEnv::reveal_all(),
+ start_def_id,
+ tcx.intern_substs(&[main_ret_ty.into()]),
+ )
+ .unwrap()
+ .unwrap()
+ .polymorphize(tcx);
+ let start_func_id = import_function(tcx, m, start_instance);
+
+ let main_val = bcx.ins().func_addr(m.target_config().pointer_type(), main_func_ref);
+
+ let func_ref = m.declare_func_in_func(start_func_id, &mut bcx.func);
+ let call_inst =
+ bcx.ins().call(func_ref, &[main_val, arg_argc, arg_argv, arg_sigpipe]);
+ bcx.inst_results(call_inst)[0]
+ } else {
+ // using user-defined start fn
+ let call_inst = bcx.ins().call(main_func_ref, &[arg_argc, arg_argv]);
+ bcx.inst_results(call_inst)[0]
+ };
+
+ bcx.ins().return_(&[result]);
+ bcx.seal_all_blocks();
+ bcx.finalize();
+ }
++
++ if let Err(err) = m.define_function(cmain_func_id, &mut ctx) {
++ tcx.sess.fatal(&format!("entry symbol `{entry_name}` defined multiple times: {err}"));
++ }
++
+ unwind_context.add_function(cmain_func_id, &ctx, m.isa());
+ }
+}
--- /dev/null
- match bcx.func.dfg[arg_inst] {
+//! Peephole optimizations that can be performed while creating clif ir.
+
+use cranelift_codegen::ir::{condcodes::IntCC, InstructionData, Opcode, Value, ValueDef};
+use cranelift_frontend::FunctionBuilder;
+
+/// If the given value was produced by the lowering of `Rvalue::Not` return the input and true,
+/// otherwise return the given value and false.
+pub(crate) fn maybe_unwrap_bool_not(bcx: &mut FunctionBuilder<'_>, arg: Value) -> (Value, bool) {
+ if let ValueDef::Result(arg_inst, 0) = bcx.func.dfg.value_def(arg) {
- match bcx.func.dfg[arg_inst] {
++ match bcx.func.dfg.insts[arg_inst] {
+ // This is the lowering of `Rvalue::Not`
+ InstructionData::IntCompareImm {
+ opcode: Opcode::IcmpImm,
+ cond: IntCC::Equal,
+ arg,
+ imm,
+ } if imm.bits() == 0 => (arg, true),
+ _ => (arg, false),
+ }
+ } else {
+ (arg, false)
+ }
+}
+
+/// Returns whether the branch is statically known to be taken or `None` if it isn't statically known.
+pub(crate) fn maybe_known_branch_taken(
+ bcx: &FunctionBuilder<'_>,
+ arg: Value,
+ test_zero: bool,
+) -> Option<bool> {
+ let arg_inst = if let ValueDef::Result(arg_inst, 0) = bcx.func.dfg.value_def(arg) {
+ arg_inst
+ } else {
+ return None;
+ };
+
++ match bcx.func.dfg.insts[arg_inst] {
+ InstructionData::UnaryImm { opcode: Opcode::Iconst, imm } => {
+ if test_zero {
+ Some(imm.bits() == 0)
+ } else {
+ Some(imm.bits() != 0)
+ }
+ }
+ _ => None,
+ }
+}
--- /dev/null
- | (types::F64, types::I64) => fx.bcx.ins().bitcast(dst_ty, data),
- _ if src_ty.is_vector() && dst_ty.is_vector() => fx.bcx.ins().bitcast(dst_ty, data),
+//! Definition of [`CValue`] and [`CPlace`]
+
+use crate::prelude::*;
+
+use cranelift_codegen::ir::immediates::Offset32;
+
+fn codegen_field<'tcx>(
+ fx: &mut FunctionCx<'_, '_, 'tcx>,
+ base: Pointer,
+ extra: Option<Value>,
+ layout: TyAndLayout<'tcx>,
+ field: mir::Field,
+) -> (Pointer, TyAndLayout<'tcx>) {
+ let field_offset = layout.fields.offset(field.index());
+ let field_layout = layout.field(&*fx, field.index());
+
+ let simple = |fx: &mut FunctionCx<'_, '_, '_>| {
+ (base.offset_i64(fx, i64::try_from(field_offset.bytes()).unwrap()), field_layout)
+ };
+
+ if let Some(extra) = extra {
+ if field_layout.is_sized() {
+ return simple(fx);
+ }
+ match field_layout.ty.kind() {
+ ty::Slice(..) | ty::Str | ty::Foreign(..) => simple(fx),
+ ty::Adt(def, _) if def.repr().packed() => {
+ assert_eq!(layout.align.abi.bytes(), 1);
+ simple(fx)
+ }
+ _ => {
+ // We have to align the offset for DST's
+ let unaligned_offset = field_offset.bytes();
+ let (_, unsized_align) =
+ crate::unsize::size_and_align_of_dst(fx, field_layout, extra);
+
+ let one = fx.bcx.ins().iconst(fx.pointer_type, 1);
+ let align_sub_1 = fx.bcx.ins().isub(unsized_align, one);
+ let and_lhs = fx.bcx.ins().iadd_imm(align_sub_1, unaligned_offset as i64);
+ let zero = fx.bcx.ins().iconst(fx.pointer_type, 0);
+ let and_rhs = fx.bcx.ins().isub(zero, unsized_align);
+ let offset = fx.bcx.ins().band(and_lhs, and_rhs);
+
+ (base.offset_value(fx, offset), field_layout)
+ }
+ }
+ } else {
+ simple(fx)
+ }
+}
+
+fn scalar_pair_calculate_b_offset(tcx: TyCtxt<'_>, a_scalar: Scalar, b_scalar: Scalar) -> Offset32 {
+ let b_offset = a_scalar.size(&tcx).align_to(b_scalar.align(&tcx).abi);
+ Offset32::new(b_offset.bytes().try_into().unwrap())
+}
+
+/// A read-only value
+#[derive(Debug, Copy, Clone)]
+pub(crate) struct CValue<'tcx>(CValueInner, TyAndLayout<'tcx>);
+
+#[derive(Debug, Copy, Clone)]
+enum CValueInner {
+ ByRef(Pointer, Option<Value>),
+ ByVal(Value),
+ ByValPair(Value, Value),
+}
+
+impl<'tcx> CValue<'tcx> {
+ pub(crate) fn by_ref(ptr: Pointer, layout: TyAndLayout<'tcx>) -> CValue<'tcx> {
+ CValue(CValueInner::ByRef(ptr, None), layout)
+ }
+
+ pub(crate) fn by_ref_unsized(
+ ptr: Pointer,
+ meta: Value,
+ layout: TyAndLayout<'tcx>,
+ ) -> CValue<'tcx> {
+ CValue(CValueInner::ByRef(ptr, Some(meta)), layout)
+ }
+
+ pub(crate) fn by_val(value: Value, layout: TyAndLayout<'tcx>) -> CValue<'tcx> {
+ CValue(CValueInner::ByVal(value), layout)
+ }
+
+ pub(crate) fn by_val_pair(
+ value: Value,
+ extra: Value,
+ layout: TyAndLayout<'tcx>,
+ ) -> CValue<'tcx> {
+ CValue(CValueInner::ByValPair(value, extra), layout)
+ }
+
+ pub(crate) fn layout(&self) -> TyAndLayout<'tcx> {
+ self.1
+ }
+
+ // FIXME remove
+ pub(crate) fn force_stack(self, fx: &mut FunctionCx<'_, '_, 'tcx>) -> (Pointer, Option<Value>) {
+ let layout = self.1;
+ match self.0 {
+ CValueInner::ByRef(ptr, meta) => (ptr, meta),
+ CValueInner::ByVal(_) | CValueInner::ByValPair(_, _) => {
+ let cplace = CPlace::new_stack_slot(fx, layout);
+ cplace.write_cvalue(fx, self);
+ (cplace.to_ptr(), None)
+ }
+ }
+ }
+
+ // FIXME remove
+ /// Forces the data value of a dyn* value to the stack and returns a pointer to it as well as the
+ /// vtable pointer.
+ pub(crate) fn dyn_star_force_data_on_stack(
+ self,
+ fx: &mut FunctionCx<'_, '_, 'tcx>,
+ ) -> (Value, Value) {
+ assert!(self.1.ty.is_dyn_star());
+
+ match self.0 {
+ CValueInner::ByRef(ptr, None) => {
+ let (a_scalar, b_scalar) = match self.1.abi {
+ Abi::ScalarPair(a, b) => (a, b),
+ _ => unreachable!("dyn_star_force_data_on_stack({:?})", self),
+ };
+ let b_offset = scalar_pair_calculate_b_offset(fx.tcx, a_scalar, b_scalar);
+ let clif_ty2 = scalar_to_clif_type(fx.tcx, b_scalar);
+ let mut flags = MemFlags::new();
+ flags.set_notrap();
+ let vtable = ptr.offset(fx, b_offset).load(fx, clif_ty2, flags);
+ (ptr.get_addr(fx), vtable)
+ }
+ CValueInner::ByValPair(data, vtable) => {
+ let stack_slot = fx.bcx.create_sized_stack_slot(StackSlotData {
+ kind: StackSlotKind::ExplicitSlot,
+ // FIXME Don't force the size to a multiple of 16 bytes once Cranelift gets a way to
+ // specify stack slot alignment.
+ size: (u32::try_from(fx.target_config.pointer_type().bytes()).unwrap() + 15)
+ / 16
+ * 16,
+ });
+ let data_ptr = Pointer::stack_slot(stack_slot);
+ let mut flags = MemFlags::new();
+ flags.set_notrap();
+ data_ptr.store(fx, data, flags);
+
+ (data_ptr.get_addr(fx), vtable)
+ }
+ CValueInner::ByRef(_, Some(_)) | CValueInner::ByVal(_) => {
+ unreachable!("dyn_star_force_data_on_stack({:?})", self)
+ }
+ }
+ }
+
+ pub(crate) fn try_to_ptr(self) -> Option<(Pointer, Option<Value>)> {
+ match self.0 {
+ CValueInner::ByRef(ptr, meta) => Some((ptr, meta)),
+ CValueInner::ByVal(_) | CValueInner::ByValPair(_, _) => None,
+ }
+ }
+
+ /// Load a value with layout.abi of scalar
+ pub(crate) fn load_scalar(self, fx: &mut FunctionCx<'_, '_, 'tcx>) -> Value {
+ let layout = self.1;
+ match self.0 {
+ CValueInner::ByRef(ptr, None) => {
+ let clif_ty = match layout.abi {
+ Abi::Scalar(scalar) => scalar_to_clif_type(fx.tcx, scalar),
+ Abi::Vector { element, count } => scalar_to_clif_type(fx.tcx, element)
+ .by(u32::try_from(count).unwrap())
+ .unwrap(),
+ _ => unreachable!("{:?}", layout.ty),
+ };
+ let mut flags = MemFlags::new();
+ flags.set_notrap();
+ ptr.load(fx, clif_ty, flags)
+ }
+ CValueInner::ByVal(value) => value,
+ CValueInner::ByRef(_, Some(_)) => bug!("load_scalar for unsized value not allowed"),
+ CValueInner::ByValPair(_, _) => bug!("Please use load_scalar_pair for ByValPair"),
+ }
+ }
+
+ /// Load a value pair with layout.abi of scalar pair
+ pub(crate) fn load_scalar_pair(self, fx: &mut FunctionCx<'_, '_, 'tcx>) -> (Value, Value) {
+ let layout = self.1;
+ match self.0 {
+ CValueInner::ByRef(ptr, None) => {
+ let (a_scalar, b_scalar) = match layout.abi {
+ Abi::ScalarPair(a, b) => (a, b),
+ _ => unreachable!("load_scalar_pair({:?})", self),
+ };
+ let b_offset = scalar_pair_calculate_b_offset(fx.tcx, a_scalar, b_scalar);
+ let clif_ty1 = scalar_to_clif_type(fx.tcx, a_scalar);
+ let clif_ty2 = scalar_to_clif_type(fx.tcx, b_scalar);
+ let mut flags = MemFlags::new();
+ flags.set_notrap();
+ let val1 = ptr.load(fx, clif_ty1, flags);
+ let val2 = ptr.offset(fx, b_offset).load(fx, clif_ty2, flags);
+ (val1, val2)
+ }
+ CValueInner::ByRef(_, Some(_)) => {
+ bug!("load_scalar_pair for unsized value not allowed")
+ }
+ CValueInner::ByVal(_) => bug!("Please use load_scalar for ByVal"),
+ CValueInner::ByValPair(val1, val2) => (val1, val2),
+ }
+ }
+
+ pub(crate) fn value_field(
+ self,
+ fx: &mut FunctionCx<'_, '_, 'tcx>,
+ field: mir::Field,
+ ) -> CValue<'tcx> {
+ let layout = self.1;
+ match self.0 {
+ CValueInner::ByVal(val) => match layout.abi {
+ Abi::Vector { element: _, count } => {
+ let count = u8::try_from(count).expect("SIMD type with more than 255 lanes???");
+ let field = u8::try_from(field.index()).unwrap();
+ assert!(field < count);
+ let lane = fx.bcx.ins().extractlane(val, field);
+ let field_layout = layout.field(&*fx, usize::from(field));
+ CValue::by_val(lane, field_layout)
+ }
+ _ => unreachable!("value_field for ByVal with abi {:?}", layout.abi),
+ },
+ CValueInner::ByValPair(val1, val2) => match layout.abi {
+ Abi::ScalarPair(_, _) => {
+ let val = match field.as_u32() {
+ 0 => val1,
+ 1 => val2,
+ _ => bug!("field should be 0 or 1"),
+ };
+ let field_layout = layout.field(&*fx, usize::from(field));
+ CValue::by_val(val, field_layout)
+ }
+ _ => unreachable!("value_field for ByValPair with abi {:?}", layout.abi),
+ },
+ CValueInner::ByRef(ptr, None) => {
+ let (field_ptr, field_layout) = codegen_field(fx, ptr, None, layout, field);
+ CValue::by_ref(field_ptr, field_layout)
+ }
+ CValueInner::ByRef(_, Some(_)) => todo!(),
+ }
+ }
+
+ /// Like [`CValue::value_field`] except handling ADTs containing a single array field in a way
+ /// such that you can access individual lanes.
+ pub(crate) fn value_lane(
+ self,
+ fx: &mut FunctionCx<'_, '_, 'tcx>,
+ lane_idx: u64,
+ ) -> CValue<'tcx> {
+ let layout = self.1;
+ assert!(layout.ty.is_simd());
+ let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx);
+ let lane_layout = fx.layout_of(lane_ty);
+ assert!(lane_idx < lane_count);
+ match self.0 {
+ CValueInner::ByVal(val) => match layout.abi {
+ Abi::Vector { element: _, count: _ } => {
+ assert!(lane_count <= u8::MAX.into(), "SIMD type with more than 255 lanes???");
+ let lane_idx = u8::try_from(lane_idx).unwrap();
+ let lane = fx.bcx.ins().extractlane(val, lane_idx);
+ CValue::by_val(lane, lane_layout)
+ }
+ _ => unreachable!("value_lane for ByVal with abi {:?}", layout.abi),
+ },
+ CValueInner::ByValPair(_, _) => unreachable!(),
+ CValueInner::ByRef(ptr, None) => {
+ let field_offset = lane_layout.size * lane_idx;
+ let field_ptr = ptr.offset_i64(fx, i64::try_from(field_offset.bytes()).unwrap());
+ CValue::by_ref(field_ptr, lane_layout)
+ }
+ CValueInner::ByRef(_, Some(_)) => unreachable!(),
+ }
+ }
+
+ pub(crate) fn unsize_value(self, fx: &mut FunctionCx<'_, '_, 'tcx>, dest: CPlace<'tcx>) {
+ crate::unsize::coerce_unsized_into(fx, self, dest);
+ }
+
+ pub(crate) fn coerce_dyn_star(self, fx: &mut FunctionCx<'_, '_, 'tcx>, dest: CPlace<'tcx>) {
+ crate::unsize::coerce_dyn_star(fx, self, dest);
+ }
+
+ /// If `ty` is signed, `const_val` must already be sign extended.
+ pub(crate) fn const_val(
+ fx: &mut FunctionCx<'_, '_, 'tcx>,
+ layout: TyAndLayout<'tcx>,
+ const_val: ty::ScalarInt,
+ ) -> CValue<'tcx> {
+ assert_eq!(const_val.size(), layout.size, "{:#?}: {:?}", const_val, layout);
+ use cranelift_codegen::ir::immediates::{Ieee32, Ieee64};
+
+ let clif_ty = fx.clif_type(layout.ty).unwrap();
+
+ if let ty::Bool = layout.ty.kind() {
+ assert!(
+ const_val == ty::ScalarInt::FALSE || const_val == ty::ScalarInt::TRUE,
+ "Invalid bool 0x{:032X}",
+ const_val
+ );
+ }
+
+ let val = match layout.ty.kind() {
+ ty::Uint(UintTy::U128) | ty::Int(IntTy::I128) => {
+ let const_val = const_val.to_bits(layout.size).unwrap();
+ let lsb = fx.bcx.ins().iconst(types::I64, const_val as u64 as i64);
+ let msb = fx.bcx.ins().iconst(types::I64, (const_val >> 64) as u64 as i64);
+ fx.bcx.ins().iconcat(lsb, msb)
+ }
+ ty::Bool | ty::Char | ty::Uint(_) | ty::Int(_) | ty::Ref(..) | ty::RawPtr(..) => {
+ fx.bcx.ins().iconst(clif_ty, const_val.to_bits(layout.size).unwrap() as i64)
+ }
+ ty::Float(FloatTy::F32) => {
+ fx.bcx.ins().f32const(Ieee32::with_bits(u32::try_from(const_val).unwrap()))
+ }
+ ty::Float(FloatTy::F64) => {
+ fx.bcx.ins().f64const(Ieee64::with_bits(u64::try_from(const_val).unwrap()))
+ }
+ _ => panic!(
+ "CValue::const_val for non bool/char/float/integer/pointer type {:?} is not allowed",
+ layout.ty
+ ),
+ };
+
+ CValue::by_val(val, layout)
+ }
+
+ pub(crate) fn cast_pointer_to(self, layout: TyAndLayout<'tcx>) -> Self {
+ assert!(matches!(self.layout().ty.kind(), ty::Ref(..) | ty::RawPtr(..) | ty::FnPtr(..)));
+ assert!(matches!(layout.ty.kind(), ty::Ref(..) | ty::RawPtr(..) | ty::FnPtr(..)));
+ assert_eq!(self.layout().abi, layout.abi);
+ CValue(self.0, layout)
+ }
+}
+
+/// A place where you can write a value to or read a value from
+#[derive(Debug, Copy, Clone)]
+pub(crate) struct CPlace<'tcx> {
+ inner: CPlaceInner,
+ layout: TyAndLayout<'tcx>,
+}
+
+#[derive(Debug, Copy, Clone)]
+pub(crate) enum CPlaceInner {
+ Var(Local, Variable),
+ VarPair(Local, Variable, Variable),
+ VarLane(Local, Variable, u8),
+ Addr(Pointer, Option<Value>),
+}
+
+impl<'tcx> CPlace<'tcx> {
+ pub(crate) fn layout(&self) -> TyAndLayout<'tcx> {
+ self.layout
+ }
+
+ pub(crate) fn inner(&self) -> &CPlaceInner {
+ &self.inner
+ }
+
+ pub(crate) fn new_stack_slot(
+ fx: &mut FunctionCx<'_, '_, 'tcx>,
+ layout: TyAndLayout<'tcx>,
+ ) -> CPlace<'tcx> {
+ assert!(layout.is_sized());
+ if layout.size.bytes() == 0 {
+ return CPlace {
+ inner: CPlaceInner::Addr(Pointer::dangling(layout.align.pref), None),
+ layout,
+ };
+ }
+
+ if layout.size.bytes() >= u64::from(u32::MAX - 16) {
+ fx.tcx
+ .sess
+ .fatal(&format!("values of type {} are too big to store on the stack", layout.ty));
+ }
+
+ let stack_slot = fx.bcx.create_sized_stack_slot(StackSlotData {
+ kind: StackSlotKind::ExplicitSlot,
+ // FIXME Don't force the size to a multiple of 16 bytes once Cranelift gets a way to
+ // specify stack slot alignment.
+ size: (u32::try_from(layout.size.bytes()).unwrap() + 15) / 16 * 16,
+ });
+ CPlace { inner: CPlaceInner::Addr(Pointer::stack_slot(stack_slot), None), layout }
+ }
+
+ pub(crate) fn new_var(
+ fx: &mut FunctionCx<'_, '_, 'tcx>,
+ local: Local,
+ layout: TyAndLayout<'tcx>,
+ ) -> CPlace<'tcx> {
+ let var = Variable::from_u32(fx.next_ssa_var);
+ fx.next_ssa_var += 1;
+ fx.bcx.declare_var(var, fx.clif_type(layout.ty).unwrap());
+ CPlace { inner: CPlaceInner::Var(local, var), layout }
+ }
+
+ pub(crate) fn new_var_pair(
+ fx: &mut FunctionCx<'_, '_, 'tcx>,
+ local: Local,
+ layout: TyAndLayout<'tcx>,
+ ) -> CPlace<'tcx> {
+ let var1 = Variable::from_u32(fx.next_ssa_var);
+ fx.next_ssa_var += 1;
+ let var2 = Variable::from_u32(fx.next_ssa_var);
+ fx.next_ssa_var += 1;
+
+ let (ty1, ty2) = fx.clif_pair_type(layout.ty).unwrap();
+ fx.bcx.declare_var(var1, ty1);
+ fx.bcx.declare_var(var2, ty2);
+ CPlace { inner: CPlaceInner::VarPair(local, var1, var2), layout }
+ }
+
+ pub(crate) fn for_ptr(ptr: Pointer, layout: TyAndLayout<'tcx>) -> CPlace<'tcx> {
+ CPlace { inner: CPlaceInner::Addr(ptr, None), layout }
+ }
+
+ pub(crate) fn for_ptr_with_extra(
+ ptr: Pointer,
+ extra: Value,
+ layout: TyAndLayout<'tcx>,
+ ) -> CPlace<'tcx> {
+ CPlace { inner: CPlaceInner::Addr(ptr, Some(extra)), layout }
+ }
+
+ pub(crate) fn to_cvalue(self, fx: &mut FunctionCx<'_, '_, 'tcx>) -> CValue<'tcx> {
+ let layout = self.layout();
+ match self.inner {
+ CPlaceInner::Var(_local, var) => {
+ let val = fx.bcx.use_var(var);
+ //fx.bcx.set_val_label(val, cranelift_codegen::ir::ValueLabel::new(var.index()));
+ CValue::by_val(val, layout)
+ }
+ CPlaceInner::VarPair(_local, var1, var2) => {
+ let val1 = fx.bcx.use_var(var1);
+ //fx.bcx.set_val_label(val1, cranelift_codegen::ir::ValueLabel::new(var1.index()));
+ let val2 = fx.bcx.use_var(var2);
+ //fx.bcx.set_val_label(val2, cranelift_codegen::ir::ValueLabel::new(var2.index()));
+ CValue::by_val_pair(val1, val2, layout)
+ }
+ CPlaceInner::VarLane(_local, var, lane) => {
+ let val = fx.bcx.use_var(var);
+ //fx.bcx.set_val_label(val, cranelift_codegen::ir::ValueLabel::new(var.index()));
+ let val = fx.bcx.ins().extractlane(val, lane);
+ CValue::by_val(val, layout)
+ }
+ CPlaceInner::Addr(ptr, extra) => {
+ if let Some(extra) = extra {
+ CValue::by_ref_unsized(ptr, extra, layout)
+ } else {
+ CValue::by_ref(ptr, layout)
+ }
+ }
+ }
+ }
+
+ pub(crate) fn to_ptr(self) -> Pointer {
+ match self.to_ptr_maybe_unsized() {
+ (ptr, None) => ptr,
+ (_, Some(_)) => bug!("Expected sized cplace, found {:?}", self),
+ }
+ }
+
+ pub(crate) fn to_ptr_maybe_unsized(self) -> (Pointer, Option<Value>) {
+ match self.inner {
+ CPlaceInner::Addr(ptr, extra) => (ptr, extra),
+ CPlaceInner::Var(_, _)
+ | CPlaceInner::VarPair(_, _, _)
+ | CPlaceInner::VarLane(_, _, _) => bug!("Expected CPlace::Addr, found {:?}", self),
+ }
+ }
+
+ pub(crate) fn write_cvalue(self, fx: &mut FunctionCx<'_, '_, 'tcx>, from: CValue<'tcx>) {
+ assert_assignable(fx, from.layout().ty, self.layout().ty, 16);
+
+ self.write_cvalue_maybe_transmute(fx, from, "write_cvalue");
+ }
+
+ pub(crate) fn write_cvalue_transmute(
+ self,
+ fx: &mut FunctionCx<'_, '_, 'tcx>,
+ from: CValue<'tcx>,
+ ) {
+ self.write_cvalue_maybe_transmute(fx, from, "write_cvalue_transmute");
+ }
+
+ fn write_cvalue_maybe_transmute(
+ self,
+ fx: &mut FunctionCx<'_, '_, 'tcx>,
+ from: CValue<'tcx>,
+ method: &'static str,
+ ) {
+ fn transmute_value<'tcx>(
+ fx: &mut FunctionCx<'_, '_, 'tcx>,
+ var: Variable,
+ data: Value,
+ dst_ty: Type,
+ ) {
+ let src_ty = fx.bcx.func.dfg.value_type(data);
+ assert_eq!(
+ src_ty.bytes(),
+ dst_ty.bytes(),
+ "write_cvalue_transmute: {:?} -> {:?}",
+ src_ty,
+ dst_ty,
+ );
+ let data = match (src_ty, dst_ty) {
+ (_, _) if src_ty == dst_ty => data,
+
+ // This is a `write_cvalue_transmute`.
+ (types::I32, types::F32)
+ | (types::F32, types::I32)
+ | (types::I64, types::F64)
++ | (types::F64, types::I64) => codegen_bitcast(fx, dst_ty, data),
++ _ if src_ty.is_vector() && dst_ty.is_vector() => codegen_bitcast(fx, dst_ty, data),
+ _ if src_ty.is_vector() || dst_ty.is_vector() => {
+ // FIXME do something more efficient for transmutes between vectors and integers.
+ let stack_slot = fx.bcx.create_sized_stack_slot(StackSlotData {
+ kind: StackSlotKind::ExplicitSlot,
+ // FIXME Don't force the size to a multiple of 16 bytes once Cranelift gets a way to
+ // specify stack slot alignment.
+ size: (src_ty.bytes() + 15) / 16 * 16,
+ });
+ let ptr = Pointer::stack_slot(stack_slot);
+ ptr.store(fx, data, MemFlags::trusted());
+ ptr.load(fx, dst_ty, MemFlags::trusted())
+ }
+
+ // `CValue`s should never contain SSA-only types, so if you ended
+ // up here having seen an error like `B1 -> I8`, then before
+ // calling `write_cvalue` you need to add a `bint` instruction.
+ _ => unreachable!("write_cvalue_transmute: {:?} -> {:?}", src_ty, dst_ty),
+ };
+ //fx.bcx.set_val_label(data, cranelift_codegen::ir::ValueLabel::new(var.index()));
+ fx.bcx.def_var(var, data);
+ }
+
+ assert_eq!(self.layout().size, from.layout().size);
+
+ if fx.clif_comments.enabled() {
+ use cranelift_codegen::cursor::{Cursor, CursorPosition};
+ let cur_block = match fx.bcx.cursor().position() {
+ CursorPosition::After(block) => block,
+ _ => unreachable!(),
+ };
+ fx.add_comment(
+ fx.bcx.func.layout.last_inst(cur_block).unwrap(),
+ format!(
+ "{}: {:?}: {:?} <- {:?}: {:?}",
+ method,
+ self.inner(),
+ self.layout().ty,
+ from.0,
+ from.layout().ty
+ ),
+ );
+ }
+
+ let dst_layout = self.layout();
+ let to_ptr = match self.inner {
+ CPlaceInner::Var(_local, var) => {
+ if let ty::Array(element, len) = dst_layout.ty.kind() {
+ // Can only happen for vector types
+ let len =
+ u32::try_from(len.eval_usize(fx.tcx, ParamEnv::reveal_all())).unwrap();
+ let vector_ty = fx.clif_type(*element).unwrap().by(len).unwrap();
+
+ let data = match from.0 {
+ CValueInner::ByRef(ptr, None) => {
+ let mut flags = MemFlags::new();
+ flags.set_notrap();
+ ptr.load(fx, vector_ty, flags)
+ }
+ CValueInner::ByVal(_)
+ | CValueInner::ByValPair(_, _)
+ | CValueInner::ByRef(_, Some(_)) => bug!("array should be ByRef"),
+ };
+
+ fx.bcx.def_var(var, data);
+ return;
+ }
+ let data = CValue(from.0, dst_layout).load_scalar(fx);
+ let dst_ty = fx.clif_type(self.layout().ty).unwrap();
+ transmute_value(fx, var, data, dst_ty);
+ return;
+ }
+ CPlaceInner::VarPair(_local, var1, var2) => {
+ let (ptr, meta) = from.force_stack(fx);
+ assert!(meta.is_none());
+ let (data1, data2) =
+ CValue(CValueInner::ByRef(ptr, None), dst_layout).load_scalar_pair(fx);
+ let (dst_ty1, dst_ty2) = fx.clif_pair_type(self.layout().ty).unwrap();
+ transmute_value(fx, var1, data1, dst_ty1);
+ transmute_value(fx, var2, data2, dst_ty2);
+ return;
+ }
+ CPlaceInner::VarLane(_local, var, lane) => {
+ let data = from.load_scalar(fx);
+
+ // First get the old vector
+ let vector = fx.bcx.use_var(var);
+ //fx.bcx.set_val_label(vector, cranelift_codegen::ir::ValueLabel::new(var.index()));
+
+ // Next insert the written lane into the vector
+ let vector = fx.bcx.ins().insertlane(vector, data, lane);
+
+ // Finally write the new vector
+ //fx.bcx.set_val_label(vector, cranelift_codegen::ir::ValueLabel::new(var.index()));
+ fx.bcx.def_var(var, vector);
+
+ return;
+ }
+ CPlaceInner::Addr(ptr, None) => {
+ if dst_layout.size == Size::ZERO || dst_layout.abi == Abi::Uninhabited {
+ return;
+ }
+ ptr
+ }
+ CPlaceInner::Addr(_, Some(_)) => bug!("Can't write value to unsized place {:?}", self),
+ };
+
+ let mut flags = MemFlags::new();
+ flags.set_notrap();
+ match from.layout().abi {
+ // FIXME make Abi::Vector work too
+ Abi::Scalar(_) => {
+ let val = from.load_scalar(fx);
+ to_ptr.store(fx, val, flags);
+ return;
+ }
+ Abi::ScalarPair(a_scalar, b_scalar) => {
+ let (value, extra) = from.load_scalar_pair(fx);
+ let b_offset = scalar_pair_calculate_b_offset(fx.tcx, a_scalar, b_scalar);
+ to_ptr.store(fx, value, flags);
+ to_ptr.offset(fx, b_offset).store(fx, extra, flags);
+ return;
+ }
+ _ => {}
+ }
+
+ match from.0 {
+ CValueInner::ByVal(val) => {
+ to_ptr.store(fx, val, flags);
+ }
+ CValueInner::ByValPair(_, _) => {
+ bug!("Non ScalarPair abi {:?} for ByValPair CValue", dst_layout.abi);
+ }
+ CValueInner::ByRef(from_ptr, None) => {
+ let from_addr = from_ptr.get_addr(fx);
+ let to_addr = to_ptr.get_addr(fx);
+ let src_layout = from.1;
+ let size = dst_layout.size.bytes();
+ let src_align = src_layout.align.abi.bytes() as u8;
+ let dst_align = dst_layout.align.abi.bytes() as u8;
+ fx.bcx.emit_small_memory_copy(
+ fx.target_config,
+ to_addr,
+ from_addr,
+ size,
+ dst_align,
+ src_align,
+ true,
+ flags,
+ );
+ }
+ CValueInner::ByRef(_, Some(_)) => todo!(),
+ }
+ }
+
+ pub(crate) fn place_opaque_cast(
+ self,
+ fx: &mut FunctionCx<'_, '_, 'tcx>,
+ ty: Ty<'tcx>,
+ ) -> CPlace<'tcx> {
+ CPlace { inner: self.inner, layout: fx.layout_of(ty) }
+ }
+
+ pub(crate) fn place_field(
+ self,
+ fx: &mut FunctionCx<'_, '_, 'tcx>,
+ field: mir::Field,
+ ) -> CPlace<'tcx> {
+ let layout = self.layout();
+
+ match self.inner {
+ CPlaceInner::Var(local, var) => match layout.ty.kind() {
+ ty::Array(_, _) => {
+ // Can only happen for vector types
+ return CPlace {
+ inner: CPlaceInner::VarLane(local, var, field.as_u32().try_into().unwrap()),
+ layout: layout.field(fx, field.as_u32().try_into().unwrap()),
+ };
+ }
+ ty::Adt(adt_def, substs) if layout.ty.is_simd() => {
+ let f0_ty = adt_def.non_enum_variant().fields[0].ty(fx.tcx, substs);
+
+ match f0_ty.kind() {
+ ty::Array(_, _) => {
+ assert_eq!(field.as_u32(), 0);
+ return CPlace {
+ inner: CPlaceInner::Var(local, var),
+ layout: layout.field(fx, field.as_u32().try_into().unwrap()),
+ };
+ }
+ _ => {
+ return CPlace {
+ inner: CPlaceInner::VarLane(
+ local,
+ var,
+ field.as_u32().try_into().unwrap(),
+ ),
+ layout: layout.field(fx, field.as_u32().try_into().unwrap()),
+ };
+ }
+ }
+ }
+ _ => {}
+ },
+ CPlaceInner::VarPair(local, var1, var2) => {
+ let layout = layout.field(&*fx, field.index());
+
+ match field.as_u32() {
+ 0 => return CPlace { inner: CPlaceInner::Var(local, var1), layout },
+ 1 => return CPlace { inner: CPlaceInner::Var(local, var2), layout },
+ _ => unreachable!("field should be 0 or 1"),
+ }
+ }
+ _ => {}
+ }
+
+ let (base, extra) = self.to_ptr_maybe_unsized();
+
+ let (field_ptr, field_layout) = codegen_field(fx, base, extra, layout, field);
+ if field_layout.is_unsized() {
+ if let ty::Foreign(_) = field_layout.ty.kind() {
+ assert!(extra.is_none());
+ CPlace::for_ptr(field_ptr, field_layout)
+ } else {
+ CPlace::for_ptr_with_extra(field_ptr, extra.unwrap(), field_layout)
+ }
+ } else {
+ CPlace::for_ptr(field_ptr, field_layout)
+ }
+ }
+
+ /// Like [`CPlace::place_field`] except handling ADTs containing a single array field in a way
+ /// such that you can access individual lanes.
+ pub(crate) fn place_lane(
+ self,
+ fx: &mut FunctionCx<'_, '_, 'tcx>,
+ lane_idx: u64,
+ ) -> CPlace<'tcx> {
+ let layout = self.layout();
+ assert!(layout.ty.is_simd());
+ let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx);
+ let lane_layout = fx.layout_of(lane_ty);
+ assert!(lane_idx < lane_count);
+
+ match self.inner {
+ CPlaceInner::Var(local, var) => {
+ assert!(matches!(layout.abi, Abi::Vector { .. }));
+ CPlace {
+ inner: CPlaceInner::VarLane(local, var, lane_idx.try_into().unwrap()),
+ layout: lane_layout,
+ }
+ }
+ CPlaceInner::VarPair(_, _, _) => unreachable!(),
+ CPlaceInner::VarLane(_, _, _) => unreachable!(),
+ CPlaceInner::Addr(ptr, None) => {
+ let field_offset = lane_layout.size * lane_idx;
+ let field_ptr = ptr.offset_i64(fx, i64::try_from(field_offset.bytes()).unwrap());
+ CPlace::for_ptr(field_ptr, lane_layout)
+ }
+ CPlaceInner::Addr(_, Some(_)) => unreachable!(),
+ }
+ }
+
+ pub(crate) fn place_index(
+ self,
+ fx: &mut FunctionCx<'_, '_, 'tcx>,
+ index: Value,
+ ) -> CPlace<'tcx> {
+ let (elem_layout, ptr) = match self.layout().ty.kind() {
+ ty::Array(elem_ty, _) => (fx.layout_of(*elem_ty), self.to_ptr()),
+ ty::Slice(elem_ty) => (fx.layout_of(*elem_ty), self.to_ptr_maybe_unsized().0),
+ _ => bug!("place_index({:?})", self.layout().ty),
+ };
+
+ let offset = fx.bcx.ins().imul_imm(index, elem_layout.size.bytes() as i64);
+
+ CPlace::for_ptr(ptr.offset_value(fx, offset), elem_layout)
+ }
+
+ pub(crate) fn place_deref(self, fx: &mut FunctionCx<'_, '_, 'tcx>) -> CPlace<'tcx> {
+ let inner_layout = fx.layout_of(self.layout().ty.builtin_deref(true).unwrap().ty);
+ if has_ptr_meta(fx.tcx, inner_layout.ty) {
+ let (addr, extra) = self.to_cvalue(fx).load_scalar_pair(fx);
+ CPlace::for_ptr_with_extra(Pointer::new(addr), extra, inner_layout)
+ } else {
+ CPlace::for_ptr(Pointer::new(self.to_cvalue(fx).load_scalar(fx)), inner_layout)
+ }
+ }
+
+ pub(crate) fn place_ref(
+ self,
+ fx: &mut FunctionCx<'_, '_, 'tcx>,
+ layout: TyAndLayout<'tcx>,
+ ) -> CValue<'tcx> {
+ if has_ptr_meta(fx.tcx, self.layout().ty) {
+ let (ptr, extra) = self.to_ptr_maybe_unsized();
+ CValue::by_val_pair(
+ ptr.get_addr(fx),
+ extra.expect("unsized type without metadata"),
+ layout,
+ )
+ } else {
+ CValue::by_val(self.to_ptr().get_addr(fx), layout)
+ }
+ }
+
+ pub(crate) fn downcast_variant(
+ self,
+ fx: &FunctionCx<'_, '_, 'tcx>,
+ variant: VariantIdx,
+ ) -> Self {
+ assert!(self.layout().is_sized());
+ let layout = self.layout().for_variant(fx, variant);
+ CPlace { inner: self.inner, layout }
+ }
+}
+
+#[track_caller]
+pub(crate) fn assert_assignable<'tcx>(
+ fx: &FunctionCx<'_, '_, 'tcx>,
+ from_ty: Ty<'tcx>,
+ to_ty: Ty<'tcx>,
+ limit: usize,
+) {
+ if limit == 0 {
+ // assert_assignable exists solely to catch bugs in cg_clif. it isn't necessary for
+ // soundness. don't attempt to check deep types to avoid exponential behavior in certain
+ // cases.
+ return;
+ }
+ match (from_ty.kind(), to_ty.kind()) {
+ (ty::Ref(_, a, _), ty::Ref(_, b, _))
+ | (
+ ty::RawPtr(TypeAndMut { ty: a, mutbl: _ }),
+ ty::RawPtr(TypeAndMut { ty: b, mutbl: _ }),
+ ) => {
+ assert_assignable(fx, *a, *b, limit - 1);
+ }
+ (ty::Ref(_, a, _), ty::RawPtr(TypeAndMut { ty: b, mutbl: _ }))
+ | (ty::RawPtr(TypeAndMut { ty: a, mutbl: _ }), ty::Ref(_, b, _)) => {
+ assert_assignable(fx, *a, *b, limit - 1);
+ }
+ (ty::FnPtr(_), ty::FnPtr(_)) => {
+ let from_sig = fx.tcx.normalize_erasing_late_bound_regions(
+ ParamEnv::reveal_all(),
+ from_ty.fn_sig(fx.tcx),
+ );
+ let to_sig = fx
+ .tcx
+ .normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), to_ty.fn_sig(fx.tcx));
+ assert_eq!(
+ from_sig, to_sig,
+ "Can't write fn ptr with incompatible sig {:?} to place with sig {:?}\n\n{:#?}",
+ from_sig, to_sig, fx,
+ );
+ // fn(&T) -> for<'l> fn(&'l T) is allowed
+ }
+ (&ty::Dynamic(from_traits, _, _from_kind), &ty::Dynamic(to_traits, _, _to_kind)) => {
+ // FIXME(dyn-star): Do the right thing with DynKinds
+ for (from, to) in from_traits.iter().zip(to_traits) {
+ let from =
+ fx.tcx.normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), from);
+ let to = fx.tcx.normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), to);
+ assert_eq!(
+ from, to,
+ "Can't write trait object of incompatible traits {:?} to place with traits {:?}\n\n{:#?}",
+ from_traits, to_traits, fx,
+ );
+ }
+ // dyn for<'r> Trait<'r> -> dyn Trait<'_> is allowed
+ }
+ (&ty::Tuple(types_a), &ty::Tuple(types_b)) => {
+ let mut types_a = types_a.iter();
+ let mut types_b = types_b.iter();
+ loop {
+ match (types_a.next(), types_b.next()) {
+ (Some(a), Some(b)) => assert_assignable(fx, a, b, limit - 1),
+ (None, None) => return,
+ (Some(_), None) | (None, Some(_)) => panic!("{:#?}/{:#?}", from_ty, to_ty),
+ }
+ }
+ }
+ (&ty::Adt(adt_def_a, substs_a), &ty::Adt(adt_def_b, substs_b))
+ if adt_def_a.did() == adt_def_b.did() =>
+ {
+ let mut types_a = substs_a.types();
+ let mut types_b = substs_b.types();
+ loop {
+ match (types_a.next(), types_b.next()) {
+ (Some(a), Some(b)) => assert_assignable(fx, a, b, limit - 1),
+ (None, None) => return,
+ (Some(_), None) | (None, Some(_)) => panic!("{:#?}/{:#?}", from_ty, to_ty),
+ }
+ }
+ }
+ (ty::Array(a, _), ty::Array(b, _)) => assert_assignable(fx, *a, *b, limit - 1),
+ (&ty::Closure(def_id_a, substs_a), &ty::Closure(def_id_b, substs_b))
+ if def_id_a == def_id_b =>
+ {
+ let mut types_a = substs_a.types();
+ let mut types_b = substs_b.types();
+ loop {
+ match (types_a.next(), types_b.next()) {
+ (Some(a), Some(b)) => assert_assignable(fx, a, b, limit - 1),
+ (None, None) => return,
+ (Some(_), None) | (None, Some(_)) => panic!("{:#?}/{:#?}", from_ty, to_ty),
+ }
+ }
+ }
+ (ty::Param(_), _) | (_, ty::Param(_)) if fx.tcx.sess.opts.unstable_opts.polymorphize => {
+ // No way to check if it is correct or not with polymorphization enabled
+ }
+ _ => {
+ assert_eq!(
+ from_ty,
+ to_ty,
+ "Can't write value with incompatible type {:?} to place with type {:?}\n\n{:#?}",
+ from_ty.kind(),
+ to_ty.kind(),
+ fx,
+ );
+ }
+ }
+}
--- /dev/null
- rustc $0 -o ${0/.rs/.bin} -Cdebuginfo=1 --edition 2021
+#!/usr/bin/env bash
+#![deny(unsafe_code)] /*This line is ignored by bash
+# This block is ignored by rustc
+set -e
+echo "[BUILD] y.rs" 1>&2
++rustc $0 -o ${0/.rs/.bin} -Cdebuginfo=1 --edition 2021 -Cpanic=abort
+exec ${0/.rs/.bin} $@
+*/
+
+//! The build system for cg_clif
+//!
+//! # Manual compilation
+//!
+//! If your system doesn't support shell scripts you can manually compile and run this file using
+//! for example:
+//!
+//! ```shell
+//! $ rustc y.rs -o y.bin
+//! $ ./y.bin
+//! ```
+//!
+//! # Naming
+//!
+//! The name `y.rs` was chosen to not conflict with rustc's `x.py`.
+
+#[path = "build_system/mod.rs"]
+mod build_system;
+
+fn main() {
+ build_system::main();
+}
projects / rust.git / commitdiff