--- /dev/null
- version = "1.0.53"
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 3
+
+[[package]]
+name = "anyhow"
- checksum = "94a45b455c14666b85fc40a019e8ab9eb75e3a124e05494f5397122bc9eb06e0"
++version = "1.0.56"
+source = "registry+https://github.com/rust-lang/crates.io-index"
- version = "1.0.1"
++checksum = "4361135be9122e0870de935d7c439aef945b9f9ddd4199a553b5270b49c82a27"
+
+[[package]]
+name = "ar"
+version = "0.8.0"
+source = "git+https://github.com/bjorn3/rust-ar.git?branch=do_not_remove_cg_clif_ranlib#de9ab0e56bf3a208381d342aa5b60f9ff2891648"
+
+[[package]]
+name = "autocfg"
- checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a"
++version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
- version = "0.81.0"
++checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
+
+[[package]]
+name = "bitflags"
+version = "1.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
+
+[[package]]
+name = "cfg-if"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
+
+[[package]]
+name = "cranelift-bforest"
- checksum = "71447555acc6c875c52c407d572fc1327dc5c34cba72b4b2e7ad048aa4e4fd19"
++version = "0.82.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
- version = "0.81.0"
++checksum = "d16922317bd7dd104d509a373887822caa0242fc1def00de66abb538db221db4"
+dependencies = [
+ "cranelift-entity",
+]
+
+[[package]]
+name = "cranelift-codegen"
- checksum = "ec9a10261891a7a919b0d4f6aa73582e88441d9a8f6173c88efbe4a5a362ea67"
++version = "0.82.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
- version = "0.81.0"
++checksum = "8b80bf40380256307b68a3dcbe1b91cac92a533e212b5b635abc3e4525781a0a"
+dependencies = [
+ "cranelift-bforest",
+ "cranelift-codegen-meta",
+ "cranelift-codegen-shared",
+ "cranelift-entity",
+ "gimli",
+ "log",
+ "regalloc",
+ "smallvec",
+ "target-lexicon",
+]
+
+[[package]]
+name = "cranelift-codegen-meta"
- checksum = "815755d76fcbcf6e17ab888545b28ab775f917cb12ce0797e60cd41a2288692c"
++version = "0.82.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
- version = "0.81.0"
++checksum = "703d0ed7d3bc6c7a814ca12858175bf4e93167a3584127858c686e4b5dd6e432"
+dependencies = [
+ "cranelift-codegen-shared",
+]
+
+[[package]]
+name = "cranelift-codegen-shared"
- checksum = "23ea92f2a67335a2e4d3c9c65624c3b14ae287d595b0650822c41824febab66b"
++version = "0.82.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
- version = "0.81.0"
++checksum = "80f52311e1c90de12dcf8c4b9999c6ebfd1ed360373e88c357160936844511f6"
+
+[[package]]
+name = "cranelift-entity"
- checksum = "bd25847875e388c500ad3624b4d2e14067955c93185194a7222246a25b91c975"
++version = "0.82.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
- version = "0.81.0"
++checksum = "66bc82ef522c1f643baf7d4d40b7c52643ee4549d8960b0e6a047daacb83f897"
+
+[[package]]
+name = "cranelift-frontend"
- checksum = "308bcfb7eb47bdf5ff6e1ace262af4ed39ec19f204c751fffb037e0e82a0c8bf"
++version = "0.82.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
- version = "0.81.0"
++checksum = "3cc35e4251864b17515845ba47447bca88fec9ca1a4186b19fe42526e36140e8"
+dependencies = [
+ "cranelift-codegen",
+ "log",
+ "smallvec",
+ "target-lexicon",
+]
+
+[[package]]
+name = "cranelift-jit"
- checksum = "f560b3a314b8d15facf411e5d29b917c3e787a2bbc3fcdc5183bc0c5b7d4fe01"
++version = "0.82.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
- version = "0.81.0"
++checksum = "93c66d594ad3bfe4e58b1fbd8d17877a7c6564a5f2d6f78cbbf4b0182af1927f"
+dependencies = [
+ "anyhow",
+ "cranelift-codegen",
+ "cranelift-entity",
+ "cranelift-module",
+ "cranelift-native",
+ "libc",
+ "log",
+ "region",
+ "target-lexicon",
+ "winapi",
+]
+
+[[package]]
+name = "cranelift-module"
- checksum = "3a57aba9e603d694d1430ff38bd914bae23ef9c2e44b25a65e318905807e654c"
++version = "0.82.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
- version = "0.81.0"
++checksum = "bf356697c40232aa09e1e3fb8a350ee894e849ccecc4eac56ff0570a4575c325"
+dependencies = [
+ "anyhow",
+ "cranelift-codegen",
+]
+
+[[package]]
+name = "cranelift-native"
- checksum = "12cdc799aee673be2317e631d4569a1ba0a7e77a07a7ce45557086d2e02e9514"
++version = "0.82.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
- version = "0.81.0"
++checksum = "b882b2251c9845d509d92aebfdb6c8bb3b3b48e207ac951f21fbd20cfe7f90b3"
+dependencies = [
+ "cranelift-codegen",
+ "libc",
+ "target-lexicon",
+]
+
+[[package]]
+name = "cranelift-object"
- checksum = "502a7333836052fcdf4425d7f7a21264d99f862d32b9c3a0e47cd920487a9b60"
++version = "0.82.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
- version = "1.3.1"
++checksum = "2d3f1a88e654e567d2591169239ed157ab290811a729a6468f53999c01001263"
+dependencies = [
+ "anyhow",
+ "cranelift-codegen",
+ "cranelift-module",
+ "log",
+ "object",
+ "target-lexicon",
+]
+
+[[package]]
+name = "crc32fast"
- checksum = "a2209c310e29876f7f0b2721e7e26b84aff178aa3da5d091f9bfbf47669e60e3"
++version = "1.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
- version = "0.2.116"
++checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d"
+dependencies = [
+ "cfg-if",
+]
+
+[[package]]
+name = "gimli"
+version = "0.26.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "78cc372d058dcf6d5ecd98510e7fbc9e5aec4d21de70f65fea8fecebcd881bd4"
+dependencies = [
+ "indexmap",
+]
+
+[[package]]
+name = "hashbrown"
+version = "0.11.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e"
+
+[[package]]
+name = "indexmap"
+version = "1.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "282a6247722caba404c065016bbfa522806e51714c34f5dfc3e4a3a46fcb4223"
+dependencies = [
+ "autocfg",
+ "hashbrown",
+]
+
+[[package]]
+name = "libc"
- checksum = "565dbd88872dbe4cc8a46e527f26483c1d1f7afa6b884a3bd6cd893d4f98da74"
++version = "0.2.119"
+source = "registry+https://github.com/rust-lang/crates.io-index"
- version = "0.12.2"
++checksum = "1bf2e165bb3457c8e098ea76f3e3bc9db55f87aa90d52d0e6be741470916aaa4"
+
+[[package]]
+name = "libloading"
+version = "0.6.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "351a32417a12d5f7e82c368a66781e307834dae04c6ce0cd4456d52989229883"
+dependencies = [
+ "cfg-if",
+ "winapi",
+]
+
+[[package]]
+name = "log"
+version = "0.4.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710"
+dependencies = [
+ "cfg-if",
+]
+
+[[package]]
+name = "mach"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b823e83b2affd8f40a9ee8c29dbc56404c1e34cd2710921f2801e2cf29527afa"
+dependencies = [
+ "libc",
+]
+
+[[package]]
+name = "memchr"
+version = "2.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a"
+
+[[package]]
+name = "object"
+version = "0.27.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "67ac1d3f9a1d3616fd9a60c8d74296f22406a238b6a72f5cc1e6f314df4ffbf9"
+dependencies = [
+ "crc32fast",
+ "indexmap",
+ "memchr",
+]
+
++[[package]]
++name = "once_cell"
++version = "1.10.0"
++source = "registry+https://github.com/rust-lang/crates.io-index"
++checksum = "87f3e037eac156d1775da914196f0f37741a274155e34a0b7e427c35d2a2ecb9"
++
+[[package]]
+name = "regalloc"
+version = "0.0.34"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "62446b1d3ebf980bdc68837700af1d77b37bc430e524bf95319c6eada2a4cc02"
+dependencies = [
+ "log",
+ "rustc-hash",
+ "smallvec",
+]
+
+[[package]]
+name = "region"
+version = "2.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "877e54ea2adcd70d80e9179344c97f93ef0dffd6b03e1f4529e6e83ab2fa9ae0"
+dependencies = [
+ "bitflags",
+ "libc",
+ "mach",
+ "winapi",
+]
+
+[[package]]
+name = "rustc-hash"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
+
+[[package]]
+name = "rustc_codegen_cranelift"
+version = "0.1.0"
+dependencies = [
+ "ar",
+ "cranelift-codegen",
+ "cranelift-frontend",
+ "cranelift-jit",
+ "cranelift-module",
+ "cranelift-native",
+ "cranelift-object",
+ "gimli",
+ "indexmap",
+ "libloading",
+ "object",
++ "once_cell",
+ "smallvec",
+ "target-lexicon",
+]
+
+[[package]]
+name = "smallvec"
+version = "1.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f2dd574626839106c320a323308629dcb1acfc96e32a8cba364ddc61ac23ee83"
+
+[[package]]
+name = "target-lexicon"
- checksum = "d9bffcddbc2458fa3e6058414599e3c838a022abae82e5c67b4f7f80298d5bff"
++version = "0.12.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
++checksum = "d7fa7e55043acb85fca6b3c01485a2eeb6b69c5d21002e273c79e465f43b7ac1"
+
+[[package]]
+name = "winapi"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
+dependencies = [
+ "winapi-i686-pc-windows-gnu",
+ "winapi-x86_64-pc-windows-gnu",
+]
+
+[[package]]
+name = "winapi-i686-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
+
+[[package]]
+name = "winapi-x86_64-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
--- /dev/null
- cranelift-codegen = { version = "0.81.0", features = ["unwind", "all-arch"] }
- cranelift-frontend = "0.81.0"
- cranelift-module = "0.81.0"
- cranelift-native = "0.81.0"
- cranelift-jit = { version = "0.81.0", optional = true }
- cranelift-object = "0.81.0"
+[package]
+name = "rustc_codegen_cranelift"
+version = "0.1.0"
+edition = "2021"
+
+[lib]
+crate-type = ["dylib"]
+
+[dependencies]
+# These have to be in sync with each other
++cranelift-codegen = { version = "0.82.1", features = ["unwind", "all-arch"] }
++cranelift-frontend = "0.82.1"
++cranelift-module = "0.82.1"
++cranelift-native = "0.82.1"
++cranelift-jit = { version = "0.82.1", optional = true }
++cranelift-object = "0.82.1"
+target-lexicon = "0.12.0"
+gimli = { version = "0.26.0", default-features = false, features = ["write"]}
+object = { version = "0.27.0", default-features = false, features = ["std", "read_core", "write", "archive", "coff", "elf", "macho", "pe"] }
+
+ar = { git = "https://github.com/bjorn3/rust-ar.git", branch = "do_not_remove_cg_clif_ranlib" }
+indexmap = "1.8.0"
+libloading = { version = "0.6.0", optional = true }
++once_cell = "1.10.0"
+smallvec = "1.6.1"
+
+[patch.crates-io]
+# Uncomment to use local checkout of cranelift
+#cranelift-codegen = { path = "../wasmtime/cranelift/codegen" }
+#cranelift-frontend = { path = "../wasmtime/cranelift/frontend" }
+#cranelift-module = { path = "../wasmtime/cranelift/module" }
+#cranelift-native = { path = "../wasmtime/cranelift/native" }
+#cranelift-jit = { path = "../wasmtime/cranelift/jit" }
+#cranelift-object = { path = "../wasmtime/cranelift/object" }
+
+#gimli = { path = "../" }
+
+[features]
+# Enable features not ready to be enabled when compiling as part of rustc
+unstable-features = ["jit", "inline_asm"]
+jit = ["cranelift-jit", "libloading"]
+inline_asm = []
+
+# Disable optimizations and debuginfo of build scripts and some of the heavy build deps, as the
+# execution time of build scripts is so fast that optimizing them slows down the total build time.
+[profile.release.build-override]
+opt-level = 0
+debug = false
+
+[profile.release.package.cranelift-codegen-meta]
+opt-level = 0
+debug = false
+
+[package.metadata.rust-analyzer]
+rustc_private = true
--- /dev/null
- version = "0.1.70"
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 3
+
+[[package]]
+name = "addr2line"
+version = "0.16.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3e61f2b7f93d2c7d2b08263acaa4a363b3e276806c68af6134c44f523bf1aacd"
+dependencies = [
+ "compiler_builtins",
+ "gimli",
+ "rustc-std-workspace-alloc",
+ "rustc-std-workspace-core",
+]
+
+[[package]]
+name = "adler"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
+dependencies = [
+ "compiler_builtins",
+ "rustc-std-workspace-core",
+]
+
+[[package]]
+name = "alloc"
+version = "0.0.0"
+dependencies = [
+ "compiler_builtins",
+ "core",
+]
+
+[[package]]
+name = "autocfg"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
+
+[[package]]
+name = "cc"
+version = "1.0.73"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11"
+
+[[package]]
+name = "cfg-if"
+version = "0.1.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822"
+dependencies = [
+ "compiler_builtins",
+ "rustc-std-workspace-core",
+]
+
+[[package]]
+name = "compiler_builtins"
- version = "0.2.119"
++version = "0.1.71"
++source = "registry+https://github.com/rust-lang/crates.io-index"
++checksum = "163437f05ca8f29d7e9128ea728dedf5eb620e445fbca273641d3a3050305f23"
+dependencies = [
+ "rustc-std-workspace-core",
+]
+
+[[package]]
+name = "core"
+version = "0.0.0"
+
+[[package]]
+name = "dlmalloc"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a6fe28e0bf9357092740362502f5cc7955d8dc125ebda71dec72336c2e15c62e"
+dependencies = [
+ "compiler_builtins",
+ "libc",
+ "rustc-std-workspace-core",
+]
+
+[[package]]
+name = "fortanix-sgx-abi"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c56c422ef86062869b2d57ae87270608dc5929969dd130a6e248979cf4fb6ca6"
+dependencies = [
+ "compiler_builtins",
+ "rustc-std-workspace-core",
+]
+
+[[package]]
+name = "getopts"
+version = "0.2.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "14dbbfd5c71d70241ecf9e6f13737f7b5ce823821063188d7e46c41d371eebd5"
+dependencies = [
+ "rustc-std-workspace-core",
+ "rustc-std-workspace-std",
+ "unicode-width",
+]
+
+[[package]]
+name = "gimli"
+version = "0.25.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f0a01e0497841a3b2db4f8afa483cce65f7e96a3498bd6c541734792aeac8fe7"
+dependencies = [
+ "compiler_builtins",
+ "rustc-std-workspace-alloc",
+ "rustc-std-workspace-core",
+]
+
+[[package]]
+name = "hashbrown"
+version = "0.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8c21d40587b92fa6a6c6e3c1bdbf87d75511db5672f9c93175574b3a00df1758"
+dependencies = [
+ "compiler_builtins",
+ "rustc-std-workspace-alloc",
+ "rustc-std-workspace-core",
+]
+
+[[package]]
+name = "hermit-abi"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1ab7905ea95c6d9af62940f9d7dd9596d54c334ae2c15300c482051292d5637f"
+dependencies = [
+ "compiler_builtins",
+ "libc",
+ "rustc-std-workspace-core",
+]
+
+[[package]]
+name = "libc"
- checksum = "1bf2e165bb3457c8e098ea76f3e3bc9db55f87aa90d52d0e6be741470916aaa4"
++version = "0.2.121"
+source = "registry+https://github.com/rust-lang/crates.io-index"
++checksum = "efaa7b300f3b5fe8eb6bf21ce3895e1751d9665086af2d64b42f19701015ff4f"
+dependencies = [
+ "rustc-std-workspace-core",
+]
+
+[[package]]
+name = "memchr"
+version = "2.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a"
+dependencies = [
+ "compiler_builtins",
+ "rustc-std-workspace-core",
+]
+
+[[package]]
+name = "miniz_oxide"
+version = "0.4.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b"
+dependencies = [
+ "adler",
+ "autocfg",
+ "compiler_builtins",
+ "rustc-std-workspace-alloc",
+ "rustc-std-workspace-core",
+]
+
+[[package]]
+name = "object"
+version = "0.26.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "39f37e50073ccad23b6d09bcb5b263f4e76d3bb6038e4a3c08e52162ffa8abc2"
+dependencies = [
+ "compiler_builtins",
+ "memchr",
+ "rustc-std-workspace-alloc",
+ "rustc-std-workspace-core",
+]
+
+[[package]]
+name = "panic_abort"
+version = "0.0.0"
+dependencies = [
+ "alloc",
+ "cfg-if",
+ "compiler_builtins",
+ "core",
+ "libc",
+]
+
+[[package]]
+name = "panic_unwind"
+version = "0.0.0"
+dependencies = [
+ "alloc",
+ "cfg-if",
+ "compiler_builtins",
+ "core",
+ "libc",
+ "unwind",
+]
+
+[[package]]
+name = "proc_macro"
+version = "0.0.0"
+dependencies = [
+ "std",
+]
+
+[[package]]
+name = "rustc-demangle"
+version = "0.1.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7ef03e0a2b150c7a90d01faf6254c9c48a41e95fb2a8c2ac1c6f0d2b9aefc342"
+dependencies = [
+ "compiler_builtins",
+ "rustc-std-workspace-core",
+]
+
+[[package]]
+name = "rustc-std-workspace-alloc"
+version = "1.99.0"
+dependencies = [
+ "alloc",
+]
+
+[[package]]
+name = "rustc-std-workspace-core"
+version = "1.99.0"
+dependencies = [
+ "core",
+]
+
+[[package]]
+name = "rustc-std-workspace-std"
+version = "1.99.0"
+dependencies = [
+ "std",
+]
+
+[[package]]
+name = "std"
+version = "0.0.0"
+dependencies = [
+ "addr2line",
+ "alloc",
+ "cfg-if",
+ "compiler_builtins",
+ "core",
+ "dlmalloc",
+ "fortanix-sgx-abi",
+ "hashbrown",
+ "hermit-abi",
+ "libc",
+ "miniz_oxide",
+ "object",
+ "panic_abort",
+ "panic_unwind",
+ "rustc-demangle",
+ "std_detect",
+ "unwind",
+ "wasi",
+]
+
+[[package]]
+name = "std_detect"
+version = "0.1.5"
+dependencies = [
+ "cfg-if",
+ "compiler_builtins",
+ "libc",
+ "rustc-std-workspace-alloc",
+ "rustc-std-workspace-core",
+]
+
+[[package]]
+name = "sysroot"
+version = "0.0.0"
+dependencies = [
+ "alloc",
+ "compiler_builtins",
+ "core",
+ "std",
+ "test",
+]
+
+[[package]]
+name = "test"
+version = "0.0.0"
+dependencies = [
+ "cfg-if",
+ "core",
+ "getopts",
+ "libc",
+ "panic_abort",
+ "panic_unwind",
+ "proc_macro",
+ "std",
+]
+
+[[package]]
+name = "unicode-width"
+version = "0.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973"
+dependencies = [
+ "compiler_builtins",
+ "rustc-std-workspace-core",
+ "rustc-std-workspace-std",
+]
+
+[[package]]
+name = "unwind"
+version = "0.0.0"
+dependencies = [
+ "cc",
+ "cfg-if",
+ "compiler_builtins",
+ "core",
+ "libc",
+]
+
+[[package]]
+name = "wasi"
+version = "0.11.0+wasi-snapshot-preview1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
+dependencies = [
+ "compiler_builtins",
+ "rustc-std-workspace-alloc",
+ "rustc-std-workspace-core",
+]
--- /dev/null
- compiler_builtins = { path = "./compiler-builtins" }
+[package]
+name = "sysroot"
+version = "0.0.0"
+
+[dependencies]
+core = { path = "./sysroot_src/library/core" }
+alloc = { path = "./sysroot_src/library/alloc" }
+std = { path = "./sysroot_src/library/std", features = ["panic_unwind", "backtrace"] }
+test = { path = "./sysroot_src/library/test" }
+
+compiler_builtins = { version = "0.1.39", default-features = false, features = ["no-asm"] }
+
+[patch.crates-io]
+rustc-std-workspace-core = { path = "./sysroot_src/library/rustc-std-workspace-core" }
+rustc-std-workspace-alloc = { path = "./sysroot_src/library/rustc-std-workspace-alloc" }
+rustc-std-workspace-std = { path = "./sysroot_src/library/rustc-std-workspace-std" }
+
+[profile.dev]
+lto = "off"
+
+[profile.release]
+debug = true
+incremental = true
+lto = "off"
++
++# Mandatory for correctly compiling compiler-builtins
++[profile.dev.package.compiler_builtins]
++debug-assertions = false
++overflow-checks = false
++codegen-units = 10000
++
++[profile.release.package.compiler_builtins]
++debug-assertions = false
++overflow-checks = false
++codegen-units = 10000
--- /dev/null
- clone_repo(
+use std::env;
+use std::ffi::OsStr;
+use std::ffi::OsString;
+use std::fs;
+use std::path::Path;
+use std::process::Command;
+
+use super::rustc_info::{get_file_name, get_rustc_path, get_rustc_version};
+use super::utils::{copy_dir_recursively, spawn_and_wait};
+
+pub(crate) fn prepare() {
+ prepare_sysroot();
+
+ eprintln!("[INSTALL] hyperfine");
+ Command::new("cargo").arg("install").arg("hyperfine").spawn().unwrap().wait().unwrap();
+
- "https://github.com/rust-random/rand.git",
++ clone_repo_shallow_github(
++ "rand",
++ "rust-random",
+ "rand",
- clone_repo(
+ "0f933f9c7176e53b2a3c7952ded484e1783f0bf1",
+ );
+ apply_patches("rand", Path::new("rand"));
+
- "https://github.com/rust-lang/regex.git",
++ clone_repo_shallow_github(
++ "regex",
++ "rust-lang",
+ "regex",
- clone_repo(
+ "341f207c1071f7290e3f228c710817c280c8dca1",
+ );
+
- "https://github.com/rust-lang/portable-simd",
++ clone_repo_shallow_github(
++ "portable-simd",
++ "rust-lang",
+ "portable-simd",
- clone_repo(
+ "b8d6b6844602f80af79cd96401339ec594d472d8",
+ );
+ apply_patches("portable-simd", Path::new("portable-simd"));
+
- "https://github.com/ebobby/simple-raytracer",
++ clone_repo_shallow_github(
++ "simple-raytracer",
++ "ebobby",
+ "simple-raytracer",
- let mut git_add_cmd = Command::new("git");
- git_add_cmd.arg("add").arg(".").current_dir(&sysroot_src);
- spawn_and_wait(git_add_cmd);
-
- let mut git_commit_cmd = Command::new("git");
- git_commit_cmd
- .arg("commit")
- .arg("-m")
- .arg("Initial commit")
- .arg("-q")
- .current_dir(&sysroot_src);
- spawn_and_wait(git_commit_cmd);
+ "804a7a21b9e673a482797aa289a18ed480e4d813",
+ );
+
+ eprintln!("[LLVM BUILD] simple-raytracer");
+ let mut build_cmd = Command::new("cargo");
+ build_cmd.arg("build").env_remove("CARGO_TARGET_DIR").current_dir("simple-raytracer");
+ spawn_and_wait(build_cmd);
+ fs::copy(
+ Path::new("simple-raytracer/target/debug").join(get_file_name("main", "bin")),
+ // FIXME use get_file_name here too once testing is migrated to rust
+ "simple-raytracer/raytracer_cg_llvm",
+ )
+ .unwrap();
+}
+
+fn prepare_sysroot() {
+ let rustc_path = get_rustc_path();
+ let sysroot_src_orig = rustc_path.parent().unwrap().join("../lib/rustlib/src/rust");
+ let sysroot_src = env::current_dir().unwrap().join("build_sysroot").join("sysroot_src");
+
+ assert!(sysroot_src_orig.exists());
+
+ if sysroot_src.exists() {
+ fs::remove_dir_all(&sysroot_src).unwrap();
+ }
+ fs::create_dir_all(sysroot_src.join("library")).unwrap();
+ eprintln!("[COPY] sysroot src");
+ copy_dir_recursively(&sysroot_src_orig.join("library"), &sysroot_src.join("library"));
+
+ let rustc_version = get_rustc_version();
+ fs::write(Path::new("build_sysroot").join("rustc_version"), &rustc_version).unwrap();
+
+ eprintln!("[GIT] init");
+ let mut git_init_cmd = Command::new("git");
+ git_init_cmd.arg("init").arg("-q").current_dir(&sysroot_src);
+ spawn_and_wait(git_init_cmd);
+
-
- clone_repo(
- "build_sysroot/compiler-builtins",
- "https://github.com/rust-lang/compiler-builtins.git",
- "0.1.70",
- );
- apply_patches("compiler-builtins", Path::new("build_sysroot/compiler-builtins"));
++ init_git_repo(&sysroot_src);
+
+ apply_patches("sysroot", &sysroot_src);
+}
+
++#[allow(dead_code)]
+fn clone_repo(target_dir: &str, repo: &str, rev: &str) {
+ eprintln!("[CLONE] {}", repo);
+ // Ignore exit code as the repo may already have been checked out
+ Command::new("git").arg("clone").arg(repo).arg(target_dir).spawn().unwrap().wait().unwrap();
+
+ let mut clean_cmd = Command::new("git");
+ clean_cmd.arg("checkout").arg("--").arg(".").current_dir(target_dir);
+ spawn_and_wait(clean_cmd);
+
+ let mut checkout_cmd = Command::new("git");
+ checkout_cmd.arg("checkout").arg("-q").arg(rev).current_dir(target_dir);
+ spawn_and_wait(checkout_cmd);
+}
+
++fn clone_repo_shallow_github(target_dir: &str, username: &str, repo: &str, rev: &str) {
++ if cfg!(windows) {
++ // Older windows doesn't have tar or curl by default. Fall back to using git.
++ clone_repo(target_dir, &format!("https://github.com/{}/{}.git", username, repo), rev);
++ return;
++ }
++
++ let archive_url = format!("https://github.com/{}/{}/archive/{}.tar.gz", username, repo, rev);
++ let archive_file = format!("{}.tar.gz", rev);
++ let archive_dir = format!("{}-{}", repo, rev);
++
++ eprintln!("[DOWNLOAD] {}/{} from {}", username, repo, archive_url);
++
++ // Remove previous results if they exists
++ let _ = std::fs::remove_file(&archive_file);
++ let _ = std::fs::remove_dir_all(&archive_dir);
++ let _ = std::fs::remove_dir_all(target_dir);
++
++ // Download zip archive
++ let mut download_cmd = Command::new("curl");
++ download_cmd.arg("--location").arg("--output").arg(&archive_file).arg(archive_url);
++ spawn_and_wait(download_cmd);
++
++ // Unpack tar archive
++ let mut unpack_cmd = Command::new("tar");
++ unpack_cmd.arg("xf").arg(&archive_file);
++ spawn_and_wait(unpack_cmd);
++
++ // Rename unpacked dir to the expected name
++ std::fs::rename(archive_dir, target_dir).unwrap();
++
++ init_git_repo(Path::new(target_dir));
++
++ // Cleanup
++ std::fs::remove_file(archive_file).unwrap();
++}
++
++fn init_git_repo(repo_dir: &Path) {
++ let mut git_init_cmd = Command::new("git");
++ git_init_cmd.arg("init").arg("-q").current_dir(repo_dir);
++ spawn_and_wait(git_init_cmd);
++
++ let mut git_add_cmd = Command::new("git");
++ git_add_cmd.arg("add").arg(".").current_dir(repo_dir);
++ spawn_and_wait(git_add_cmd);
++
++ let mut git_commit_cmd = Command::new("git");
++ git_commit_cmd.arg("commit").arg("-m").arg("Initial commit").arg("-q").current_dir(repo_dir);
++ spawn_and_wait(git_commit_cmd);
++}
++
+fn get_patches(crate_name: &str) -> Vec<OsString> {
+ let mut patches: Vec<_> = fs::read_dir("patches")
+ .unwrap()
+ .map(|entry| entry.unwrap().path())
+ .filter(|path| path.extension() == Some(OsStr::new("patch")))
+ .map(|path| path.file_name().unwrap().to_owned())
+ .filter(|file_name| {
+ file_name.to_str().unwrap().split_once("-").unwrap().1.starts_with(crate_name)
+ })
+ .collect();
+ patches.sort();
+ patches
+}
+
+fn apply_patches(crate_name: &str, target_dir: &Path) {
+ for patch in get_patches(crate_name) {
+ eprintln!("[PATCH] {:?} <- {:?}", target_dir.file_name().unwrap(), patch);
+ let patch_arg = env::current_dir().unwrap().join("patches").join(patch);
+ let mut apply_patch_cmd = Command::new("git");
+ apply_patch_cmd.arg("am").arg(patch_arg).arg("-q").current_dir(target_dir);
+ spawn_and_wait(apply_patch_cmd);
+ }
+}
--- /dev/null
- #![feature(allocator_api, rustc_private)]
- #![cfg_attr(any(unix, target_os = "redox"), feature(libc))]
-
- // The minimum alignment guaranteed by the architecture. This value is used to
- // add fast paths for low alignment values.
- #[cfg(all(any(target_arch = "x86",
- target_arch = "arm",
- target_arch = "mips",
- target_arch = "powerpc",
- target_arch = "powerpc64")))]
- const MIN_ALIGN: usize = 8;
- #[cfg(all(any(target_arch = "x86_64",
- target_arch = "aarch64",
- target_arch = "mips64",
- target_arch = "s390x",
- target_arch = "sparc64")))]
- const MIN_ALIGN: usize = 16;
+// Copyright 2015 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+#![no_std]
- pub(crate) unsafe fn realloc_fallback(&self, ptr: *mut u8, old_layout: Layout,
- new_size: usize) -> *mut u8 {
+
+pub struct System;
++
+#[cfg(any(windows, unix, target_os = "redox"))]
+mod realloc_fallback {
+ use core::alloc::{GlobalAlloc, Layout};
+ use core::cmp;
+ use core::ptr;
+ impl super::System {
- extern crate libc;
++ pub(crate) unsafe fn realloc_fallback(
++ &self,
++ ptr: *mut u8,
++ old_layout: Layout,
++ new_size: usize,
++ ) -> *mut u8 {
+ // Docs for GlobalAlloc::realloc require this to be valid:
+ let new_layout = Layout::from_size_align_unchecked(new_size, old_layout.align());
+ let new_ptr = GlobalAlloc::alloc(self, new_layout);
+ if !new_ptr.is_null() {
+ let size = cmp::min(old_layout.size(), new_size);
+ ptr::copy_nonoverlapping(ptr, new_ptr, size);
+ GlobalAlloc::dealloc(self, ptr, old_layout);
+ }
+ new_ptr
+ }
+ }
+}
+#[cfg(any(unix, target_os = "redox"))]
+mod platform {
- use MIN_ALIGN;
++ use core::alloc::{GlobalAlloc, Layout};
++ use core::ffi::c_void;
+ use core::ptr;
- use core::alloc::{GlobalAlloc, Layout};
+ use System;
- if layout.align() <= MIN_ALIGN && layout.align() <= layout.size() {
- libc::malloc(layout.size()) as *mut u8
- } else {
- #[cfg(target_os = "macos")]
- {
- if layout.align() > (1 << 31) {
- return ptr::null_mut()
- }
- }
- aligned_malloc(&layout)
- }
++ extern "C" {
++ fn posix_memalign(memptr: *mut *mut c_void, align: usize, size: usize) -> i32;
++ fn free(p: *mut c_void);
++ }
+ unsafe impl GlobalAlloc for System {
+ #[inline]
+ unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
- if layout.align() <= MIN_ALIGN && layout.align() <= layout.size() {
- libc::calloc(layout.size(), 1) as *mut u8
- } else {
- let ptr = self.alloc(layout.clone());
- if !ptr.is_null() {
- ptr::write_bytes(ptr, 0, layout.size());
- }
- ptr
++ aligned_malloc(&layout)
+ }
+ #[inline]
+ unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 {
- libc::free(ptr as *mut libc::c_void)
++ let ptr = self.alloc(layout.clone());
++ if !ptr.is_null() {
++ ptr::write_bytes(ptr, 0, layout.size());
+ }
++ ptr
+ }
+ #[inline]
+ unsafe fn dealloc(&self, ptr: *mut u8, _layout: Layout) {
- if layout.align() <= MIN_ALIGN && layout.align() <= new_size {
- libc::realloc(ptr as *mut libc::c_void, new_size) as *mut u8
- } else {
- self.realloc_fallback(ptr, layout, new_size)
- }
++ free(ptr as *mut c_void)
+ }
+ #[inline]
+ unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 {
- #[cfg(any(target_os = "android",
- target_os = "hermit",
- target_os = "redox",
- target_os = "solaris"))]
- #[inline]
- unsafe fn aligned_malloc(layout: &Layout) -> *mut u8 {
- // On android we currently target API level 9 which unfortunately
- // doesn't have the `posix_memalign` API used below. Instead we use
- // `memalign`, but this unfortunately has the property on some systems
- // where the memory returned cannot be deallocated by `free`!
- //
- // Upon closer inspection, however, this appears to work just fine with
- // Android, so for this platform we should be fine to call `memalign`
- // (which is present in API level 9). Some helpful references could
- // possibly be chromium using memalign [1], attempts at documenting that
- // memalign + free is ok [2] [3], or the current source of chromium
- // which still uses memalign on android [4].
- //
- // [1]: https://codereview.chromium.org/10796020/
- // [2]: https://code.google.com/p/android/issues/detail?id=35391
- // [3]: https://bugs.chromium.org/p/chromium/issues/detail?id=138579
- // [4]: https://chromium.googlesource.com/chromium/src/base/+/master/
- // /memory/aligned_memory.cc
- libc::memalign(layout.align(), layout.size()) as *mut u8
- }
- #[cfg(not(any(target_os = "android",
- target_os = "hermit",
- target_os = "redox",
- target_os = "solaris")))]
- #[inline]
++ self.realloc_fallback(ptr, layout, new_size)
+ }
+ }
- let ret = libc::posix_memalign(&mut out, layout.align(), layout.size());
- if ret != 0 {
- ptr::null_mut()
- } else {
- out as *mut u8
- }
+ unsafe fn aligned_malloc(layout: &Layout) -> *mut u8 {
+ let mut out = ptr::null_mut();
- use MIN_ALIGN;
- use System;
++ let ret = posix_memalign(&mut out, layout.align(), layout.size());
++ if ret != 0 { ptr::null_mut() } else { out as *mut u8 }
+ }
+}
+#[cfg(windows)]
+#[allow(nonstandard_style)]
+mod platform {
- let ptr = if layout.align() <= MIN_ALIGN {
- HeapAlloc(GetProcessHeap(), flags, layout.size())
- } else {
- let size = layout.size() + layout.align();
- let ptr = HeapAlloc(GetProcessHeap(), flags, size);
- if ptr.is_null() {
- ptr
- } else {
- align_ptr(ptr, layout.align())
- }
- };
- ptr as *mut u8
+ use core::alloc::{GlobalAlloc, Layout};
++ use System;
+ type LPVOID = *mut u8;
+ type HANDLE = LPVOID;
+ type SIZE_T = usize;
+ type DWORD = u32;
+ type BOOL = i32;
+ extern "system" {
+ fn GetProcessHeap() -> HANDLE;
+ fn HeapAlloc(hHeap: HANDLE, dwFlags: DWORD, dwBytes: SIZE_T) -> LPVOID;
+ fn HeapReAlloc(hHeap: HANDLE, dwFlags: DWORD, lpMem: LPVOID, dwBytes: SIZE_T) -> LPVOID;
+ fn HeapFree(hHeap: HANDLE, dwFlags: DWORD, lpMem: LPVOID) -> BOOL;
+ fn GetLastError() -> DWORD;
+ }
+ #[repr(C)]
+ struct Header(*mut u8);
+ const HEAP_ZERO_MEMORY: DWORD = 0x00000008;
+ unsafe fn get_header<'a>(ptr: *mut u8) -> &'a mut Header {
+ &mut *(ptr as *mut Header).offset(-1)
+ }
+ unsafe fn align_ptr(ptr: *mut u8, align: usize) -> *mut u8 {
+ let aligned = ptr.add(align - (ptr as usize & (align - 1)));
+ *get_header(aligned) = Header(ptr);
+ aligned
+ }
+ #[inline]
+ unsafe fn allocate_with_flags(layout: Layout, flags: DWORD) -> *mut u8 {
- if layout.align() <= MIN_ALIGN {
- let err = HeapFree(GetProcessHeap(), 0, ptr as LPVOID);
- debug_assert!(err != 0, "Failed to free heap memory: {}",
- GetLastError());
- } else {
- let header = get_header(ptr);
- let err = HeapFree(GetProcessHeap(), 0, header.0 as LPVOID);
- debug_assert!(err != 0, "Failed to free heap memory: {}",
- GetLastError());
- }
++ let size = layout.size() + layout.align();
++ let ptr = HeapAlloc(GetProcessHeap(), flags, size);
++ (if ptr.is_null() { ptr } else { align_ptr(ptr, layout.align()) }) as *mut u8
+ }
+ unsafe impl GlobalAlloc for System {
+ #[inline]
+ unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
+ allocate_with_flags(layout, 0)
+ }
+ #[inline]
+ unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 {
+ allocate_with_flags(layout, HEAP_ZERO_MEMORY)
+ }
+ #[inline]
+ unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) {
- if layout.align() <= MIN_ALIGN {
- HeapReAlloc(GetProcessHeap(), 0, ptr as LPVOID, new_size) as *mut u8
- } else {
- self.realloc_fallback(ptr, layout, new_size)
- }
++ let header = get_header(ptr);
++ let err = HeapFree(GetProcessHeap(), 0, header.0 as LPVOID);
++ debug_assert!(err != 0, "Failed to free heap memory: {}", GetLastError());
+ }
+ #[inline]
+ unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 {
++ self.realloc_fallback(ptr, layout, new_size)
+ }
+ }
+}
--- /dev/null
- #![feature(rustc_attrs)]
+// Adapted from rustc run-pass test suite
+
+#![feature(arbitrary_self_types, unsize, coerce_unsized, dispatch_from_dyn)]
+
+use std::{
+ ops::{Deref, CoerceUnsized, DispatchFromDyn},
+ marker::Unsize,
+};
+
+struct Ptr<T: ?Sized>(Box<T>);
+
+impl<T: ?Sized> Deref for Ptr<T> {
+ type Target = T;
+
+ fn deref(&self) -> &T {
+ &*self.0
+ }
+}
+
+impl<T: Unsize<U> + ?Sized, U: ?Sized> CoerceUnsized<Ptr<U>> for Ptr<T> {}
+impl<T: Unsize<U> + ?Sized, U: ?Sized> DispatchFromDyn<Ptr<U>> for Ptr<T> {}
+
+struct Wrapper<T: ?Sized>(T);
+
+impl<T: ?Sized> Deref for Wrapper<T> {
+ type Target = T;
+
+ fn deref(&self) -> &T {
+ &self.0
+ }
+}
+
+impl<T: CoerceUnsized<U>, U> CoerceUnsized<Wrapper<U>> for Wrapper<T> {}
+impl<T: DispatchFromDyn<U>, U> DispatchFromDyn<Wrapper<U>> for Wrapper<T> {}
+
+
+trait Trait {
+ // This method isn't object-safe yet. Unsized by-value `self` is object-safe (but not callable
+ // without unsized_locals), but wrappers arond `Self` currently are not.
+ // FIXME (mikeyhew) uncomment this when unsized rvalues object-safety is implemented
+ // fn wrapper(self: Wrapper<Self>) -> i32;
+ fn ptr_wrapper(self: Ptr<Wrapper<Self>>) -> i32;
+ fn wrapper_ptr(self: Wrapper<Ptr<Self>>) -> i32;
+ fn wrapper_ptr_wrapper(self: Wrapper<Ptr<Wrapper<Self>>>) -> i32;
+}
+
+impl Trait for i32 {
+ fn ptr_wrapper(self: Ptr<Wrapper<Self>>) -> i32 {
+ **self
+ }
+ fn wrapper_ptr(self: Wrapper<Ptr<Self>>) -> i32 {
+ **self
+ }
+ fn wrapper_ptr_wrapper(self: Wrapper<Ptr<Wrapper<Self>>>) -> i32 {
+ ***self
+ }
+}
+
+fn main() {
+ let pw = Ptr(Box::new(Wrapper(5))) as Ptr<Wrapper<dyn Trait>>;
+ assert_eq!(pw.ptr_wrapper(), 5);
+
+ let wp = Wrapper(Ptr(Box::new(6))) as Wrapper<Ptr<dyn Trait>>;
+ assert_eq!(wp.wrapper_ptr(), 6);
+
+ let wpw = Wrapper(Ptr(Box::new(Wrapper(7)))) as Wrapper<Ptr<Wrapper<dyn Trait>>>;
+ assert_eq!(wpw.wrapper_ptr_wrapper(), 7);
+}
--- /dev/null
- no_core, lang_items, intrinsics, unboxed_closures, type_ascription, extern_types,
- untagged_unions, decl_macro, rustc_attrs, transparent_unions, auto_traits,
- thread_local,
+#![feature(
- pub struct Box<T: ?Sized>(*mut T);
++ no_core,
++ lang_items,
++ intrinsics,
++ unboxed_closures,
++ extern_types,
++ decl_macro,
++ rustc_attrs,
++ transparent_unions,
++ auto_traits,
++ thread_local
+)]
+#![no_core]
+#![allow(dead_code)]
+
+#[lang = "sized"]
+pub trait Sized {}
+
+#[lang = "unsize"]
+pub trait Unsize<T: ?Sized> {}
+
+#[lang = "coerce_unsized"]
+pub trait CoerceUnsized<T> {}
+
+impl<'a, 'b: 'a, T: ?Sized + Unsize<U>, U: ?Sized> CoerceUnsized<&'a U> for &'b T {}
+impl<'a, T: ?Sized + Unsize<U>, U: ?Sized> CoerceUnsized<&'a mut U> for &'a mut T {}
+impl<T: ?Sized + Unsize<U>, U: ?Sized> CoerceUnsized<*const U> for *const T {}
+impl<T: ?Sized + Unsize<U>, U: ?Sized> CoerceUnsized<*mut U> for *mut T {}
+
+#[lang = "dispatch_from_dyn"]
+pub trait DispatchFromDyn<T> {}
+
+// &T -> &U
+impl<'a, T: ?Sized+Unsize<U>, U: ?Sized> DispatchFromDyn<&'a U> for &'a T {}
+// &mut T -> &mut U
+impl<'a, T: ?Sized+Unsize<U>, U: ?Sized> DispatchFromDyn<&'a mut U> for &'a mut T {}
+// *const T -> *const U
+impl<T: ?Sized+Unsize<U>, U: ?Sized> DispatchFromDyn<*const U> for *const T {}
+// *mut T -> *mut U
+impl<T: ?Sized+Unsize<U>, U: ?Sized> DispatchFromDyn<*mut U> for *mut T {}
+impl<T: ?Sized + Unsize<U>, U: ?Sized> DispatchFromDyn<Box<U>> for Box<T> {}
+
+#[lang = "receiver"]
+pub trait Receiver {}
+
+impl<T: ?Sized> Receiver for &T {}
+impl<T: ?Sized> Receiver for &mut T {}
+impl<T: ?Sized> Receiver for Box<T> {}
+
+#[lang = "copy"]
+pub unsafe trait Copy {}
+
+unsafe impl Copy for bool {}
+unsafe impl Copy for u8 {}
+unsafe impl Copy for u16 {}
+unsafe impl Copy for u32 {}
+unsafe impl Copy for u64 {}
+unsafe impl Copy for u128 {}
+unsafe impl Copy for usize {}
+unsafe impl Copy for i8 {}
+unsafe impl Copy for i16 {}
+unsafe impl Copy for i32 {}
+unsafe impl Copy for isize {}
+unsafe impl Copy for f32 {}
++unsafe impl Copy for f64 {}
+unsafe impl Copy for char {}
+unsafe impl<'a, T: ?Sized> Copy for &'a T {}
+unsafe impl<T: ?Sized> Copy for *const T {}
+unsafe impl<T: ?Sized> Copy for *mut T {}
+unsafe impl<T: Copy> Copy for Option<T> {}
+
+#[lang = "sync"]
+pub unsafe trait Sync {}
+
+unsafe impl Sync for bool {}
+unsafe impl Sync for u8 {}
+unsafe impl Sync for u16 {}
+unsafe impl Sync for u32 {}
+unsafe impl Sync for u64 {}
+unsafe impl Sync for usize {}
+unsafe impl Sync for i8 {}
+unsafe impl Sync for i16 {}
+unsafe impl Sync for i32 {}
+unsafe impl Sync for isize {}
+unsafe impl Sync for char {}
+unsafe impl<'a, T: ?Sized> Sync for &'a T {}
+unsafe impl Sync for [u8; 16] {}
+
+#[lang = "freeze"]
+unsafe auto trait Freeze {}
+
+unsafe impl<T: ?Sized> Freeze for PhantomData<T> {}
+unsafe impl<T: ?Sized> Freeze for *const T {}
+unsafe impl<T: ?Sized> Freeze for *mut T {}
+unsafe impl<T: ?Sized> Freeze for &T {}
+unsafe impl<T: ?Sized> Freeze for &mut T {}
+
+#[lang = "structural_peq"]
+pub trait StructuralPartialEq {}
+
+#[lang = "structural_teq"]
+pub trait StructuralEq {}
+
+#[lang = "not"]
+pub trait Not {
+ type Output;
+
+ fn not(self) -> Self::Output;
+}
+
+impl Not for bool {
+ type Output = bool;
+
+ fn not(self) -> bool {
+ !self
+ }
+}
+
+#[lang = "mul"]
+pub trait Mul<RHS = Self> {
+ type Output;
+
+ #[must_use]
+ fn mul(self, rhs: RHS) -> Self::Output;
+}
+
+impl Mul for u8 {
+ type Output = Self;
+
+ fn mul(self, rhs: Self) -> Self::Output {
+ self * rhs
+ }
+}
+
+impl Mul for usize {
+ type Output = Self;
+
+ fn mul(self, rhs: Self) -> Self::Output {
+ self * rhs
+ }
+}
+
+#[lang = "add"]
+pub trait Add<RHS = Self> {
+ type Output;
+
+ fn add(self, rhs: RHS) -> Self::Output;
+}
+
+impl Add for u8 {
+ type Output = Self;
+
+ fn add(self, rhs: Self) -> Self {
+ self + rhs
+ }
+}
+
+impl Add for i8 {
+ type Output = Self;
+
+ fn add(self, rhs: Self) -> Self {
+ self + rhs
+ }
+}
+
+impl Add for usize {
+ type Output = Self;
+
+ fn add(self, rhs: Self) -> Self {
+ self + rhs
+ }
+}
+
+#[lang = "sub"]
+pub trait Sub<RHS = Self> {
+ type Output;
+
+ fn sub(self, rhs: RHS) -> Self::Output;
+}
+
+impl Sub for usize {
+ type Output = Self;
+
+ fn sub(self, rhs: Self) -> Self {
+ self - rhs
+ }
+}
+
+impl Sub for u8 {
+ type Output = Self;
+
+ fn sub(self, rhs: Self) -> Self {
+ self - rhs
+ }
+}
+
+impl Sub for i8 {
+ type Output = Self;
+
+ fn sub(self, rhs: Self) -> Self {
+ self - rhs
+ }
+}
+
+impl Sub for i16 {
+ type Output = Self;
+
+ fn sub(self, rhs: Self) -> Self {
+ self - rhs
+ }
+}
+
+#[lang = "rem"]
+pub trait Rem<RHS = Self> {
+ type Output;
+
+ fn rem(self, rhs: RHS) -> Self::Output;
+}
+
+impl Rem for usize {
+ type Output = Self;
+
+ fn rem(self, rhs: Self) -> Self {
+ self % rhs
+ }
+}
+
+#[lang = "bitor"]
+pub trait BitOr<RHS = Self> {
+ type Output;
+
+ #[must_use]
+ fn bitor(self, rhs: RHS) -> Self::Output;
+}
+
+impl BitOr for bool {
+ type Output = bool;
+
+ fn bitor(self, rhs: bool) -> bool {
+ self | rhs
+ }
+}
+
+impl<'a> BitOr<bool> for &'a bool {
+ type Output = bool;
+
+ fn bitor(self, rhs: bool) -> bool {
+ *self | rhs
+ }
+}
+
+#[lang = "eq"]
+pub trait PartialEq<Rhs: ?Sized = Self> {
+ fn eq(&self, other: &Rhs) -> bool;
+ fn ne(&self, other: &Rhs) -> bool;
+}
+
+impl PartialEq for u8 {
+ fn eq(&self, other: &u8) -> bool {
+ (*self) == (*other)
+ }
+ fn ne(&self, other: &u8) -> bool {
+ (*self) != (*other)
+ }
+}
+
+impl PartialEq for u16 {
+ fn eq(&self, other: &u16) -> bool {
+ (*self) == (*other)
+ }
+ fn ne(&self, other: &u16) -> bool {
+ (*self) != (*other)
+ }
+}
+
+impl PartialEq for u32 {
+ fn eq(&self, other: &u32) -> bool {
+ (*self) == (*other)
+ }
+ fn ne(&self, other: &u32) -> bool {
+ (*self) != (*other)
+ }
+}
+
+
+impl PartialEq for u64 {
+ fn eq(&self, other: &u64) -> bool {
+ (*self) == (*other)
+ }
+ fn ne(&self, other: &u64) -> bool {
+ (*self) != (*other)
+ }
+}
+
+impl PartialEq for u128 {
+ fn eq(&self, other: &u128) -> bool {
+ (*self) == (*other)
+ }
+ fn ne(&self, other: &u128) -> bool {
+ (*self) != (*other)
+ }
+}
+
+impl PartialEq for usize {
+ fn eq(&self, other: &usize) -> bool {
+ (*self) == (*other)
+ }
+ fn ne(&self, other: &usize) -> bool {
+ (*self) != (*other)
+ }
+}
+
+impl PartialEq for i8 {
+ fn eq(&self, other: &i8) -> bool {
+ (*self) == (*other)
+ }
+ fn ne(&self, other: &i8) -> bool {
+ (*self) != (*other)
+ }
+}
+
+impl PartialEq for i32 {
+ fn eq(&self, other: &i32) -> bool {
+ (*self) == (*other)
+ }
+ fn ne(&self, other: &i32) -> bool {
+ (*self) != (*other)
+ }
+}
+
+impl PartialEq for isize {
+ fn eq(&self, other: &isize) -> bool {
+ (*self) == (*other)
+ }
+ fn ne(&self, other: &isize) -> bool {
+ (*self) != (*other)
+ }
+}
+
+impl PartialEq for char {
+ fn eq(&self, other: &char) -> bool {
+ (*self) == (*other)
+ }
+ fn ne(&self, other: &char) -> bool {
+ (*self) != (*other)
+ }
+}
+
+impl<T: ?Sized> PartialEq for *const T {
+ fn eq(&self, other: &*const T) -> bool {
+ *self == *other
+ }
+ fn ne(&self, other: &*const T) -> bool {
+ *self != *other
+ }
+}
+
+impl <T: PartialEq> PartialEq for Option<T> {
+ fn eq(&self, other: &Self) -> bool {
+ match (self, other) {
+ (Some(lhs), Some(rhs)) => *lhs == *rhs,
+ (None, None) => true,
+ _ => false,
+ }
+ }
+
+ fn ne(&self, other: &Self) -> bool {
+ match (self, other) {
+ (Some(lhs), Some(rhs)) => *lhs != *rhs,
+ (None, None) => false,
+ _ => true,
+ }
+ }
+}
+
+#[lang = "shl"]
+pub trait Shl<RHS = Self> {
+ type Output;
+
+ #[must_use]
+ fn shl(self, rhs: RHS) -> Self::Output;
+}
+
+impl Shl for u128 {
+ type Output = u128;
+
+ fn shl(self, rhs: u128) -> u128 {
+ self << rhs
+ }
+}
+
+#[lang = "neg"]
+pub trait Neg {
+ type Output;
+
+ fn neg(self) -> Self::Output;
+}
+
+impl Neg for i8 {
+ type Output = i8;
+
+ fn neg(self) -> i8 {
+ -self
+ }
+}
+
+impl Neg for i16 {
+ type Output = i16;
+
+ fn neg(self) -> i16 {
+ self
+ }
+}
+
+impl Neg for isize {
+ type Output = isize;
+
+ fn neg(self) -> isize {
+ -self
+ }
+}
+
+impl Neg for f32 {
+ type Output = f32;
+
+ fn neg(self) -> f32 {
+ -self
+ }
+}
+
+pub enum Option<T> {
+ Some(T),
+ None,
+}
+
+pub use Option::*;
+
+#[lang = "phantom_data"]
+pub struct PhantomData<T: ?Sized>;
+
+#[lang = "fn_once"]
+#[rustc_paren_sugar]
+pub trait FnOnce<Args> {
+ #[lang = "fn_once_output"]
+ type Output;
+
+ extern "rust-call" fn call_once(self, args: Args) -> Self::Output;
+}
+
+#[lang = "fn_mut"]
+#[rustc_paren_sugar]
+pub trait FnMut<Args>: FnOnce<Args> {
+ extern "rust-call" fn call_mut(&mut self, args: Args) -> Self::Output;
+}
+
+#[lang = "panic"]
+#[track_caller]
+pub fn panic(_msg: &str) -> ! {
+ unsafe {
+ libc::puts("Panicking\n\0" as *const str as *const i8);
+ intrinsics::abort();
+ }
+}
+
+#[lang = "panic_bounds_check"]
+#[track_caller]
+fn panic_bounds_check(index: usize, len: usize) -> ! {
+ unsafe {
+ libc::printf("index out of bounds: the len is %d but the index is %d\n\0" as *const str as *const i8, len, index);
+ intrinsics::abort();
+ }
+}
+
+#[lang = "eh_personality"]
+fn eh_personality() -> ! {
+ loop {}
+}
+
+#[lang = "drop_in_place"]
+#[allow(unconditional_recursion)]
+pub unsafe fn drop_in_place<T: ?Sized>(to_drop: *mut T) {
+ // Code here does not matter - this is replaced by the
+ // real drop glue by the compiler.
+ drop_in_place(to_drop);
+}
+
+#[lang = "deref"]
+pub trait Deref {
+ type Target: ?Sized;
+
+ fn deref(&self) -> &Self::Target;
+}
+
++pub struct Unique<T: ?Sized> {
++ pub pointer: *const T,
++ pub _marker: PhantomData<T>,
++}
++
++impl<T: ?Sized, U: ?Sized> CoerceUnsized<Unique<U>> for Unique<T> where T: Unsize<U> {}
++
++impl<T: ?Sized, U: ?Sized> DispatchFromDyn<Unique<U>> for Unique<T> where T: Unsize<U> {}
++
+#[lang = "owned_box"]
- unsafe fn box_free<T: ?Sized>(ptr: *mut T) {
- libc::free(ptr as *mut u8);
++pub struct Box<T: ?Sized>(Unique<T>, ());
+
+impl<T: ?Sized + Unsize<U>, U: ?Sized> CoerceUnsized<Box<U>> for Box<T> {}
+
+impl<T: ?Sized> Drop for Box<T> {
+ fn drop(&mut self) {
+ // drop is currently performed by compiler.
+ }
+}
+
+impl<T> Deref for Box<T> {
+ type Target = T;
+
+ fn deref(&self) -> &Self::Target {
+ &**self
+ }
+}
+
+#[lang = "exchange_malloc"]
+unsafe fn allocate(size: usize, _align: usize) -> *mut u8 {
+ libc::malloc(size)
+}
+
+#[lang = "box_free"]
++unsafe fn box_free<T: ?Sized>(ptr: Unique<T>, alloc: ()) {
++ libc::free(ptr.pointer as *mut u8);
+}
+
+#[lang = "drop"]
+pub trait Drop {
+ fn drop(&mut self);
+}
+
+#[lang = "manually_drop"]
+#[repr(transparent)]
+pub struct ManuallyDrop<T: ?Sized> {
+ pub value: T,
+}
+
+#[lang = "maybe_uninit"]
+#[repr(transparent)]
+pub union MaybeUninit<T> {
+ pub uninit: (),
+ pub value: ManuallyDrop<T>,
+}
+
+pub mod intrinsics {
+ extern "rust-intrinsic" {
+ pub fn abort() -> !;
+ pub fn size_of<T>() -> usize;
+ pub fn size_of_val<T: ?::Sized>(val: *const T) -> usize;
+ pub fn min_align_of<T>() -> usize;
+ pub fn min_align_of_val<T: ?::Sized>(val: *const T) -> usize;
+ pub fn copy<T>(src: *const T, dst: *mut T, count: usize);
+ pub fn transmute<T, U>(e: T) -> U;
+ pub fn ctlz_nonzero<T>(x: T) -> T;
+ pub fn needs_drop<T>() -> bool;
+ pub fn bitreverse<T>(x: T) -> T;
+ pub fn bswap<T>(x: T) -> T;
+ pub fn write_bytes<T>(dst: *mut T, val: u8, count: usize);
+ }
+}
+
+pub mod libc {
+ #[cfg_attr(unix, link(name = "c"))]
+ #[cfg_attr(target_env = "msvc", link(name = "msvcrt"))]
+ extern "C" {
+ pub fn puts(s: *const i8) -> i32;
+ pub fn printf(format: *const i8, ...) -> i32;
+ pub fn malloc(size: usize) -> *mut u8;
+ pub fn free(ptr: *mut u8);
+ pub fn memcpy(dst: *mut u8, src: *const u8, size: usize);
+ pub fn memmove(dst: *mut u8, src: *const u8, size: usize);
+ pub fn strncpy(dst: *mut u8, src: *const u8, size: usize);
+ }
+}
+
+#[lang = "index"]
+pub trait Index<Idx: ?Sized> {
+ type Output: ?Sized;
+ fn index(&self, index: Idx) -> &Self::Output;
+}
+
+impl<T> Index<usize> for [T; 3] {
+ type Output = T;
+
+ fn index(&self, index: usize) -> &Self::Output {
+ &self[index]
+ }
+}
+
+impl<T> Index<usize> for [T] {
+ type Output = T;
+
+ fn index(&self, index: usize) -> &Self::Output {
+ &self[index]
+ }
+}
+
+extern {
+ type VaListImpl;
+}
+
+#[lang = "va_list"]
+#[repr(transparent)]
+pub struct VaList<'a>(&'a mut VaListImpl);
+
+#[rustc_builtin_macro]
+#[rustc_macro_transparency = "semitransparent"]
+pub macro stringify($($t:tt)*) { /* compiler built-in */ }
+
+#[rustc_builtin_macro]
+#[rustc_macro_transparency = "semitransparent"]
+pub macro file() { /* compiler built-in */ }
+
+#[rustc_builtin_macro]
+#[rustc_macro_transparency = "semitransparent"]
+pub macro line() { /* compiler built-in */ }
+
+#[rustc_builtin_macro]
+#[rustc_macro_transparency = "semitransparent"]
+pub macro cfg() { /* compiler built-in */ }
+
+#[rustc_builtin_macro]
+#[rustc_macro_transparency = "semitransparent"]
+pub macro global_asm() { /* compiler built-in */ }
+
+pub static A_STATIC: u8 = 42;
+
+#[lang = "panic_location"]
+struct PanicLocation {
+ file: &'static str,
+ line: u32,
+ column: u32,
+}
+
+#[no_mangle]
+#[cfg(not(windows))]
+pub fn get_tls() -> u8 {
+ #[thread_local]
+ static A: u8 = 42;
+
+ A
+}
--- /dev/null
- unsafe extern "C" fn my_puts(s: *const i8) {
- puts(s);
- }
-
+#![feature(no_core, lang_items, never_type, linkage, extern_types, thread_local, box_syntax)]
+#![no_core]
+#![allow(dead_code, non_camel_case_types)]
+
+extern crate mini_core;
+
+use mini_core::*;
+use mini_core::libc::*;
+
- struct Unique<T: ?Sized> {
- pointer: *const T,
- _marker: PhantomData<T>,
- }
-
- impl<T: ?Sized, U: ?Sized> CoerceUnsized<Unique<U>> for Unique<T> where T: Unsize<U> {}
+macro_rules! assert {
+ ($e:expr) => {
+ if !$e {
+ panic(stringify!(! $e));
+ }
+ };
+}
+
+macro_rules! assert_eq {
+ ($l:expr, $r: expr) => {
+ if $l != $r {
+ panic(stringify!($l != $r));
+ }
+ }
+}
+
+#[lang = "termination"]
+trait Termination {
+ fn report(self) -> i32;
+}
+
+impl Termination for () {
+ fn report(self) -> i32 {
+ unsafe {
+ NUM = 6 * 7 + 1 + (1u8 == 1u8) as u8; // 44
+ assert_eq!(*NUM_REF as i32, 44);
+ }
+ 0
+ }
+}
+
+trait SomeTrait {
+ fn object_safe(&self);
+}
+
+impl SomeTrait for &'static str {
+ fn object_safe(&self) {
+ unsafe {
+ puts(*self as *const str as *const i8);
+ }
+ }
+}
+
+struct NoisyDrop {
+ text: &'static str,
+ inner: NoisyDropInner,
+}
+
+struct NoisyDropInner;
+
+impl Drop for NoisyDrop {
+ fn drop(&mut self) {
+ unsafe {
+ puts(self.text as *const str as *const i8);
+ }
+ }
+}
+
+impl Drop for NoisyDropInner {
+ fn drop(&mut self) {
+ unsafe {
+ puts("Inner got dropped!\0" as *const str as *const i8);
+ }
+ }
+}
+
+impl SomeTrait for NoisyDrop {
+ fn object_safe(&self) {}
+}
+
+enum Ordering {
+ Less = -1,
+ Equal = 0,
+ Greater = 1,
+}
+
+#[lang = "start"]
+fn start<T: Termination + 'static>(
+ main: fn() -> T,
+ argc: isize,
+ argv: *const *const u8,
+) -> isize {
+ if argc == 3 {
+ unsafe { puts(*argv as *const i8); }
+ unsafe { puts(*((argv as usize + intrinsics::size_of::<*const u8>()) as *const *const i8)); }
+ unsafe { puts(*((argv as usize + 2 * intrinsics::size_of::<*const u8>()) as *const *const i8)); }
+ }
+
+ main().report() as isize
+}
+
+static mut NUM: u8 = 6 * 7;
+static NUM_REF: &'static u8 = unsafe { &NUM };
+
+
+unsafe fn zeroed<T>() -> T {
+ let mut uninit = MaybeUninit { uninit: () };
+ intrinsics::write_bytes(&mut uninit.value.value as *mut T, 0, 1);
+ uninit.value.value
+}
+
+fn take_f32(_f: f32) {}
+fn take_unique(_u: Unique<()>) {}
+
+fn return_u128_pair() -> (u128, u128) {
+ (0, 0)
+}
+
+fn call_return_u128_pair() {
+ return_u128_pair();
+}
+
+#[allow(unreachable_code)] // FIXME false positive
+fn main() {
+ take_unique(Unique {
+ pointer: 0 as *const (),
+ _marker: PhantomData,
+ });
+ take_f32(0.1);
+
+ call_return_u128_pair();
+
+ let slice = &[0, 1] as &[i32];
+ let slice_ptr = slice as *const [i32] as *const i32;
+
+ assert_eq!(slice_ptr as usize % 4, 0);
+
+ //return;
+
+ unsafe {
+ printf("Hello %s\n\0" as *const str as *const i8, "printf\0" as *const str as *const i8);
+
+ let hello: &[u8] = b"Hello\0" as &[u8; 6];
+ let ptr: *const i8 = hello as *const [u8] as *const i8;
+ puts(ptr);
+
+ let world: Box<&str> = box "World!\0";
+ puts(*world as *const str as *const i8);
+ world as Box<dyn SomeTrait>;
+
+ assert_eq!(intrinsics::bitreverse(0b10101000u8), 0b00010101u8);
+
+ assert_eq!(intrinsics::bswap(0xabu8), 0xabu8);
+ assert_eq!(intrinsics::bswap(0xddccu16), 0xccddu16);
+ assert_eq!(intrinsics::bswap(0xffee_ddccu32), 0xccdd_eeffu32);
+ assert_eq!(intrinsics::bswap(0x1234_5678_ffee_ddccu64), 0xccdd_eeff_7856_3412u64);
+
+ assert_eq!(intrinsics::size_of_val(hello) as u8, 6);
+
+ let chars = &['C', 'h', 'a', 'r', 's'];
+ let chars = chars as &[char];
+ assert_eq!(intrinsics::size_of_val(chars) as u8, 4 * 5);
+
+ let a: &dyn SomeTrait = &"abc\0";
+ a.object_safe();
+
+ assert_eq!(intrinsics::size_of_val(a) as u8, 16);
+ assert_eq!(intrinsics::size_of_val(&0u32) as u8, 4);
+
+ assert_eq!(intrinsics::min_align_of::<u16>() as u8, 2);
+ assert_eq!(intrinsics::min_align_of_val(&a) as u8, intrinsics::min_align_of::<&str>() as u8);
+
+ assert!(!intrinsics::needs_drop::<u8>());
+ assert!(intrinsics::needs_drop::<NoisyDrop>());
+
+ Unique {
+ pointer: 0 as *const &str,
+ _marker: PhantomData,
+ } as Unique<dyn SomeTrait>;
+
+ struct MyDst<T: ?Sized>(T);
+
+ intrinsics::size_of_val(&MyDst([0u8; 4]) as &MyDst<[u8]>);
+
+ struct Foo {
+ x: u8,
+ y: !,
+ }
+
+ unsafe fn uninitialized<T>() -> T {
+ MaybeUninit { uninit: () }.value.value
+ }
+
+ zeroed::<(u8, u8)>();
+ #[allow(unreachable_code)]
+ {
+ if false {
+ zeroed::<!>();
+ zeroed::<Foo>();
+ uninitialized::<Foo>();
+ }
+ }
+ }
+
+ let _ = box NoisyDrop {
+ text: "Boxed outer got dropped!\0",
+ inner: NoisyDropInner,
+ } as Box<dyn SomeTrait>;
+
+ const FUNC_REF: Option<fn()> = Some(main);
+ match FUNC_REF {
+ Some(_) => {},
+ None => assert!(false),
+ }
+
+ match Ordering::Less {
+ Ordering::Less => {},
+ _ => assert!(false),
+ }
+
+ [NoisyDropInner, NoisyDropInner];
+
+ let x = &[0u32, 42u32] as &[u32];
+ match x {
+ [] => assert_eq!(0u32, 1),
+ [_, ref y @ ..] => assert_eq!(&x[1] as *const u32 as usize, &y[0] as *const u32 as usize),
+ }
+
+ assert_eq!(((|()| 42u8) as fn(()) -> u8)(()), 42);
+
+ #[cfg(not(any(jit, windows)))]
+ {
+ extern {
+ #[linkage = "extern_weak"]
+ static ABC: *const u8;
+ }
+
+ {
+ extern {
+ #[linkage = "extern_weak"]
+ static ABC: *const u8;
+ }
+ }
+
+ unsafe { assert_eq!(ABC as usize, 0); }
+ }
+
+ &mut (|| Some(0 as *const ())) as &mut dyn FnMut() -> Option<*const ()>;
+
+ let f = 1000.0;
+ assert_eq!(f as u8, 255);
+ let f2 = -1000.0;
+ assert_eq!(f2 as i8, -128);
+ assert_eq!(f2 as u8, 0);
+
+ let amount = 0;
+ assert_eq!(1u128 << amount, 1);
+
+ static ANOTHER_STATIC: &u8 = &A_STATIC;
+ assert_eq!(*ANOTHER_STATIC, 42);
+
+ check_niche_behavior();
+
+ extern "C" {
+ type ExternType;
+ }
+
+ struct ExternTypeWrapper {
+ _a: ExternType,
+ }
+
+ let nullptr = 0 as *const ();
+ let extern_nullptr = nullptr as *const ExternTypeWrapper;
+ extern_nullptr as *const ();
+ let slice_ptr = &[] as *const [u8];
+ slice_ptr as *const u8;
+
+ let repeat = [Some(42); 2];
+ assert_eq!(repeat[0], Some(42));
+ assert_eq!(repeat[1], Some(42));
+
+ from_decimal_string();
+
+ #[cfg(not(any(jit, windows)))]
+ test_tls();
+
+ #[cfg(all(not(jit), target_arch = "x86_64", target_os = "linux"))]
+ unsafe {
+ global_asm_test();
+ }
+
+ // Both statics have a reference that points to the same anonymous allocation.
+ static REF1: &u8 = &42;
+ static REF2: &u8 = REF1;
+ assert_eq!(*REF1, *REF2);
+}
+
+#[cfg(all(not(jit), target_arch = "x86_64", target_os = "linux"))]
+extern "C" {
+ fn global_asm_test();
+}
+
+#[cfg(all(not(jit), target_arch = "x86_64", target_os = "linux"))]
+global_asm! {
+ "
+ .global global_asm_test
+ global_asm_test:
+ // comment that would normally be removed by LLVM
+ ret
+ "
+}
+
+#[repr(C)]
+enum c_void {
+ _1,
+ _2,
+}
+
+type c_int = i32;
+type c_ulong = u64;
+
+type pthread_t = c_ulong;
+
+#[repr(C)]
+struct pthread_attr_t {
+ __size: [u64; 7],
+}
+
+#[link(name = "pthread")]
+extern "C" {
+ fn pthread_attr_init(attr: *mut pthread_attr_t) -> c_int;
+
+ fn pthread_create(
+ native: *mut pthread_t,
+ attr: *const pthread_attr_t,
+ f: extern "C" fn(_: *mut c_void) -> *mut c_void,
+ value: *mut c_void
+ ) -> c_int;
+
+ fn pthread_join(
+ native: pthread_t,
+ value: *mut *mut c_void
+ ) -> c_int;
+}
+
+#[thread_local]
+#[cfg(not(jit))]
+static mut TLS: u8 = 42;
+
+#[cfg(not(jit))]
+extern "C" fn mutate_tls(_: *mut c_void) -> *mut c_void {
+ unsafe { TLS = 0; }
+ 0 as *mut c_void
+}
+
+#[cfg(not(jit))]
+fn test_tls() {
+ unsafe {
+ let mut attr: pthread_attr_t = zeroed();
+ let mut thread: pthread_t = 0;
+
+ assert_eq!(TLS, 42);
+
+ if pthread_attr_init(&mut attr) != 0 {
+ assert!(false);
+ }
+
+ if pthread_create(&mut thread, &attr, mutate_tls, 0 as *mut c_void) != 0 {
+ assert!(false);
+ }
+
+ let mut res = 0 as *mut c_void;
+ pthread_join(thread, &mut res);
+
+ // TLS of main thread must not have been changed by the other thread.
+ assert_eq!(TLS, 42);
+
+ puts("TLS works!\n\0" as *const str as *const i8);
+ }
+}
+
+// Copied ui/issues/issue-61696.rs
+
+pub enum Infallible {}
+
+// The check that the `bool` field of `V1` is encoding a "niche variant"
+// (i.e. not `V1`, so `V3` or `V4`) used to be mathematically incorrect,
+// causing valid `V1` values to be interpreted as other variants.
+pub enum E1 {
+ V1 { f: bool },
+ V2 { f: Infallible },
+ V3,
+ V4,
+}
+
+// Computing the discriminant used to be done using the niche type (here `u8`,
+// from the `bool` field of `V1`), overflowing for variants with large enough
+// indices (`V3` and `V4`), causing them to be interpreted as other variants.
+pub enum E2<X> {
+ V1 { f: bool },
+
+ /*_00*/ _01(X), _02(X), _03(X), _04(X), _05(X), _06(X), _07(X),
+ _08(X), _09(X), _0A(X), _0B(X), _0C(X), _0D(X), _0E(X), _0F(X),
+ _10(X), _11(X), _12(X), _13(X), _14(X), _15(X), _16(X), _17(X),
+ _18(X), _19(X), _1A(X), _1B(X), _1C(X), _1D(X), _1E(X), _1F(X),
+ _20(X), _21(X), _22(X), _23(X), _24(X), _25(X), _26(X), _27(X),
+ _28(X), _29(X), _2A(X), _2B(X), _2C(X), _2D(X), _2E(X), _2F(X),
+ _30(X), _31(X), _32(X), _33(X), _34(X), _35(X), _36(X), _37(X),
+ _38(X), _39(X), _3A(X), _3B(X), _3C(X), _3D(X), _3E(X), _3F(X),
+ _40(X), _41(X), _42(X), _43(X), _44(X), _45(X), _46(X), _47(X),
+ _48(X), _49(X), _4A(X), _4B(X), _4C(X), _4D(X), _4E(X), _4F(X),
+ _50(X), _51(X), _52(X), _53(X), _54(X), _55(X), _56(X), _57(X),
+ _58(X), _59(X), _5A(X), _5B(X), _5C(X), _5D(X), _5E(X), _5F(X),
+ _60(X), _61(X), _62(X), _63(X), _64(X), _65(X), _66(X), _67(X),
+ _68(X), _69(X), _6A(X), _6B(X), _6C(X), _6D(X), _6E(X), _6F(X),
+ _70(X), _71(X), _72(X), _73(X), _74(X), _75(X), _76(X), _77(X),
+ _78(X), _79(X), _7A(X), _7B(X), _7C(X), _7D(X), _7E(X), _7F(X),
+ _80(X), _81(X), _82(X), _83(X), _84(X), _85(X), _86(X), _87(X),
+ _88(X), _89(X), _8A(X), _8B(X), _8C(X), _8D(X), _8E(X), _8F(X),
+ _90(X), _91(X), _92(X), _93(X), _94(X), _95(X), _96(X), _97(X),
+ _98(X), _99(X), _9A(X), _9B(X), _9C(X), _9D(X), _9E(X), _9F(X),
+ _A0(X), _A1(X), _A2(X), _A3(X), _A4(X), _A5(X), _A6(X), _A7(X),
+ _A8(X), _A9(X), _AA(X), _AB(X), _AC(X), _AD(X), _AE(X), _AF(X),
+ _B0(X), _B1(X), _B2(X), _B3(X), _B4(X), _B5(X), _B6(X), _B7(X),
+ _B8(X), _B9(X), _BA(X), _BB(X), _BC(X), _BD(X), _BE(X), _BF(X),
+ _C0(X), _C1(X), _C2(X), _C3(X), _C4(X), _C5(X), _C6(X), _C7(X),
+ _C8(X), _C9(X), _CA(X), _CB(X), _CC(X), _CD(X), _CE(X), _CF(X),
+ _D0(X), _D1(X), _D2(X), _D3(X), _D4(X), _D5(X), _D6(X), _D7(X),
+ _D8(X), _D9(X), _DA(X), _DB(X), _DC(X), _DD(X), _DE(X), _DF(X),
+ _E0(X), _E1(X), _E2(X), _E3(X), _E4(X), _E5(X), _E6(X), _E7(X),
+ _E8(X), _E9(X), _EA(X), _EB(X), _EC(X), _ED(X), _EE(X), _EF(X),
+ _F0(X), _F1(X), _F2(X), _F3(X), _F4(X), _F5(X), _F6(X), _F7(X),
+ _F8(X), _F9(X), _FA(X), _FB(X), _FC(X), _FD(X), _FE(X), _FF(X),
+
+ V3,
+ V4,
+}
+
+fn check_niche_behavior () {
+ if let E1::V2 { .. } = (E1::V1 { f: true }) {
+ intrinsics::abort();
+ }
+
+ if let E2::V1 { .. } = E2::V3::<Infallible> {
+ intrinsics::abort();
+ }
+}
+
+fn from_decimal_string() {
+ loop {
+ let multiplier = 1;
+
+ take_multiplier_ref(&multiplier);
+
+ if multiplier == 1 {
+ break;
+ }
+
+ unreachable();
+ }
+}
+
+fn take_multiplier_ref(_multiplier: &u128) {}
+
+fn unreachable() -> ! {
+ panic("unreachable")
+}
--- /dev/null
- src/distributions/uniform.rs | 3 ++-
- 1 file changed, 2 insertions(+), 1 deletion(-)
+From a8fb97120d71252538b6b026695df40d02696bdb Mon Sep 17 00:00:00 2001
+From: bjorn3 <bjorn3@users.noreply.github.com>
+Date: Sat, 15 Aug 2020 20:04:38 +0200
+Subject: [PATCH] [rand] Disable failing test
+
+---
- @@ -1085,7 +1085,7 @@ mod tests {
- _ => panic!("`UniformDurationMode` was not serialized/deserialized correctly")
- }
- }
- -
- +
- #[test]
- #[cfg(feature = "serde1")]
- fn test_uniform_serialization() {
++ src/distributions/uniform.rs | 1 +
++ 1 file changed, 1 insertion(+), 0 deletions(-)
+
+diff --git a/src/distributions/uniform.rs b/src/distributions/uniform.rs
+index 480b859..c80bb6f 100644
+--- a/src/distributions/uniform.rs
++++ b/src/distributions/uniform.rs
- + #[ignore] // FIXME
+@@ -1314,6 +1314,7 @@ mod tests {
+ not(target_arch = "wasm32"),
+ not(target_arch = "asmjs")
+ ))]
+++ #[ignore] // Requires unwinding
+ fn test_float_assertions() {
+ use super::SampleUniform;
+ use std::panic::catch_unwind;
+--
+2.20.1
+
--- /dev/null
- @@ -0,0 +1,8 @@
+From f6befc4bb51d84f5f1cf35938a168c953d421350 Mon Sep 17 00:00:00 2001
+From: bjorn3 <bjorn3@users.noreply.github.com>
+Date: Sun, 24 Nov 2019 15:10:23 +0100
+Subject: [PATCH] [core] Disable not compiling tests
+
+---
+ library/core/tests/Cargo.toml | 8 ++++++++
+ library/core/tests/num/flt2dec/mod.rs | 1 -
+ library/core/tests/num/int_macros.rs | 2 ++
+ library/core/tests/num/uint_macros.rs | 2 ++
+ library/core/tests/ptr.rs | 2 ++
+ library/core/tests/slice.rs | 2 ++
+ 6 files changed, 16 insertions(+), 1 deletion(-)
+ create mode 100644 library/core/tests/Cargo.toml
+
+diff --git a/library/core/tests/Cargo.toml b/library/core/tests/Cargo.toml
+new file mode 100644
+index 0000000..46fd999
+--- /dev/null
++++ b/library/core/tests/Cargo.toml
- diff --git a/library/core/tests/num/flt2dec/mod.rs b/library/core/tests/num/flt2dec/mod.rs
- index a35897e..f0bf645 100644
- --- a/library/core/tests/num/flt2dec/mod.rs
- +++ b/library/core/tests/num/flt2dec/mod.rs
- @@ -13,7 +13,6 @@ mod strategy {
- mod dragon;
- mod grisu;
- }
- -mod random;
-
- pub fn decode_finite<T: DecodableFloat>(v: T) -> Decoded {
- match decode(v).1 {
++@@ -0,0 +1,11 @@
++[package]
++name = "core"
++version = "0.0.0"
++edition = "2021"
++
++[lib]
++name = "coretests"
++path = "lib.rs"
- diff --git a/library/core/tests/slice.rs b/library/core/tests/slice.rs
- index 6609bc3..241b497 100644
- --- a/library/core/tests/slice.rs
- +++ b/library/core/tests/slice.rs
- @@ -1209,6 +1209,7 @@ fn brute_force_rotate_test_1() {
- }
- }
-
- +/*
- #[test]
- #[cfg(not(target_arch = "wasm32"))]
- fn sort_unstable() {
- @@ -1394,6 +1395,7 @@ fn partition_at_index() {
- v.select_nth_unstable(0);
- assert!(v == [0xDEADBEEF]);
- }
- +*/
-
- #[test]
- #[should_panic(expected = "index 0 greater than length of slice")]
+++
+++[dependencies]
+++rand = "0.7"
+diff --git a/library/core/tests/ptr.rs b/library/core/tests/ptr.rs
+index 1a6be3a..42dbd59 100644
+--- a/library/core/tests/ptr.rs
++++ b/library/core/tests/ptr.rs
+@@ -250,6 +250,7 @@ fn test_unsized_nonnull() {
+ };
+ }
+
++/*
+ #[test]
+ #[allow(warnings)]
+ // Have a symbol for the test below. It doesn’t need to be an actual variadic function, match the
+@@ -277,6 +277,7 @@ pub fn test_variadic_fnptr() {
+ let mut s = SipHasher::new();
+ assert_eq!(p.hash(&mut s), q.hash(&mut s));
+ }
++*/
+
+ #[test]
+ fn write_unaligned_drop() {
+--
+2.21.0 (Apple Git-122)
--- /dev/null
- From 0ffdd8eda8df364391c8ac6e1ce92c73ba9254d4 Mon Sep 17 00:00:00 2001
++From eb703e627e7a84f1cd8d0d87f0f69da1f0acf765 Mon Sep 17 00:00:00 2001
+From: bjorn3 <bjorn3@users.noreply.github.com>
+Date: Fri, 3 Dec 2021 12:16:30 +0100
+Subject: [PATCH] Disable long running tests
+
+---
- library/core/tests/slice.rs | 3 +++
- 1 file changed, 3 insertions(+)
++ library/core/tests/slice.rs | 2 ++
++ 1 file changed, 2 insertions(+)
+
+diff --git a/library/core/tests/slice.rs b/library/core/tests/slice.rs
- index 2c8f00a..44847ee 100644
++index 8402833..84592e0 100644
+--- a/library/core/tests/slice.rs
++++ b/library/core/tests/slice.rs
- @@ -2332,7 +2332,8 @@ macro_rules! empty_max_mut {
- };
++@@ -1809,6 +1809,7 @@ fn sort_unstable() {
++ assert!(v == [0xDEADBEEF]);
+ }
+
++/*
- #[cfg(not(miri))] // Comparing usize::MAX many elements takes forever in Miri (and in rustc without optimizations)
- take_tests! {
- slice: &[(); usize::MAX], method: take,
- (take_in_bounds_max_range_to, (..usize::MAX), Some(EMPTY_MAX), &[(); 0]),
- @@ -2345,3 +2347,4 @@ take_tests! {
++ #[test]
++ #[cfg(not(target_arch = "wasm32"))]
++ #[cfg_attr(miri, ignore)] // Miri is too slow
++@@ -1914,6 +1915,7 @@ fn select_nth_unstable() {
++ v.select_nth_unstable(0);
++ assert!(v == [0xDEADBEEF]);
++ }
+++*/
++
++ #[test]
++ #[should_panic(expected = "index 0 greater than length of slice")]
++@@ -2462,6 +2462,7 @@ take_tests! {
++ #[cfg(not(miri))] // unused in Miri
++ const EMPTY_MAX: &'static [()] = &[(); usize::MAX];
++
+++/*
++ // can't be a constant due to const mutability rules
++ #[cfg(not(miri))] // unused in Miri
++ macro_rules! empty_max_mut {
++@@ -2485,6 +2486,7 @@ take_tests! {
+ (take_mut_oob_max_range_to_inclusive, (..=usize::MAX), None, empty_max_mut!()),
+ (take_mut_in_bounds_max_range_from, (usize::MAX..), Some(&mut [] as _), empty_max_mut!()),
+ }
++*/
++
++ #[test]
++ fn test_slice_from_ptr_range() {
+--
+2.26.2.7.g19db9cfb68
+
--- /dev/null
- channel = "nightly-2022-02-23"
+[toolchain]
++channel = "nightly-2022-03-19"
+components = ["rust-src", "rustc-dev", "llvm-tools-preview"]
--- /dev/null
- diff --git a/Cargo.toml b/Cargo.toml
- index 5bd1147cad5..10d68a2ff14 100644
- --- a/Cargo.toml
- +++ b/Cargo.toml
- @@ -111,5 +111,7 @@ rustc-std-workspace-std = { path = 'library/rustc-std-workspace-std' }
- rustc-std-workspace-alloc = { path = 'library/rustc-std-workspace-alloc' }
- rustc-std-workspace-std = { path = 'library/rustc-std-workspace-std' }
-
- +compiler_builtins = { path = "../build_sysroot/compiler-builtins" }
- +
- [patch."https://github.com/rust-lang/rust-clippy"]
- clippy_lints = { path = "src/tools/clippy/clippy_lints" }
+#!/bin/bash
+set -e
+
+./y.rs build --no-unstable-features
+source scripts/config.sh
+
+echo "[SETUP] Rust fork"
+git clone https://github.com/rust-lang/rust.git || true
+pushd rust
+git fetch
+git checkout -- .
+git checkout "$(rustc -V | cut -d' ' -f3 | tr -d '(')"
+
+git apply - <<EOF
+diff --git a/library/alloc/Cargo.toml b/library/alloc/Cargo.toml
+index d95b5b7f17f..00b6f0e3635 100644
+--- a/library/alloc/Cargo.toml
++++ b/library/alloc/Cargo.toml
+@@ -8,7 +8,7 @@ edition = "2018"
+
+ [dependencies]
+ core = { path = "../core" }
+-compiler_builtins = { version = "0.1.40", features = ['rustc-dep-of-std'] }
++compiler_builtins = { version = "0.1.66", features = ['rustc-dep-of-std', 'no-asm'] }
+
+ [dev-dependencies]
+ rand = "0.7"
+ rand_xorshift = "0.2"
++diff --git a/src/tools/compiletest/src/header.rs b/src/tools/compiletest/src/header.rs
++index 887d27fd6dca4..2c2239f2b83d1 100644
++--- a/src/tools/compiletest/src/header.rs
+++++ b/src/tools/compiletest/src/header.rs
++@@ -806,8 +806,8 @@ pub fn make_test_description<R: Read>(
++ cfg: Option<&str>,
++ ) -> test::TestDesc {
++ let mut ignore = false;
++ #[cfg(not(bootstrap))]
++- let ignore_message: Option<String> = None;
+++ let ignore_message: Option<&str> = None;
++ let mut should_fail = false;
++
++ let rustc_has_profiler_support = env::var_os("RUSTC_PROFILER_SUPPORT").is_some();
++
++diff --git a/src/tools/compiletest/src/runtest.rs b/src/tools/compiletest/src/runtest.rs
++index 8431aa7b818..a3ff7e68ce5 100644
++--- a/src/tools/compiletest/src/runtest.rs
+++++ b/src/tools/compiletest/src/runtest.rs
++@@ -3489,11 +3489,7 @@ fn normalize_output(&self, output: &str, custom_rules: &[(String, String)]) -> S
++ .join("library");
++ normalize_path(&src_dir, "$(echo '$SRC_DIR')");
++
++- if let Some(virtual_rust_source_base_dir) =
++- option_env!("CFG_VIRTUAL_RUST_SOURCE_BASE_DIR").map(PathBuf::from)
++- {
++- normalize_path(&virtual_rust_source_base_dir.join("library"), "$(echo '$SRC_DIR')");
++- }
+++ normalize_path(&Path::new("$(cd ../build_sysroot/sysroot_src/library; pwd)"), "$(echo '$SRC_DIR')");
++
++ // Paths into the build directory
++ let test_build_dir = &self.config.build_base;
+EOF
+
+cat > config.toml <<EOF
++changelog-seen = 2
++
+[llvm]
+ninja = false
+
+[build]
+rustc = "$(pwd)/../build/bin/cg_clif"
+cargo = "$(rustup which cargo)"
+full-bootstrap = true
+local-rebuild = true
+
+[rust]
+codegen-backends = ["cranelift"]
+deny-warnings = false
+verbose-tests = false
+EOF
+popd
--- /dev/null
- rm -r src/test/ui/{extern/,panics/,unsized-locals/,lto/,simd*,linkage*,unwind-*.rs} || true
- for test in $(rg --files-with-matches "asm!|catch_unwind|should_panic|lto|// needs-asm-support" src/test/ui); do
+#!/bin/bash
+set -e
+
+cd $(dirname "$0")/../
+
+source ./scripts/setup_rust_fork.sh
+
+echo "[TEST] Test suite of rustc"
+pushd rust
+
+command -v rg >/dev/null 2>&1 || cargo install ripgrep
+
- for test in $(rg -i --files-with-matches "//(\[\w+\])?~|// error-pattern:|// build-fail|// run-fail|-Cllvm-args" src/test/ui); do
++rm -r src/test/ui/{extern/,unsized-locals/,lto/,linkage*} || true
++for test in $(rg --files-with-matches "asm!|lto|// needs-asm-support|// needs-unwind" src/test/{ui,incremental}); do
+ rm $test
+done
+
- # these all depend on unwinding support
++for test in $(rg -i --files-with-matches "//(\[\w+\])?~[^\|]*\s*ERR|// error-pattern:|// build-fail|// run-fail|-Cllvm-args" src/test/ui); do
+ rm $test
+done
+
+git checkout -- src/test/ui/issues/auxiliary/issue-3136-a.rs # contains //~ERROR, but shouldn't be removed
+
- rm src/test/ui/array-slice-vec/box-of-array-of-drop-*.rs
- rm src/test/ui/array-slice-vec/slice-panic-*.rs
- rm src/test/ui/array-slice-vec/nested-vec-3.rs
- rm src/test/ui/cleanup-rvalue-temp-during-incomplete-alloc.rs
- rm src/test/ui/issues/issue-26655.rs
- rm src/test/ui/issues/issue-29485.rs
- rm src/test/ui/issues/issue-30018-panic.rs
++# missing features
++# ================
++
++# requires stack unwinding
+rm src/test/ui/backtrace.rs
- rm src/test/ui/sepcomp/sepcomp-unwind.rs
- rm src/test/ui/structs-enums/unit-like-struct-drop-run.rs
- rm src/test/ui/drop/terminate-in-initializer.rs
- rm src/test/ui/threads-sendsync/task-stderr.rs
- rm src/test/ui/numbers-arithmetic/int-abs-overflow.rs
- rm src/test/ui/drop/drop-trait-enum.rs
+rm src/test/ui/process/multi-panic.rs
- rm src/test/ui/runtime/rt-explody-panic-payloads.rs
+rm src/test/ui/numbers-arithmetic/issue-8460.rs
- rm src/test/ui/threads-sendsync/unwind-resource.rs
+rm src/test/incremental/change_crate_dep_kind.rs
- rm src/test/ui/issues/issue-28950.rs # depends on stack size optimizations
- rm src/test/ui/codegen/init-large-type.rs # same
++rm src/test/incremental/issue-80691-bad-eval-cache.rs # -Cpanic=abort causes abort instead of exit(101)
++rm src/test/ui/panic-while-printing.rs
++rm src/test/ui/test-attrs/test-panic-while-printing.rs
++rm src/test/ui/test-attrs/test-type.rs
+
- rm src/test/ui/issues/issue-51947.rs # same
++# requires compiling with -Cpanic=unwind
++rm src/test/ui/test-attrs/test-fn-signature-verification-for-explicit-return-type.rs # "Cannot run dynamic test fn out-of-process"
++rm src/test/ui/async-await/async-fn-size-moved-locals.rs # -Cpanic=abort shrinks some generator by one byte
++rm src/test/ui/async-await/async-fn-size-uninit-locals.rs # same
++rm src/test/ui/generator/size-moved-locals.rs # same
++
++# vendor intrinsics
+rm src/test/ui/sse2.rs # cpuid not supported, so sse2 not detected
++rm src/test/ui/intrinsics/const-eval-select-x86_64.rs # requires x86_64 vendor intrinsics
++rm src/test/ui/simd/array-type.rs # "Index argument for `simd_insert` is not a constant"
++rm src/test/ui/simd/intrinsic/generic-bitmask-pass.rs # simd_bitmask unimplemented
++rm src/test/ui/simd/intrinsic/generic-as.rs # simd_as unimplemented
++rm src/test/ui/simd/intrinsic/generic-arithmetic-saturating-pass.rs # simd_saturating_add unimplemented
++rm src/test/ui/simd/intrinsic/float-math-pass.rs # simd_fcos unimplemented
++rm src/test/ui/simd/intrinsic/generic-gather-pass.rs # simd_gather unimplemented
++rm src/test/ui/simd/intrinsic/generic-select-pass.rs # simd_select_bitmask unimplemented
++rm src/test/ui/simd/issue-85915-simd-ptrs.rs # simd_gather unimplemented
++rm src/test/ui/simd/issue-89193.rs # simd_gather unimplemented
++rm src/test/ui/simd/simd-bitmask.rs # simd_bitmask unimplemented
++
++# exotic linkages
+rm src/test/ui/issues/issue-33992.rs # unsupported linkages
- rm src/test/ui/async-await/async-fn-size-moved-locals.rs # -Cpanic=abort shrinks some generator by one byte
- rm src/test/ui/async-await/async-fn-size-uninit-locals.rs # same
- rm src/test/ui/generator/size-moved-locals.rs # same
- rm src/test/ui/fn/dyn-fn-alignment.rs # wants a 256 byte alignment
- rm src/test/ui/test-attrs/test-fn-signature-verification-for-explicit-return-type.rs # "Cannot run dynamic test fn out-of-process"
- rm src/test/ui/intrinsics/intrinsic-nearby.rs # unimplemented nearbyintf32 and nearbyintf64 intrinsics
-
- rm src/test/incremental/hashes/inline_asm.rs # inline asm
- rm src/test/incremental/issue-72386.rs # same
- rm src/test/incremental/lto.rs # requires lto
- rm src/test/incremental/dirty_clean.rs # TODO
+rm src/test/incremental/hashes/function_interfaces.rs # same
+rm src/test/incremental/hashes/statics.rs # same
++
++# variadic arguments
++rm src/test/ui/abi/mir/mir_codegen_calls_variadic.rs # requires float varargs
++rm src/test/ui/abi/variadic-ffi.rs # requires callee side vararg support
++
++# unsized locals
++rm -r src/test/run-pass-valgrind/unsized-locals
++
++# misc unimplemented things
++rm src/test/ui/intrinsics/intrinsic-nearby.rs # unimplemented nearbyintf32 and nearbyintf64 intrinsics
++rm src/test/ui/target-feature/missing-plusminus.rs # error not implemented
++rm src/test/ui/fn/dyn-fn-alignment.rs # wants a 256 byte alignment
++rm -r src/test/run-make/emit-named-files # requires full --emit support
++
++# optimization tests
++# ==================
++rm src/test/ui/issues/issue-28950.rs # depends on stack size optimizations
++rm src/test/ui/codegen/init-large-type.rs # same
++rm -r src/test/run-make/fmt-write-bloat/ # tests an optimization
++
++# backend specific tests
++# ======================
++rm src/test/incremental/thinlto/cgu_invalidated_when_import_{added,removed}.rs # requires LLVM
++rm src/test/ui/abi/stack-protector.rs # requires stack protector support
++
++# giving different but possibly correct results
++# =============================================
+rm src/test/ui/numbers-arithmetic/saturating-float-casts.rs # intrinsic gives different but valid result
++rm src/test/ui/simd/intrinsic/float-minmax-pass.rs # same
+rm src/test/ui/mir/mir_misc_casts.rs # depends on deduplication of constants
+rm src/test/ui/mir/mir_raw_fat_ptr.rs # same
+rm src/test/ui/consts/issue-33537.rs # same
- rm -r src/test/run-make/emit-named-files # requires full --emit support
-
- rm -r src/test/run-pass-valgrind/unsized-locals
-
- rm src/test/ui/json-bom-plus-crlf-multifile.rs # differing warning
- rm src/test/ui/json-bom-plus-crlf.rs # same
- rm src/test/ui/intrinsics/const-eval-select-x86_64.rs # same
- rm src/test/ui/match/issue-82392.rs # differing error
- rm src/test/ui/consts/min_const_fn/address_of_const.rs # same
- rm src/test/ui/consts/issue-miri-1910.rs # same
- rm src/test/ui/generic-associated-types/bugs/issue-80626.rs # same
- rm src/test/ui/generic-associated-types/bugs/issue-89008.rs # same
- rm src/test/ui/type-alias-impl-trait/cross_crate_ice*.rs # requires removed aux dep
+
++# doesn't work due to the way the rustc test suite is invoked.
++# should work when using ./x.py test the way it is intended
++# ============================================================
+rm -r src/test/run-make/emit-shared-files # requires the rustdoc executable in build/bin/
+rm -r src/test/run-make/unstable-flag-required # same
+rm -r src/test/run-make/rustdoc-* # same
- rm src/test/ui/cfg/cfg-panic.rs
- rm -r src/test/ui/hygiene/
+
++# genuine bugs
++# ============
+rm src/test/ui/allocator/no_std-alloc-error-handler-default.rs # missing rust_oom definition
- rm -r src/test/run-make/fmt-write-bloat/ # tests an optimization
- rm src/test/ui/abi/mir/mir_codegen_calls_variadic.rs # requires float varargs
- rm src/test/ui/abi/variadic-ffi.rs # requires callee side vararg support
-
- rm src/test/ui/command/command-current-dir.rs # can't find libstd.so
-
- rm src/test/ui/abi/stack-protector.rs # requires stack protector support
-
- rm src/test/incremental/issue-80691-bad-eval-cache.rs # wrong exit code
+
+rm -r src/test/ui/polymorphization/ # polymorphization not yet supported
+rm src/test/codegen-units/polymorphization/unused_type_parameters.rs # same
+
- rm src/test/incremental/thinlto/cgu_invalidated_when_import_{added,removed}.rs # requires LLVM
+rm src/test/incremental/spike-neg1.rs # errors out for some reason
+rm src/test/incremental/spike-neg2.rs # same
+rm src/test/ui/issues/issue-74564-if-expr-stack-overflow.rs # gives a stackoverflow before the backend runs
++rm src/test/ui/mir/ssa-analysis-regression-50041.rs # produces ICE
+
++rm src/test/ui/simd/intrinsic/generic-reduction-pass.rs # simd_reduce_add_unordered doesn't accept an accumulator for integer vectors
++
++# bugs in the test suite
++# ======================
++rm src/test/ui/unsafe/union.rs # has UB caught by cg_clif. see rust-lang/rust#95075
+
+echo "[TEST] rustc test suite"
+RUST_TEST_NOCAPTURE=1 COMPILETEST_FORCE_STAGE0=1 ./x.py test --stage 0 src/test/{codegen-units,run-make,run-pass-valgrind,ui,incremental}
+popd
--- /dev/null
- # FIXME Re-add -Zmir-opt-level=2 once rust-lang/rust#67529 is fixed.
+#!/usr/bin/env bash
+
+set -e
+
+source scripts/config.sh
+source scripts/ext_config.sh
+export RUSTC=false # ensure that cg_llvm isn't accidentally used
+MY_RUSTC="$(pwd)/build/bin/cg_clif $RUSTFLAGS -L crate=target/out --out-dir target/out -Cdebuginfo=2"
+
+function no_sysroot_tests() {
+ echo "[BUILD] mini_core"
+ $MY_RUSTC example/mini_core.rs --crate-name mini_core --crate-type lib,dylib --target "$TARGET_TRIPLE"
+
+ echo "[BUILD] example"
+ $MY_RUSTC example/example.rs --crate-type lib --target "$TARGET_TRIPLE"
+
+ if [[ "$JIT_SUPPORTED" = "1" ]]; then
+ echo "[JIT] mini_core_hello_world"
+ CG_CLIF_JIT_ARGS="abc bcd" $MY_RUSTC -Zunstable-options -Cllvm-args=mode=jit -Cprefer-dynamic example/mini_core_hello_world.rs --cfg jit --target "$HOST_TRIPLE"
+
+ echo "[JIT-lazy] mini_core_hello_world"
+ CG_CLIF_JIT_ARGS="abc bcd" $MY_RUSTC -Zunstable-options -Cllvm-args=mode=jit-lazy -Cprefer-dynamic example/mini_core_hello_world.rs --cfg jit --target "$HOST_TRIPLE"
+ else
+ echo "[JIT] mini_core_hello_world (skipped)"
+ fi
+
+ echo "[AOT] mini_core_hello_world"
+ $MY_RUSTC example/mini_core_hello_world.rs --crate-name mini_core_hello_world --crate-type bin -g --target "$TARGET_TRIPLE"
+ $RUN_WRAPPER ./target/out/mini_core_hello_world abc bcd
+ # (echo "break set -n main"; echo "run"; sleep 1; echo "si -c 10"; sleep 1; echo "frame variable") | lldb -- ./target/out/mini_core_hello_world abc bcd
+}
+
+function base_sysroot_tests() {
+ echo "[AOT] arbitrary_self_types_pointers_and_wrappers"
+ $MY_RUSTC example/arbitrary_self_types_pointers_and_wrappers.rs --crate-name arbitrary_self_types_pointers_and_wrappers --crate-type bin --target "$TARGET_TRIPLE"
+ $RUN_WRAPPER ./target/out/arbitrary_self_types_pointers_and_wrappers
+
+ echo "[AOT] issue_91827_extern_types"
+ $MY_RUSTC example/issue-91827-extern-types.rs --crate-name issue_91827_extern_types --crate-type bin --target "$TARGET_TRIPLE"
+ $RUN_WRAPPER ./target/out/issue_91827_extern_types
+
+ echo "[AOT] alloc_system"
+ $MY_RUSTC example/alloc_system.rs --crate-type lib --target "$TARGET_TRIPLE"
+
+ echo "[AOT] alloc_example"
+ $MY_RUSTC example/alloc_example.rs --crate-type bin --target "$TARGET_TRIPLE"
+ $RUN_WRAPPER ./target/out/alloc_example
+
+ if [[ "$JIT_SUPPORTED" = "1" ]]; then
+ echo "[JIT] std_example"
+ $MY_RUSTC -Zunstable-options -Cllvm-args=mode=jit -Cprefer-dynamic example/std_example.rs --target "$HOST_TRIPLE"
+
+ echo "[JIT-lazy] std_example"
+ $MY_RUSTC -Zunstable-options -Cllvm-args=mode=jit-lazy -Cprefer-dynamic example/std_example.rs --target "$HOST_TRIPLE"
+ else
+ echo "[JIT] std_example (skipped)"
+ fi
+
+ echo "[AOT] dst_field_align"
+ $MY_RUSTC example/dst-field-align.rs --crate-name dst_field_align --crate-type bin --target "$TARGET_TRIPLE"
+ $RUN_WRAPPER ./target/out/dst_field_align || (echo $?; false)
+
+ echo "[AOT] std_example"
+ $MY_RUSTC example/std_example.rs --crate-type bin --target "$TARGET_TRIPLE"
+ $RUN_WRAPPER ./target/out/std_example arg
+
+ echo "[AOT] subslice-patterns-const-eval"
+ $MY_RUSTC example/subslice-patterns-const-eval.rs --crate-type bin -Cpanic=abort --target "$TARGET_TRIPLE"
+ $RUN_WRAPPER ./target/out/subslice-patterns-const-eval
+
+ echo "[AOT] track-caller-attribute"
+ $MY_RUSTC example/track-caller-attribute.rs --crate-type bin -Cpanic=abort --target "$TARGET_TRIPLE"
+ $RUN_WRAPPER ./target/out/track-caller-attribute
+
+ echo "[AOT] mod_bench"
+ $MY_RUSTC example/mod_bench.rs --crate-type bin --target "$TARGET_TRIPLE"
+ $RUN_WRAPPER ./target/out/mod_bench
+}
+
+function extended_sysroot_tests() {
+ pushd rand
+ ../build/cargo-clif clean
+ if [[ "$HOST_TRIPLE" = "$TARGET_TRIPLE" ]]; then
+ echo "[TEST] rust-random/rand"
+ ../build/cargo-clif test --workspace
+ else
+ echo "[AOT] rust-random/rand"
+ ../build/cargo-clif build --workspace --target $TARGET_TRIPLE --tests
+ fi
+ popd
+
+ pushd simple-raytracer
+ if [[ "$HOST_TRIPLE" = "$TARGET_TRIPLE" ]]; then
+ echo "[BENCH COMPILE] ebobby/simple-raytracer"
+ hyperfine --runs "${RUN_RUNS:-10}" --warmup 1 --prepare "../build/cargo-clif clean" \
+ "RUSTC=rustc RUSTFLAGS='' cargo build" \
+ "../build/cargo-clif build"
+
+ echo "[BENCH RUN] ebobby/simple-raytracer"
+ cp ./target/debug/main ./raytracer_cg_clif
+ hyperfine --runs "${RUN_RUNS:-10}" ./raytracer_cg_llvm ./raytracer_cg_clif
+ else
+ ../build/cargo-clif clean
+ echo "[BENCH COMPILE] ebobby/simple-raytracer (skipped)"
+ echo "[COMPILE] ebobby/simple-raytracer"
+ ../build/cargo-clif build --target $TARGET_TRIPLE
+ echo "[BENCH RUN] ebobby/simple-raytracer (skipped)"
+ fi
+ popd
+
+ pushd build_sysroot/sysroot_src/library/core/tests
+ echo "[TEST] libcore"
+ ../../../../../build/cargo-clif clean
+ if [[ "$HOST_TRIPLE" = "$TARGET_TRIPLE" ]]; then
+ ../../../../../build/cargo-clif test
+ else
+ ../../../../../build/cargo-clif build --target $TARGET_TRIPLE --tests
+ fi
+ popd
+
+ pushd regex
+ echo "[TEST] rust-lang/regex example shootout-regex-dna"
+ ../build/cargo-clif clean
+ export RUSTFLAGS="$RUSTFLAGS --cap-lints warn" # newer aho_corasick versions throw a deprecation warning
+ # Make sure `[codegen mono items] start` doesn't poison the diff
+ ../build/cargo-clif build --example shootout-regex-dna --target $TARGET_TRIPLE
+ if [[ "$HOST_TRIPLE" = "$TARGET_TRIPLE" ]]; then
+ cat examples/regexdna-input.txt \
+ | ../build/cargo-clif run --example shootout-regex-dna --target $TARGET_TRIPLE \
+ | grep -v "Spawned thread" > res.txt
+ diff -u res.txt examples/regexdna-output.txt
+ fi
+
+ if [[ "$HOST_TRIPLE" = "$TARGET_TRIPLE" ]]; then
+ echo "[TEST] rust-lang/regex tests"
+ ../build/cargo-clif test --tests -- --exclude-should-panic --test-threads 1 -Zunstable-options -q
+ else
+ echo "[AOT] rust-lang/regex tests"
+ ../build/cargo-clif build --tests --target $TARGET_TRIPLE
+ fi
+ popd
+
+ pushd portable-simd
+ echo "[TEST] rust-lang/portable-simd"
+ ../build/cargo-clif clean
+ ../build/cargo-clif build --all-targets --target $TARGET_TRIPLE
+ if [[ "$HOST_TRIPLE" = "$TARGET_TRIPLE" ]]; then
+ ../build/cargo-clif test -q
+ fi
+ popd
+}
+
+case "$1" in
+ "no_sysroot")
+ no_sysroot_tests
+ ;;
+ "base_sysroot")
+ base_sysroot_tests
+ ;;
+ "extended_sysroot")
+ extended_sysroot_tests
+ ;;
+ *)
+ echo "unknown test suite"
+ ;;
+esac
--- /dev/null
- // FIXME Mark current_block block as cold once Cranelift supports it
+//! Handling of everything related to the calling convention. Also fills `fx.local_map`.
+
+mod comments;
+mod pass_mode;
+mod returning;
+
+use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
+use rustc_middle::ty::layout::FnAbiOf;
+use rustc_target::abi::call::{Conv, FnAbi};
+use rustc_target::spec::abi::Abi;
+
+use cranelift_codegen::ir::{AbiParam, SigRef};
+
+use self::pass_mode::*;
+use crate::prelude::*;
+
+pub(crate) use self::returning::codegen_return;
+
+fn clif_sig_from_fn_abi<'tcx>(
+ tcx: TyCtxt<'tcx>,
+ default_call_conv: CallConv,
+ fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
+) -> Signature {
+ let call_conv = match fn_abi.conv {
+ Conv::Rust | Conv::C => default_call_conv,
+ Conv::X86_64SysV => CallConv::SystemV,
+ Conv::X86_64Win64 => CallConv::WindowsFastcall,
+ Conv::ArmAapcs
+ | Conv::CCmseNonSecureCall
+ | Conv::Msp430Intr
+ | Conv::PtxKernel
+ | Conv::X86Fastcall
+ | Conv::X86Intr
+ | Conv::X86Stdcall
+ | Conv::X86ThisCall
+ | Conv::X86VectorCall
+ | Conv::AmdGpuKernel
+ | Conv::AvrInterrupt
+ | Conv::AvrNonBlockingInterrupt => todo!("{:?}", fn_abi.conv),
+ };
+ let inputs = fn_abi.args.iter().map(|arg_abi| arg_abi.get_abi_param(tcx).into_iter()).flatten();
+
+ let (return_ptr, returns) = fn_abi.ret.get_abi_return(tcx);
+ // Sometimes the first param is an pointer to the place where the return value needs to be stored.
+ let params: Vec<_> = return_ptr.into_iter().chain(inputs).collect();
+
+ Signature { params, returns, call_conv }
+}
+
+pub(crate) fn get_function_sig<'tcx>(
+ tcx: TyCtxt<'tcx>,
+ triple: &target_lexicon::Triple,
+ inst: Instance<'tcx>,
+) -> Signature {
+ assert!(!inst.substs.needs_infer());
+ clif_sig_from_fn_abi(
+ tcx,
+ CallConv::triple_default(triple),
+ &RevealAllLayoutCx(tcx).fn_abi_of_instance(inst, ty::List::empty()),
+ )
+}
+
+/// Instance must be monomorphized
+pub(crate) fn import_function<'tcx>(
+ tcx: TyCtxt<'tcx>,
+ module: &mut dyn Module,
+ inst: Instance<'tcx>,
+) -> FuncId {
+ let name = tcx.symbol_name(inst).name;
+ let sig = get_function_sig(tcx, module.isa().triple(), inst);
+ module.declare_function(name, Linkage::Import, &sig).unwrap()
+}
+
+impl<'tcx> FunctionCx<'_, '_, 'tcx> {
+ /// Instance must be monomorphized
+ pub(crate) fn get_function_ref(&mut self, inst: Instance<'tcx>) -> FuncRef {
+ let func_id = import_function(self.tcx, self.module, inst);
+ let func_ref = self.module.declare_func_in_func(func_id, &mut self.bcx.func);
+
+ if self.clif_comments.enabled() {
+ self.add_comment(func_ref, format!("{:?}", inst));
+ }
+
+ func_ref
+ }
+
+ pub(crate) fn lib_call(
+ &mut self,
+ name: &str,
+ params: Vec<AbiParam>,
+ returns: Vec<AbiParam>,
+ args: &[Value],
+ ) -> &[Value] {
+ let sig = Signature { params, returns, call_conv: self.target_config.default_call_conv };
+ let func_id = self.module.declare_function(name, Linkage::Import, &sig).unwrap();
+ let func_ref = self.module.declare_func_in_func(func_id, &mut self.bcx.func);
++ if self.clif_comments.enabled() {
++ self.add_comment(func_ref, format!("{:?}", name));
++ }
+ let call_inst = self.bcx.ins().call(func_ref, args);
+ if self.clif_comments.enabled() {
+ self.add_comment(call_inst, format!("easy_call {}", name));
+ }
+ let results = self.bcx.inst_results(call_inst);
+ assert!(results.len() <= 2, "{}", results.len());
+ results
+ }
+
+ pub(crate) fn easy_call(
+ &mut self,
+ name: &str,
+ args: &[CValue<'tcx>],
+ return_ty: Ty<'tcx>,
+ ) -> CValue<'tcx> {
+ let (input_tys, args): (Vec<_>, Vec<_>) = args
+ .iter()
+ .map(|arg| {
+ (AbiParam::new(self.clif_type(arg.layout().ty).unwrap()), arg.load_scalar(self))
+ })
+ .unzip();
+ let return_layout = self.layout_of(return_ty);
+ let return_tys = if let ty::Tuple(tup) = return_ty.kind() {
+ tup.iter().map(|ty| AbiParam::new(self.clif_type(ty).unwrap())).collect()
+ } else {
+ vec![AbiParam::new(self.clif_type(return_ty).unwrap())]
+ };
+ let ret_vals = self.lib_call(name, input_tys, return_tys, &args);
+ match *ret_vals {
+ [] => CValue::by_ref(
+ Pointer::const_addr(self, i64::from(self.pointer_type.bytes())),
+ return_layout,
+ ),
+ [val] => CValue::by_val(val, return_layout),
+ [val, extra] => CValue::by_val_pair(val, extra, return_layout),
+ _ => unreachable!(),
+ }
+ }
+}
+
+/// Make a [`CPlace`] capable of holding value of the specified type.
+fn make_local_place<'tcx>(
+ fx: &mut FunctionCx<'_, '_, 'tcx>,
+ local: Local,
+ layout: TyAndLayout<'tcx>,
+ is_ssa: bool,
+) -> CPlace<'tcx> {
+ let place = if is_ssa {
+ if let rustc_target::abi::Abi::ScalarPair(_, _) = layout.abi {
+ CPlace::new_var_pair(fx, local, layout)
+ } else {
+ CPlace::new_var(fx, local, layout)
+ }
+ } else {
+ CPlace::new_stack_slot(fx, layout)
+ };
+
+ self::comments::add_local_place_comments(fx, place, local);
+
+ place
+}
+
+pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_block: Block) {
+ fx.bcx.append_block_params_for_function_params(start_block);
+
+ fx.bcx.switch_to_block(start_block);
+ fx.bcx.ins().nop();
+
+ let ssa_analyzed = crate::analyze::analyze(fx);
+
+ self::comments::add_args_header_comment(fx);
+
+ let mut block_params_iter = fx.bcx.func.dfg.block_params(start_block).to_vec().into_iter();
+ let ret_place =
+ self::returning::codegen_return_param(fx, &ssa_analyzed, &mut block_params_iter);
+ assert_eq!(fx.local_map.push(ret_place), RETURN_PLACE);
+
+ // None means pass_mode == NoPass
+ enum ArgKind<'tcx> {
+ Normal(Option<CValue<'tcx>>),
+ Spread(Vec<Option<CValue<'tcx>>>),
+ }
+
+ let fn_abi = fx.fn_abi.take().unwrap();
+ let mut arg_abis_iter = fn_abi.args.iter();
+
+ let func_params = fx
+ .mir
+ .args_iter()
+ .map(|local| {
+ let arg_ty = fx.monomorphize(fx.mir.local_decls[local].ty);
+
+ // Adapted from https://github.com/rust-lang/rust/blob/145155dc96757002c7b2e9de8489416e2fdbbd57/src/librustc_codegen_llvm/mir/mod.rs#L442-L482
+ if Some(local) == fx.mir.spread_arg {
+ // This argument (e.g. the last argument in the "rust-call" ABI)
+ // is a tuple that was spread at the ABI level and now we have
+ // to reconstruct it into a tuple local variable, from multiple
+ // individual function arguments.
+
+ let tupled_arg_tys = match arg_ty.kind() {
+ ty::Tuple(ref tys) => tys,
+ _ => bug!("spread argument isn't a tuple?! but {:?}", arg_ty),
+ };
+
+ let mut params = Vec::new();
+ for (i, _arg_ty) in tupled_arg_tys.iter().enumerate() {
+ let arg_abi = arg_abis_iter.next().unwrap();
+ let param =
+ cvalue_for_param(fx, Some(local), Some(i), arg_abi, &mut block_params_iter);
+ params.push(param);
+ }
+
+ (local, ArgKind::Spread(params), arg_ty)
+ } else {
+ let arg_abi = arg_abis_iter.next().unwrap();
+ let param =
+ cvalue_for_param(fx, Some(local), None, arg_abi, &mut block_params_iter);
+ (local, ArgKind::Normal(param), arg_ty)
+ }
+ })
+ .collect::<Vec<(Local, ArgKind<'tcx>, Ty<'tcx>)>>();
+
+ assert!(fx.caller_location.is_none());
+ if fx.instance.def.requires_caller_location(fx.tcx) {
+ // Store caller location for `#[track_caller]`.
+ let arg_abi = arg_abis_iter.next().unwrap();
+ fx.caller_location =
+ Some(cvalue_for_param(fx, None, None, arg_abi, &mut block_params_iter).unwrap());
+ }
+
+ assert!(arg_abis_iter.next().is_none(), "ArgAbi left behind");
+ fx.fn_abi = Some(fn_abi);
+ assert!(block_params_iter.next().is_none(), "arg_value left behind");
+
+ self::comments::add_locals_header_comment(fx);
+
+ for (local, arg_kind, ty) in func_params {
+ let layout = fx.layout_of(ty);
+
+ let is_ssa = ssa_analyzed[local] == crate::analyze::SsaKind::Ssa;
+
+ // While this is normally an optimization to prevent an unnecessary copy when an argument is
+ // not mutated by the current function, this is necessary to support unsized arguments.
+ if let ArgKind::Normal(Some(val)) = arg_kind {
+ if let Some((addr, meta)) = val.try_to_ptr() {
+ // Ownership of the value at the backing storage for an argument is passed to the
+ // callee per the ABI, so it is fine to borrow the backing storage of this argument
+ // to prevent a copy.
+
+ let place = if let Some(meta) = meta {
+ CPlace::for_ptr_with_extra(addr, meta, val.layout())
+ } else {
+ CPlace::for_ptr(addr, val.layout())
+ };
+
+ self::comments::add_local_place_comments(fx, place, local);
+
+ assert_eq!(fx.local_map.push(place), local);
+ continue;
+ }
+ }
+
+ let place = make_local_place(fx, local, layout, is_ssa);
+ assert_eq!(fx.local_map.push(place), local);
+
+ match arg_kind {
+ ArgKind::Normal(param) => {
+ if let Some(param) = param {
+ place.write_cvalue(fx, param);
+ }
+ }
+ ArgKind::Spread(params) => {
+ for (i, param) in params.into_iter().enumerate() {
+ if let Some(param) = param {
+ place.place_field(fx, mir::Field::new(i)).write_cvalue(fx, param);
+ }
+ }
+ }
+ }
+ }
+
+ for local in fx.mir.vars_and_temps_iter() {
+ let ty = fx.monomorphize(fx.mir.local_decls[local].ty);
+ let layout = fx.layout_of(ty);
+
+ let is_ssa = ssa_analyzed[local] == crate::analyze::SsaKind::Ssa;
+
+ let place = make_local_place(fx, local, layout, is_ssa);
+ assert_eq!(fx.local_map.push(place), local);
+ }
+
+ fx.bcx.ins().jump(*fx.block_map.get(START_BLOCK).unwrap(), &[]);
+}
+
+struct CallArgument<'tcx> {
+ value: CValue<'tcx>,
+ is_owned: bool,
+}
+
+// FIXME avoid intermediate `CValue` before calling `adjust_arg_for_abi`
+fn codegen_call_argument_operand<'tcx>(
+ fx: &mut FunctionCx<'_, '_, 'tcx>,
+ operand: &Operand<'tcx>,
+) -> CallArgument<'tcx> {
+ CallArgument {
+ value: codegen_operand(fx, operand),
+ is_owned: matches!(operand, Operand::Move(_)),
+ }
+}
+
+pub(crate) fn codegen_terminator_call<'tcx>(
+ fx: &mut FunctionCx<'_, '_, 'tcx>,
+ span: Span,
+ func: &Operand<'tcx>,
+ args: &[Operand<'tcx>],
+ mir_dest: Option<(Place<'tcx>, BasicBlock)>,
+) {
+ let fn_ty = fx.monomorphize(func.ty(fx.mir, fx.tcx));
+ let fn_sig =
+ fx.tcx.normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), fn_ty.fn_sig(fx.tcx));
+
+ let destination = mir_dest.map(|(place, bb)| (codegen_place(fx, place), bb));
+
+ // Handle special calls like instrinsics and empty drop glue.
+ let instance = if let ty::FnDef(def_id, substs) = *fn_ty.kind() {
+ let instance = ty::Instance::resolve(fx.tcx, ty::ParamEnv::reveal_all(), def_id, substs)
+ .unwrap()
+ .unwrap()
+ .polymorphize(fx.tcx);
+
+ if fx.tcx.symbol_name(instance).name.starts_with("llvm.") {
+ crate::intrinsics::codegen_llvm_intrinsic_call(
+ fx,
+ &fx.tcx.symbol_name(instance).name,
+ substs,
+ args,
+ destination,
+ );
+ return;
+ }
+
+ match instance.def {
+ InstanceDef::Intrinsic(_) => {
+ crate::intrinsics::codegen_intrinsic_call(fx, instance, args, destination, span);
+ return;
+ }
+ InstanceDef::DropGlue(_, None) => {
+ // empty drop glue - a nop.
+ let (_, dest) = destination.expect("Non terminating drop_in_place_real???");
+ let ret_block = fx.get_block(dest);
+ fx.bcx.ins().jump(ret_block, &[]);
+ return;
+ }
+ _ => Some(instance),
+ }
+ } else {
+ None
+ };
+
+ let extra_args = &args[fn_sig.inputs().len()..];
+ let extra_args = fx
+ .tcx
+ .mk_type_list(extra_args.iter().map(|op_arg| fx.monomorphize(op_arg.ty(fx.mir, fx.tcx))));
+ let fn_abi = if let Some(instance) = instance {
+ RevealAllLayoutCx(fx.tcx).fn_abi_of_instance(instance, extra_args)
+ } else {
+ RevealAllLayoutCx(fx.tcx).fn_abi_of_fn_ptr(fn_ty.fn_sig(fx.tcx), extra_args)
+ };
+
+ let is_cold = instance
+ .map(|inst| fx.tcx.codegen_fn_attrs(inst.def_id()).flags.contains(CodegenFnAttrFlags::COLD))
+ .unwrap_or(false);
+ if is_cold {
- trap_unreachable(fx, "[corruption] Diverging function returned");
++ fx.bcx.set_cold_block(fx.bcx.current_block().unwrap());
++ if let Some((_place, destination_block)) = destination {
++ fx.bcx.set_cold_block(fx.get_block(destination_block));
++ }
+ }
+
+ // Unpack arguments tuple for closures
+ let mut args = if fn_sig.abi == Abi::RustCall {
+ assert_eq!(args.len(), 2, "rust-call abi requires two arguments");
+ let self_arg = codegen_call_argument_operand(fx, &args[0]);
+ let pack_arg = codegen_call_argument_operand(fx, &args[1]);
+
+ let tupled_arguments = match pack_arg.value.layout().ty.kind() {
+ ty::Tuple(ref tupled_arguments) => tupled_arguments,
+ _ => bug!("argument to function with \"rust-call\" ABI is not a tuple"),
+ };
+
+ let mut args = Vec::with_capacity(1 + tupled_arguments.len());
+ args.push(self_arg);
+ for i in 0..tupled_arguments.len() {
+ args.push(CallArgument {
+ value: pack_arg.value.value_field(fx, mir::Field::new(i)),
+ is_owned: pack_arg.is_owned,
+ });
+ }
+ args
+ } else {
+ args.iter().map(|arg| codegen_call_argument_operand(fx, arg)).collect::<Vec<_>>()
+ };
+
+ // Pass the caller location for `#[track_caller]`.
+ if instance.map(|inst| inst.def.requires_caller_location(fx.tcx)).unwrap_or(false) {
+ let caller_location = fx.get_caller_location(span);
+ args.push(CallArgument { value: caller_location, is_owned: false });
+ }
+
+ let args = args;
+ assert_eq!(fn_abi.args.len(), args.len());
+
+ enum CallTarget {
+ Direct(FuncRef),
+ Indirect(SigRef, Value),
+ }
+
+ let (func_ref, first_arg_override) = match instance {
+ // Trait object call
+ Some(Instance { def: InstanceDef::Virtual(_, idx), .. }) => {
+ if fx.clif_comments.enabled() {
+ let nop_inst = fx.bcx.ins().nop();
+ fx.add_comment(
+ nop_inst,
+ format!("virtual call; self arg pass mode: {:?}", &fn_abi.args[0]),
+ );
+ }
+
+ let (ptr, method) = crate::vtable::get_ptr_and_method_ref(fx, args[0].value, idx);
+ let sig = clif_sig_from_fn_abi(fx.tcx, fx.target_config.default_call_conv, &fn_abi);
+ let sig = fx.bcx.import_signature(sig);
+
+ (CallTarget::Indirect(sig, method), Some(ptr))
+ }
+
+ // Normal call
+ Some(instance) => {
+ let func_ref = fx.get_function_ref(instance);
+ (CallTarget::Direct(func_ref), None)
+ }
+
+ // Indirect call
+ None => {
+ if fx.clif_comments.enabled() {
+ let nop_inst = fx.bcx.ins().nop();
+ fx.add_comment(nop_inst, "indirect call");
+ }
+
+ let func = codegen_operand(fx, func).load_scalar(fx);
+ let sig = clif_sig_from_fn_abi(fx.tcx, fx.target_config.default_call_conv, &fn_abi);
+ let sig = fx.bcx.import_signature(sig);
+
+ (CallTarget::Indirect(sig, func), None)
+ }
+ };
+
+ let ret_place = destination.map(|(place, _)| place);
+ self::returning::codegen_with_call_return_arg(fx, &fn_abi.ret, ret_place, |fx, return_ptr| {
+ let call_args = return_ptr
+ .into_iter()
+ .chain(first_arg_override.into_iter())
+ .chain(
+ args.into_iter()
+ .enumerate()
+ .skip(if first_arg_override.is_some() { 1 } else { 0 })
+ .map(|(i, arg)| {
+ adjust_arg_for_abi(fx, arg.value, &fn_abi.args[i], arg.is_owned).into_iter()
+ })
+ .flatten(),
+ )
+ .collect::<Vec<Value>>();
+
+ let call_inst = match func_ref {
+ CallTarget::Direct(func_ref) => fx.bcx.ins().call(func_ref, &call_args),
+ CallTarget::Indirect(sig, func_ptr) => {
+ fx.bcx.ins().call_indirect(sig, func_ptr, &call_args)
+ }
+ };
+
+ // FIXME find a cleaner way to support varargs
+ if fn_sig.c_variadic {
+ if !matches!(fn_sig.abi, Abi::C { .. }) {
+ fx.tcx
+ .sess
+ .span_fatal(span, &format!("Variadic call for non-C abi {:?}", fn_sig.abi));
+ }
+ let sig_ref = fx.bcx.func.dfg.call_signature(call_inst).unwrap();
+ let abi_params = call_args
+ .into_iter()
+ .map(|arg| {
+ let ty = fx.bcx.func.dfg.value_type(arg);
+ if !ty.is_int() {
+ // FIXME set %al to upperbound on float args once floats are supported
+ fx.tcx
+ .sess
+ .span_fatal(span, &format!("Non int ty {:?} for variadic call", ty));
+ }
+ AbiParam::new(ty)
+ })
+ .collect::<Vec<AbiParam>>();
+ fx.bcx.func.dfg.signatures[sig_ref].params = abi_params;
+ }
+
+ call_inst
+ });
+
+ if let Some((_, dest)) = destination {
+ let ret_block = fx.get_block(dest);
+ fx.bcx.ins().jump(ret_block, &[]);
+ } else {
++ fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
+ }
+}
+
+pub(crate) fn codegen_drop<'tcx>(
+ fx: &mut FunctionCx<'_, '_, 'tcx>,
+ span: Span,
+ drop_place: CPlace<'tcx>,
+) {
+ let ty = drop_place.layout().ty;
+ let drop_instance = Instance::resolve_drop_in_place(fx.tcx, ty).polymorphize(fx.tcx);
+
+ if let ty::InstanceDef::DropGlue(_, None) = drop_instance.def {
+ // we don't actually need to drop anything
+ } else {
+ match ty.kind() {
+ ty::Dynamic(..) => {
+ let (ptr, vtable) = drop_place.to_ptr_maybe_unsized();
+ let ptr = ptr.get_addr(fx);
+ let drop_fn = crate::vtable::drop_fn_of_obj(fx, vtable.unwrap());
+
+ // FIXME(eddyb) perhaps move some of this logic into
+ // `Instance::resolve_drop_in_place`?
+ let virtual_drop = Instance {
+ def: ty::InstanceDef::Virtual(drop_instance.def_id(), 0),
+ substs: drop_instance.substs,
+ };
+ let fn_abi =
+ RevealAllLayoutCx(fx.tcx).fn_abi_of_instance(virtual_drop, ty::List::empty());
+
+ let sig = clif_sig_from_fn_abi(fx.tcx, fx.target_config.default_call_conv, &fn_abi);
+ let sig = fx.bcx.import_signature(sig);
+ fx.bcx.ins().call_indirect(sig, drop_fn, &[ptr]);
+ }
+ _ => {
+ assert!(!matches!(drop_instance.def, InstanceDef::Virtual(_, _)));
+
+ let fn_abi =
+ RevealAllLayoutCx(fx.tcx).fn_abi_of_instance(drop_instance, ty::List::empty());
+
+ let arg_value = drop_place.place_ref(
+ fx,
+ fx.layout_of(fx.tcx.mk_ref(
+ fx.tcx.lifetimes.re_erased,
+ TypeAndMut { ty, mutbl: crate::rustc_hir::Mutability::Mut },
+ )),
+ );
+ let arg_value = adjust_arg_for_abi(fx, arg_value, &fn_abi.args[0], true);
+
+ let mut call_args: Vec<Value> = arg_value.into_iter().collect::<Vec<_>>();
+
+ if drop_instance.def.requires_caller_location(fx.tcx) {
+ // Pass the caller location for `#[track_caller]`.
+ let caller_location = fx.get_caller_location(span);
+ call_args.extend(
+ adjust_arg_for_abi(fx, caller_location, &fn_abi.args[1], false).into_iter(),
+ );
+ }
+
+ let func_ref = fx.get_function_ref(drop_instance);
+ fx.bcx.ins().call(func_ref, &call_args);
+ }
+ }
+ }
+}
--- /dev/null
- rustc_middle::mir::write_mir_pretty(tcx, Some(instance.def_id()), &mut buf).unwrap();
+//! Codegen of a single function
+
+use rustc_ast::InlineAsmOptions;
+use rustc_index::vec::IndexVec;
+use rustc_middle::ty::adjustment::PointerCast;
+use rustc_middle::ty::layout::FnAbiOf;
++use rustc_middle::ty::print::with_no_trimmed_paths;
+
+use indexmap::IndexSet;
+
+use crate::constant::ConstantCx;
+use crate::prelude::*;
+use crate::pretty_clif::CommentWriter;
+
+pub(crate) fn codegen_fn<'tcx>(
+ cx: &mut crate::CodegenCx<'tcx>,
+ module: &mut dyn Module,
+ instance: Instance<'tcx>,
+) {
+ let tcx = cx.tcx;
+
+ let _inst_guard =
+ crate::PrintOnPanic(|| format!("{:?} {}", instance, tcx.symbol_name(instance).name));
+ debug_assert!(!instance.substs.needs_infer());
+
+ let mir = tcx.instance_mir(instance.def);
+ let _mir_guard = crate::PrintOnPanic(|| {
+ let mut buf = Vec::new();
- crate::trap::trap_unreachable(&mut fx, "function has uninhabited argument");
++ with_no_trimmed_paths!({
++ rustc_middle::mir::pretty::write_mir_fn(tcx, mir, &mut |_, _| Ok(()), &mut buf)
++ .unwrap();
++ });
+ String::from_utf8_lossy(&buf).into_owned()
+ });
+
+ // Declare function
+ let symbol_name = tcx.symbol_name(instance);
+ let sig = get_function_sig(tcx, module.isa().triple(), instance);
+ let func_id = module.declare_function(symbol_name.name, Linkage::Local, &sig).unwrap();
+
+ cx.cached_context.clear();
+
+ // Make the FunctionBuilder
+ let mut func_ctx = FunctionBuilderContext::new();
+ let mut func = std::mem::replace(&mut cx.cached_context.func, Function::new());
+ func.name = ExternalName::user(0, func_id.as_u32());
+ func.signature = sig;
+ func.collect_debug_info();
+
+ let mut bcx = FunctionBuilder::new(&mut func, &mut func_ctx);
+
+ // Predefine blocks
+ let start_block = bcx.create_block();
+ let block_map: IndexVec<BasicBlock, Block> =
+ (0..mir.basic_blocks().len()).map(|_| bcx.create_block()).collect();
+
+ // Make FunctionCx
+ let target_config = module.target_config();
+ let pointer_type = target_config.pointer_type();
+ let clif_comments = crate::pretty_clif::CommentWriter::new(tcx, instance);
+
+ let mut fx = FunctionCx {
+ cx,
+ module,
+ tcx,
+ target_config,
+ pointer_type,
+ constants_cx: ConstantCx::new(),
+
+ instance,
+ symbol_name,
+ mir,
+ fn_abi: Some(RevealAllLayoutCx(tcx).fn_abi_of_instance(instance, ty::List::empty())),
+
+ bcx,
+ block_map,
+ local_map: IndexVec::with_capacity(mir.local_decls.len()),
+ caller_location: None, // set by `codegen_fn_prelude`
+
+ clif_comments,
+ source_info_set: indexmap::IndexSet::new(),
+ next_ssa_var: 0,
+ };
+
+ let arg_uninhabited = fx
+ .mir
+ .args_iter()
+ .any(|arg| fx.layout_of(fx.monomorphize(fx.mir.local_decls[arg].ty)).abi.is_uninhabited());
+
+ if !crate::constant::check_constants(&mut fx) {
+ fx.bcx.append_block_params_for_function_params(fx.block_map[START_BLOCK]);
+ fx.bcx.switch_to_block(fx.block_map[START_BLOCK]);
+ crate::trap::trap_unreachable(&mut fx, "compilation should have been aborted");
+ } else if arg_uninhabited {
+ fx.bcx.append_block_params_for_function_params(fx.block_map[START_BLOCK]);
+ fx.bcx.switch_to_block(fx.block_map[START_BLOCK]);
- bb_data.terminator().kind.fmt_head(&mut terminator_head).unwrap();
++ fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
+ } else {
+ tcx.sess.time("codegen clif ir", || {
+ tcx.sess
+ .time("codegen prelude", || crate::abi::codegen_fn_prelude(&mut fx, start_block));
+ codegen_fn_content(&mut fx);
+ });
+ }
+
+ // Recover all necessary data from fx, before accessing func will prevent future access to it.
+ let instance = fx.instance;
+ let clif_comments = fx.clif_comments;
+ let source_info_set = fx.source_info_set;
+ let local_map = fx.local_map;
+
+ fx.constants_cx.finalize(fx.tcx, &mut *fx.module);
+
+ crate::pretty_clif::write_clif_file(
+ tcx,
+ "unopt",
+ module.isa(),
+ instance,
+ &func,
+ &clif_comments,
+ );
+
+ // Verify function
+ verify_func(tcx, &clif_comments, &func);
+
+ compile_fn(
+ cx,
+ module,
+ instance,
+ symbol_name.name,
+ func_id,
+ func,
+ clif_comments,
+ source_info_set,
+ local_map,
+ );
+}
+
+fn compile_fn<'tcx>(
+ cx: &mut crate::CodegenCx<'tcx>,
+ module: &mut dyn Module,
+ instance: Instance<'tcx>,
+ symbol_name: &str,
+ func_id: FuncId,
+ func: Function,
+ mut clif_comments: CommentWriter,
+ source_info_set: IndexSet<SourceInfo>,
+ local_map: IndexVec<mir::Local, CPlace<'tcx>>,
+) {
+ let tcx = cx.tcx;
+
+ // Store function in context
+ let context = &mut cx.cached_context;
+ context.clear();
+ context.func = func;
+
+ // If the return block is not reachable, then the SSA builder may have inserted an `iconst.i128`
+ // instruction, which doesn't have an encoding.
+ context.compute_cfg();
+ context.compute_domtree();
+ context.eliminate_unreachable_code(module.isa()).unwrap();
+ context.dce(module.isa()).unwrap();
+ // Some Cranelift optimizations expect the domtree to not yet be computed and as such don't
+ // invalidate it when it would change.
+ context.domtree.clear();
+
+ // Perform rust specific optimizations
+ tcx.sess.time("optimize clif ir", || {
+ crate::optimize::optimize_function(
+ tcx,
+ module.isa(),
+ instance,
+ context,
+ &mut clif_comments,
+ );
+ });
+
+ // Define function
+ tcx.sess.time("define function", || {
+ context.want_disasm = crate::pretty_clif::should_write_ir(tcx);
+ module.define_function(func_id, context).unwrap()
+ });
+
+ // Write optimized function to file for debugging
+ crate::pretty_clif::write_clif_file(
+ tcx,
+ "opt",
+ module.isa(),
+ instance,
+ &context.func,
+ &clif_comments,
+ );
+
+ if let Some(disasm) = &context.mach_compile_result.as_ref().unwrap().disasm {
+ crate::pretty_clif::write_ir_file(
+ tcx,
+ || format!("{}.vcode", tcx.symbol_name(instance).name),
+ |file| file.write_all(disasm.as_bytes()),
+ )
+ }
+
+ // Define debuginfo for function
+ let isa = module.isa();
+ let debug_context = &mut cx.debug_context;
+ let unwind_context = &mut cx.unwind_context;
+ tcx.sess.time("generate debug info", || {
+ if let Some(debug_context) = debug_context {
+ debug_context.define_function(
+ instance,
+ func_id,
+ symbol_name,
+ isa,
+ context,
+ &source_info_set,
+ local_map,
+ );
+ }
+ unwind_context.add_function(func_id, &context, isa);
+ });
+}
+
+pub(crate) fn verify_func(
+ tcx: TyCtxt<'_>,
+ writer: &crate::pretty_clif::CommentWriter,
+ func: &Function,
+) {
+ tcx.sess.time("verify clif ir", || {
+ let flags = cranelift_codegen::settings::Flags::new(cranelift_codegen::settings::builder());
+ match cranelift_codegen::verify_function(&func, &flags) {
+ Ok(_) => {}
+ Err(err) => {
+ tcx.sess.err(&format!("{:?}", err));
+ let pretty_error = cranelift_codegen::print_errors::pretty_verifier_error(
+ &func,
+ Some(Box::new(writer)),
+ err,
+ );
+ tcx.sess.fatal(&format!("cranelift verify error:\n{}", pretty_error));
+ }
+ }
+ });
+}
+
+fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, '_>) {
+ for (bb, bb_data) in fx.mir.basic_blocks().iter_enumerated() {
+ let block = fx.get_block(bb);
+ fx.bcx.switch_to_block(block);
+
+ if bb_data.is_cleanup {
+ // Unwinding after panicking is not supported
+ continue;
+
+ // FIXME Once unwinding is supported and Cranelift supports marking blocks as cold, do
+ // so for cleanup blocks.
+ }
+
+ fx.bcx.ins().nop();
+ for stmt in &bb_data.statements {
+ fx.set_debug_loc(stmt.source_info);
+ codegen_stmt(fx, block, stmt);
+ }
+
+ if fx.clif_comments.enabled() {
+ let mut terminator_head = "\n".to_string();
- // FIXME Mark failure block as cold once Cranelift supports it
++ with_no_trimmed_paths!({
++ bb_data.terminator().kind.fmt_head(&mut terminator_head).unwrap();
++ });
+ let inst = fx.bcx.func.layout.last_inst(block).unwrap();
+ fx.add_comment(inst, terminator_head);
+ }
+
+ let source_info = bb_data.terminator().source_info;
+ fx.set_debug_loc(source_info);
+
+ match &bb_data.terminator().kind {
+ TerminatorKind::Goto { target } => {
+ if let TerminatorKind::Return = fx.mir[*target].terminator().kind {
+ let mut can_immediately_return = true;
+ for stmt in &fx.mir[*target].statements {
+ if let StatementKind::StorageDead(_) = stmt.kind {
+ } else {
+ // FIXME Can sometimes happen, see rust-lang/rust#70531
+ can_immediately_return = false;
+ break;
+ }
+ }
+
+ if can_immediately_return {
+ crate::abi::codegen_return(fx);
+ continue;
+ }
+ }
+
+ let block = fx.get_block(*target);
+ fx.bcx.ins().jump(block, &[]);
+ }
+ TerminatorKind::Return => {
+ crate::abi::codegen_return(fx);
+ }
+ TerminatorKind::Assert { cond, expected, msg, target, cleanup: _ } => {
+ if !fx.tcx.sess.overflow_checks() {
+ if let mir::AssertKind::OverflowNeg(_) = *msg {
+ let target = fx.get_block(*target);
+ fx.bcx.ins().jump(target, &[]);
+ continue;
+ }
+ }
+ let cond = codegen_operand(fx, cond).load_scalar(fx);
+
+ let target = fx.get_block(*target);
+ let failure = fx.bcx.create_block();
- crate::trap::trap_unreachable(
- fx,
- "[corruption] Returned from noreturn inline asm",
- );
++ fx.bcx.set_cold_block(failure);
+
+ if *expected {
+ fx.bcx.ins().brz(cond, failure, &[]);
+ } else {
+ fx.bcx.ins().brnz(cond, failure, &[]);
+ };
+ fx.bcx.ins().jump(target, &[]);
+
+ fx.bcx.switch_to_block(failure);
+ fx.bcx.ins().nop();
+
+ match msg {
+ AssertKind::BoundsCheck { ref len, ref index } => {
+ let len = codegen_operand(fx, len).load_scalar(fx);
+ let index = codegen_operand(fx, index).load_scalar(fx);
+ let location = fx.get_caller_location(source_info.span).load_scalar(fx);
+
+ codegen_panic_inner(
+ fx,
+ rustc_hir::LangItem::PanicBoundsCheck,
+ &[index, len, location],
+ source_info.span,
+ );
+ }
+ _ => {
+ let msg_str = msg.description();
+ codegen_panic(fx, msg_str, source_info.span);
+ }
+ }
+ }
+
+ TerminatorKind::SwitchInt { discr, switch_ty, targets } => {
+ let discr = codegen_operand(fx, discr).load_scalar(fx);
+
+ let use_bool_opt = switch_ty.kind() == fx.tcx.types.bool.kind()
+ || (targets.iter().count() == 1 && targets.iter().next().unwrap().0 == 0);
+ if use_bool_opt {
+ assert_eq!(targets.iter().count(), 1);
+ let (then_value, then_block) = targets.iter().next().unwrap();
+ let then_block = fx.get_block(then_block);
+ let else_block = fx.get_block(targets.otherwise());
+ let test_zero = match then_value {
+ 0 => true,
+ 1 => false,
+ _ => unreachable!("{:?}", targets),
+ };
+
+ let discr = crate::optimize::peephole::maybe_unwrap_bint(&mut fx.bcx, discr);
+ let (discr, is_inverted) =
+ crate::optimize::peephole::maybe_unwrap_bool_not(&mut fx.bcx, discr);
+ let test_zero = if is_inverted { !test_zero } else { test_zero };
+ let discr = crate::optimize::peephole::maybe_unwrap_bint(&mut fx.bcx, discr);
+ if let Some(taken) = crate::optimize::peephole::maybe_known_branch_taken(
+ &fx.bcx, discr, test_zero,
+ ) {
+ if taken {
+ fx.bcx.ins().jump(then_block, &[]);
+ } else {
+ fx.bcx.ins().jump(else_block, &[]);
+ }
+ } else {
+ if test_zero {
+ fx.bcx.ins().brz(discr, then_block, &[]);
+ fx.bcx.ins().jump(else_block, &[]);
+ } else {
+ fx.bcx.ins().brnz(discr, then_block, &[]);
+ fx.bcx.ins().jump(else_block, &[]);
+ }
+ }
+ } else {
+ let mut switch = ::cranelift_frontend::Switch::new();
+ for (value, block) in targets.iter() {
+ let block = fx.get_block(block);
+ switch.set_entry(value, block);
+ }
+ let otherwise_block = fx.get_block(targets.otherwise());
+ switch.emit(&mut fx.bcx, discr, otherwise_block);
+ }
+ }
+ TerminatorKind::Call {
+ func,
+ args,
+ destination,
+ fn_span,
+ cleanup: _,
+ from_hir_call: _,
+ } => {
+ fx.tcx.sess.time("codegen call", || {
+ crate::abi::codegen_terminator_call(fx, *fn_span, func, args, *destination)
+ });
+ }
+ TerminatorKind::InlineAsm {
+ template,
+ operands,
+ options,
+ destination,
+ line_spans: _,
+ cleanup: _,
+ } => {
+ if options.contains(InlineAsmOptions::MAY_UNWIND) {
+ fx.tcx.sess.span_fatal(
+ source_info.span,
+ "cranelift doesn't support unwinding from inline assembly.",
+ );
+ }
+
+ crate::inline_asm::codegen_inline_asm(
+ fx,
+ source_info.span,
+ template,
+ operands,
+ *options,
+ );
+
+ match *destination {
+ Some(destination) => {
+ let destination_block = fx.get_block(destination);
+ fx.bcx.ins().jump(destination_block, &[]);
+ }
+ None => {
- trap_unreachable(fx, "[corruption] Unwinding bb reached.");
++ fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
+ }
+ }
+ }
+ TerminatorKind::Resume | TerminatorKind::Abort => {
- trap_unreachable(fx, "[corruption] Hit unreachable code.");
++ // FIXME implement unwinding
++ fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
+ }
+ TerminatorKind::Unreachable => {
- cplace = cplace.place_deref(fx);
++ fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
+ }
+ TerminatorKind::Yield { .. }
+ | TerminatorKind::FalseEdge { .. }
+ | TerminatorKind::FalseUnwind { .. }
+ | TerminatorKind::DropAndReplace { .. }
+ | TerminatorKind::GeneratorDrop => {
+ bug!("shouldn't exist at codegen {:?}", bb_data.terminator());
+ }
+ TerminatorKind::Drop { place, target, unwind: _ } => {
+ let drop_place = codegen_place(fx, *place);
+ crate::abi::codegen_drop(fx, source_info.span, drop_place);
+
+ let target_block = fx.get_block(*target);
+ fx.bcx.ins().jump(target_block, &[]);
+ }
+ };
+ }
+
+ fx.bcx.seal_all_blocks();
+ fx.bcx.finalize();
+}
+
+fn codegen_stmt<'tcx>(
+ fx: &mut FunctionCx<'_, '_, 'tcx>,
+ #[allow(unused_variables)] cur_block: Block,
+ stmt: &Statement<'tcx>,
+) {
+ let _print_guard = crate::PrintOnPanic(|| format!("stmt {:?}", stmt));
+
+ fx.set_debug_loc(stmt.source_info);
+
+ #[cfg(disabled)]
+ match &stmt.kind {
+ StatementKind::StorageLive(..) | StatementKind::StorageDead(..) => {} // Those are not very useful
+ _ => {
+ if fx.clif_comments.enabled() {
+ let inst = fx.bcx.func.layout.last_inst(cur_block).unwrap();
+ fx.add_comment(inst, format!("{:?}", stmt));
+ }
+ }
+ }
+
+ match &stmt.kind {
+ StatementKind::SetDiscriminant { place, variant_index } => {
+ let place = codegen_place(fx, **place);
+ crate::discriminant::codegen_set_discriminant(fx, place, *variant_index);
+ }
+ StatementKind::Assign(to_place_and_rval) => {
+ let lval = codegen_place(fx, to_place_and_rval.0);
+ let dest_layout = lval.layout();
+ match to_place_and_rval.1 {
+ Rvalue::Use(ref operand) => {
+ let val = codegen_operand(fx, operand);
+ lval.write_cvalue(fx, val);
+ }
+ Rvalue::Ref(_, _, place) | Rvalue::AddressOf(_, place) => {
+ let place = codegen_place(fx, place);
+ let ref_ = place.place_ref(fx, lval.layout());
+ lval.write_cvalue(fx, ref_);
+ }
+ Rvalue::ThreadLocalRef(def_id) => {
+ let val = crate::constant::codegen_tls_ref(fx, def_id, lval.layout());
+ lval.write_cvalue(fx, val);
+ }
+ Rvalue::BinaryOp(bin_op, ref lhs_rhs) => {
+ let lhs = codegen_operand(fx, &lhs_rhs.0);
+ let rhs = codegen_operand(fx, &lhs_rhs.1);
+
+ let res = crate::num::codegen_binop(fx, bin_op, lhs, rhs);
+ lval.write_cvalue(fx, res);
+ }
+ Rvalue::CheckedBinaryOp(bin_op, ref lhs_rhs) => {
+ let lhs = codegen_operand(fx, &lhs_rhs.0);
+ let rhs = codegen_operand(fx, &lhs_rhs.1);
+
+ let res = if !fx.tcx.sess.overflow_checks() {
+ let val =
+ crate::num::codegen_int_binop(fx, bin_op, lhs, rhs).load_scalar(fx);
+ let is_overflow = fx.bcx.ins().iconst(types::I8, 0);
+ CValue::by_val_pair(val, is_overflow, lval.layout())
+ } else {
+ crate::num::codegen_checked_int_binop(fx, bin_op, lhs, rhs)
+ };
+
+ lval.write_cvalue(fx, res);
+ }
+ Rvalue::UnaryOp(un_op, ref operand) => {
+ let operand = codegen_operand(fx, operand);
+ let layout = operand.layout();
+ let val = operand.load_scalar(fx);
+ let res = match un_op {
+ UnOp::Not => match layout.ty.kind() {
+ ty::Bool => {
+ let res = fx.bcx.ins().icmp_imm(IntCC::Equal, val, 0);
+ CValue::by_val(fx.bcx.ins().bint(types::I8, res), layout)
+ }
+ ty::Uint(_) | ty::Int(_) => {
+ CValue::by_val(fx.bcx.ins().bnot(val), layout)
+ }
+ _ => unreachable!("un op Not for {:?}", layout.ty),
+ },
+ UnOp::Neg => match layout.ty.kind() {
+ ty::Int(IntTy::I128) => {
+ // FIXME remove this case once ineg.i128 works
+ let zero =
+ CValue::const_val(fx, layout, ty::ScalarInt::null(layout.size));
+ crate::num::codegen_int_binop(fx, BinOp::Sub, zero, operand)
+ }
+ ty::Int(_) => CValue::by_val(fx.bcx.ins().ineg(val), layout),
+ ty::Float(_) => CValue::by_val(fx.bcx.ins().fneg(val), layout),
+ _ => unreachable!("un op Neg for {:?}", layout.ty),
+ },
+ };
+ lval.write_cvalue(fx, res);
+ }
+ Rvalue::Cast(
+ CastKind::Pointer(PointerCast::ReifyFnPointer),
+ ref operand,
+ to_ty,
+ ) => {
+ let from_ty = fx.monomorphize(operand.ty(&fx.mir.local_decls, fx.tcx));
+ let to_layout = fx.layout_of(fx.monomorphize(to_ty));
+ match *from_ty.kind() {
+ ty::FnDef(def_id, substs) => {
+ let func_ref = fx.get_function_ref(
+ Instance::resolve_for_fn_ptr(
+ fx.tcx,
+ ParamEnv::reveal_all(),
+ def_id,
+ substs,
+ )
+ .unwrap()
+ .polymorphize(fx.tcx),
+ );
+ let func_addr = fx.bcx.ins().func_addr(fx.pointer_type, func_ref);
+ lval.write_cvalue(fx, CValue::by_val(func_addr, to_layout));
+ }
+ _ => bug!("Trying to ReifyFnPointer on non FnDef {:?}", from_ty),
+ }
+ }
+ Rvalue::Cast(
+ CastKind::Pointer(PointerCast::UnsafeFnPointer),
+ ref operand,
+ to_ty,
+ )
+ | Rvalue::Cast(
+ CastKind::Pointer(PointerCast::MutToConstPointer),
+ ref operand,
+ to_ty,
+ )
+ | Rvalue::Cast(
+ CastKind::Pointer(PointerCast::ArrayToPointer),
+ ref operand,
+ to_ty,
+ ) => {
+ let to_layout = fx.layout_of(fx.monomorphize(to_ty));
+ let operand = codegen_operand(fx, operand);
+ lval.write_cvalue(fx, operand.cast_pointer_to(to_layout));
+ }
+ Rvalue::Cast(CastKind::Misc, ref operand, to_ty) => {
+ let operand = codegen_operand(fx, operand);
+ let from_ty = operand.layout().ty;
+ let to_ty = fx.monomorphize(to_ty);
+
+ fn is_fat_ptr<'tcx>(fx: &FunctionCx<'_, '_, 'tcx>, ty: Ty<'tcx>) -> bool {
+ ty.builtin_deref(true)
+ .map(|ty::TypeAndMut { ty: pointee_ty, mutbl: _ }| {
+ has_ptr_meta(fx.tcx, pointee_ty)
+ })
+ .unwrap_or(false)
+ }
+
+ if is_fat_ptr(fx, from_ty) {
+ if is_fat_ptr(fx, to_ty) {
+ // fat-ptr -> fat-ptr
+ lval.write_cvalue(fx, operand.cast_pointer_to(dest_layout));
+ } else {
+ // fat-ptr -> thin-ptr
+ let (ptr, _extra) = operand.load_scalar_pair(fx);
+ lval.write_cvalue(fx, CValue::by_val(ptr, dest_layout))
+ }
+ } else if let ty::Adt(adt_def, _substs) = from_ty.kind() {
+ // enum -> discriminant value
+ assert!(adt_def.is_enum());
+ match to_ty.kind() {
+ ty::Uint(_) | ty::Int(_) => {}
+ _ => unreachable!("cast adt {} -> {}", from_ty, to_ty),
+ }
+ let to_clif_ty = fx.clif_type(to_ty).unwrap();
+
+ let discriminant = crate::discriminant::codegen_get_discriminant(
+ fx,
+ operand,
+ fx.layout_of(operand.layout().ty.discriminant_ty(fx.tcx)),
+ )
+ .load_scalar(fx);
+
+ let res = crate::cast::clif_intcast(
+ fx,
+ discriminant,
+ to_clif_ty,
+ to_ty.is_signed(),
+ );
+ lval.write_cvalue(fx, CValue::by_val(res, dest_layout));
+ } else {
+ let to_clif_ty = fx.clif_type(to_ty).unwrap();
+ let from = operand.load_scalar(fx);
+
+ let res = clif_int_or_float_cast(
+ fx,
+ from,
+ type_sign(from_ty),
+ to_clif_ty,
+ type_sign(to_ty),
+ );
+ lval.write_cvalue(fx, CValue::by_val(res, dest_layout));
+ }
+ }
+ Rvalue::Cast(
+ CastKind::Pointer(PointerCast::ClosureFnPointer(_)),
+ ref operand,
+ _to_ty,
+ ) => {
+ let operand = codegen_operand(fx, operand);
+ match *operand.layout().ty.kind() {
+ ty::Closure(def_id, substs) => {
+ let instance = Instance::resolve_closure(
+ fx.tcx,
+ def_id,
+ substs,
+ ty::ClosureKind::FnOnce,
+ )
+ .polymorphize(fx.tcx);
+ let func_ref = fx.get_function_ref(instance);
+ let func_addr = fx.bcx.ins().func_addr(fx.pointer_type, func_ref);
+ lval.write_cvalue(fx, CValue::by_val(func_addr, lval.layout()));
+ }
+ _ => bug!("{} cannot be cast to a fn ptr", operand.layout().ty),
+ }
+ }
+ Rvalue::Cast(CastKind::Pointer(PointerCast::Unsize), ref operand, _to_ty) => {
+ let operand = codegen_operand(fx, operand);
+ operand.unsize_value(fx, lval);
+ }
+ Rvalue::Discriminant(place) => {
+ let place = codegen_place(fx, place);
+ let value = place.to_cvalue(fx);
+ let discr =
+ crate::discriminant::codegen_get_discriminant(fx, value, dest_layout);
+ lval.write_cvalue(fx, discr);
+ }
+ Rvalue::Repeat(ref operand, times) => {
+ let operand = codegen_operand(fx, operand);
+ let times = fx
+ .monomorphize(times)
+ .eval(fx.tcx, ParamEnv::reveal_all())
+ .val()
+ .try_to_bits(fx.tcx.data_layout.pointer_size)
+ .unwrap();
+ if operand.layout().size.bytes() == 0 {
+ // Do nothing for ZST's
+ } else if fx.clif_type(operand.layout().ty) == Some(types::I8) {
+ let times = fx.bcx.ins().iconst(fx.pointer_type, times as i64);
+ // FIXME use emit_small_memset where possible
+ let addr = lval.to_ptr().get_addr(fx);
+ let val = operand.load_scalar(fx);
+ fx.bcx.call_memset(fx.target_config, addr, val, times);
+ } else {
+ let loop_block = fx.bcx.create_block();
+ let loop_block2 = fx.bcx.create_block();
+ let done_block = fx.bcx.create_block();
+ let index = fx.bcx.append_block_param(loop_block, fx.pointer_type);
+ let zero = fx.bcx.ins().iconst(fx.pointer_type, 0);
+ fx.bcx.ins().jump(loop_block, &[zero]);
+
+ fx.bcx.switch_to_block(loop_block);
+ let done = fx.bcx.ins().icmp_imm(IntCC::Equal, index, times as i64);
+ fx.bcx.ins().brnz(done, done_block, &[]);
+ fx.bcx.ins().jump(loop_block2, &[]);
+
+ fx.bcx.switch_to_block(loop_block2);
+ let to = lval.place_index(fx, index);
+ to.write_cvalue(fx, operand);
+ let index = fx.bcx.ins().iadd_imm(index, 1);
+ fx.bcx.ins().jump(loop_block, &[index]);
+
+ fx.bcx.switch_to_block(done_block);
+ fx.bcx.ins().nop();
+ }
+ }
+ Rvalue::Len(place) => {
+ let place = codegen_place(fx, place);
+ let usize_layout = fx.layout_of(fx.tcx.types.usize);
+ let len = codegen_array_len(fx, place);
+ lval.write_cvalue(fx, CValue::by_val(len, usize_layout));
+ }
+ Rvalue::ShallowInitBox(ref operand, content_ty) => {
+ let content_ty = fx.monomorphize(content_ty);
+ let box_layout = fx.layout_of(fx.tcx.mk_box(content_ty));
+ let operand = codegen_operand(fx, operand);
+ let operand = operand.load_scalar(fx);
+ lval.write_cvalue(fx, CValue::by_val(operand, box_layout));
+ }
+ Rvalue::NullaryOp(null_op, ty) => {
+ assert!(
+ lval.layout()
+ .ty
+ .is_sized(fx.tcx.at(stmt.source_info.span), ParamEnv::reveal_all())
+ );
+ let layout = fx.layout_of(fx.monomorphize(ty));
+ let val = match null_op {
+ NullOp::SizeOf => layout.size.bytes(),
+ NullOp::AlignOf => layout.align.abi.bytes(),
+ };
+ let val = CValue::const_val(fx, fx.layout_of(fx.tcx.types.usize), val.into());
+ lval.write_cvalue(fx, val);
+ }
+ Rvalue::Aggregate(ref kind, ref operands) => match kind.as_ref() {
+ AggregateKind::Array(_ty) => {
+ for (i, operand) in operands.iter().enumerate() {
+ let operand = codegen_operand(fx, operand);
+ let index = fx.bcx.ins().iconst(fx.pointer_type, i as i64);
+ let to = lval.place_index(fx, index);
+ to.write_cvalue(fx, operand);
+ }
+ }
+ _ => unreachable!("shouldn't exist at codegen {:?}", to_place_and_rval.1),
+ },
+ }
+ }
+ StatementKind::StorageLive(_)
+ | StatementKind::StorageDead(_)
+ | StatementKind::Nop
+ | StatementKind::FakeRead(..)
+ | StatementKind::Retag { .. }
+ | StatementKind::AscribeUserType(..) => {}
+
+ StatementKind::Coverage { .. } => fx.tcx.sess.fatal("-Zcoverage is unimplemented"),
+ StatementKind::CopyNonOverlapping(inner) => {
+ let dst = codegen_operand(fx, &inner.dst);
+ let pointee = dst
+ .layout()
+ .pointee_info_at(fx, rustc_target::abi::Size::ZERO)
+ .expect("Expected pointer");
+ let dst = dst.load_scalar(fx);
+ let src = codegen_operand(fx, &inner.src).load_scalar(fx);
+ let count = codegen_operand(fx, &inner.count).load_scalar(fx);
+ let elem_size: u64 = pointee.size.bytes();
+ let bytes =
+ if elem_size != 1 { fx.bcx.ins().imul_imm(count, elem_size as i64) } else { count };
+ fx.bcx.call_memcpy(fx.target_config, dst, src, bytes);
+ }
+ }
+}
+
+fn codegen_array_len<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, place: CPlace<'tcx>) -> Value {
+ match *place.layout().ty.kind() {
+ ty::Array(_elem_ty, len) => {
+ let len = fx.monomorphize(len).eval_usize(fx.tcx, ParamEnv::reveal_all()) as i64;
+ fx.bcx.ins().iconst(fx.pointer_type, len)
+ }
+ ty::Slice(_elem_ty) => {
+ place.to_ptr_maybe_unsized().1.expect("Length metadata for slice place")
+ }
+ _ => bug!("Rvalue::Len({:?})", place),
+ }
+}
+
+pub(crate) fn codegen_place<'tcx>(
+ fx: &mut FunctionCx<'_, '_, 'tcx>,
+ place: Place<'tcx>,
+) -> CPlace<'tcx> {
+ let mut cplace = fx.get_local_place(place.local);
+
+ for elem in place.projection {
+ match elem {
+ PlaceElem::Deref => {
- crate::trap::trap_unreachable(fx, "panic lang item returned");
++ if cplace.layout().ty.is_box() {
++ cplace = cplace
++ .place_field(fx, Field::new(0)) // Box<T> -> Unique<T>
++ .place_field(fx, Field::new(0)) // Unique<T> -> *const T
++ .place_deref(fx);
++ } else {
++ cplace = cplace.place_deref(fx);
++ }
+ }
+ PlaceElem::Field(field, _ty) => {
+ cplace = cplace.place_field(fx, field);
+ }
+ PlaceElem::Index(local) => {
+ let index = fx.get_local_place(local).to_cvalue(fx).load_scalar(fx);
+ cplace = cplace.place_index(fx, index);
+ }
+ PlaceElem::ConstantIndex { offset, min_length: _, from_end } => {
+ let offset: u64 = offset;
+ let index = if !from_end {
+ fx.bcx.ins().iconst(fx.pointer_type, offset as i64)
+ } else {
+ let len = codegen_array_len(fx, cplace);
+ fx.bcx.ins().iadd_imm(len, -(offset as i64))
+ };
+ cplace = cplace.place_index(fx, index);
+ }
+ PlaceElem::Subslice { from, to, from_end } => {
+ // These indices are generated by slice patterns.
+ // slice[from:-to] in Python terms.
+
+ let from: u64 = from;
+ let to: u64 = to;
+
+ match cplace.layout().ty.kind() {
+ ty::Array(elem_ty, _len) => {
+ assert!(!from_end, "array subslices are never `from_end`");
+ let elem_layout = fx.layout_of(*elem_ty);
+ let ptr = cplace.to_ptr();
+ cplace = CPlace::for_ptr(
+ ptr.offset_i64(fx, elem_layout.size.bytes() as i64 * (from as i64)),
+ fx.layout_of(fx.tcx.mk_array(*elem_ty, to - from)),
+ );
+ }
+ ty::Slice(elem_ty) => {
+ assert!(from_end, "slice subslices should be `from_end`");
+ let elem_layout = fx.layout_of(*elem_ty);
+ let (ptr, len) = cplace.to_ptr_maybe_unsized();
+ let len = len.unwrap();
+ cplace = CPlace::for_ptr_with_extra(
+ ptr.offset_i64(fx, elem_layout.size.bytes() as i64 * (from as i64)),
+ fx.bcx.ins().iadd_imm(len, -(from as i64 + to as i64)),
+ cplace.layout(),
+ );
+ }
+ _ => unreachable!(),
+ }
+ }
+ PlaceElem::Downcast(_adt_def, variant) => {
+ cplace = cplace.downcast_variant(fx, variant);
+ }
+ }
+ }
+
+ cplace
+}
+
+pub(crate) fn codegen_operand<'tcx>(
+ fx: &mut FunctionCx<'_, '_, 'tcx>,
+ operand: &Operand<'tcx>,
+) -> CValue<'tcx> {
+ match operand {
+ Operand::Move(place) | Operand::Copy(place) => {
+ let cplace = codegen_place(fx, *place);
+ cplace.to_cvalue(fx)
+ }
+ Operand::Constant(const_) => crate::constant::codegen_constant(fx, const_),
+ }
+}
+
+pub(crate) fn codegen_panic<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, msg_str: &str, span: Span) {
+ let location = fx.get_caller_location(span).load_scalar(fx);
+
+ let msg_ptr = fx.anonymous_str(msg_str);
+ let msg_len = fx.bcx.ins().iconst(fx.pointer_type, i64::try_from(msg_str.len()).unwrap());
+ let args = [msg_ptr, msg_len, location];
+
+ codegen_panic_inner(fx, rustc_hir::LangItem::Panic, &args, span);
+}
+
+pub(crate) fn codegen_panic_inner<'tcx>(
+ fx: &mut FunctionCx<'_, '_, 'tcx>,
+ lang_item: rustc_hir::LangItem,
+ args: &[Value],
+ span: Span,
+) {
+ let def_id =
+ fx.tcx.lang_items().require(lang_item).unwrap_or_else(|s| fx.tcx.sess.span_fatal(span, &s));
+
+ let instance = Instance::mono(fx.tcx, def_id).polymorphize(fx.tcx);
+ let symbol_name = fx.tcx.symbol_name(instance).name;
+
+ fx.lib_call(
+ &*symbol_name,
+ vec![
+ AbiParam::new(fx.pointer_type),
+ AbiParam::new(fx.pointer_type),
+ AbiParam::new(fx.pointer_type),
+ ],
+ vec![],
+ args,
+ );
+
++ fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
+}
--- /dev/null
- #![feature(rustc_private, once_cell)]
++#![feature(rustc_private)]
+#![warn(rust_2018_idioms)]
+#![warn(unused_lifetimes)]
+#![warn(unreachable_pub)]
+
+extern crate rustc_data_structures;
+extern crate rustc_driver;
+extern crate rustc_interface;
+extern crate rustc_session;
+extern crate rustc_target;
+
- use std::lazy::SyncLazy;
+use std::panic;
+
+use rustc_data_structures::profiling::{get_resident_set_size, print_time_passes_entry};
+use rustc_interface::interface;
- use rustc_session::config::ErrorOutputType;
++use rustc_session::config::{ErrorOutputType, TrimmedDefPaths};
+use rustc_session::early_error;
+use rustc_target::spec::PanicStrategy;
+
++// FIXME use std::lazy::SyncLazy once it stabilizes
++use once_cell::sync::Lazy;
++
+const BUG_REPORT_URL: &str = "https://github.com/bjorn3/rustc_codegen_cranelift/issues/new";
+
- static DEFAULT_HOOK: SyncLazy<Box<dyn Fn(&panic::PanicInfo<'_>) + Sync + Send + 'static>> =
- SyncLazy::new(|| {
++static DEFAULT_HOOK: Lazy<Box<dyn Fn(&panic::PanicInfo<'_>) + Sync + Send + 'static>> =
++ Lazy::new(|| {
+ let hook = panic::take_hook();
+ panic::set_hook(Box::new(|info| {
+ // Invoke the default handler, which prints the actual panic message and optionally a backtrace
+ (*DEFAULT_HOOK)(info);
+
+ // Separate the output with an empty line
+ eprintln!();
+
+ // Print the ICE message
+ rustc_driver::report_ice(info, BUG_REPORT_URL);
+ }));
+ hook
+ });
+
+#[derive(Default)]
+pub struct CraneliftPassesCallbacks {
+ time_passes: bool,
+}
+
+impl rustc_driver::Callbacks for CraneliftPassesCallbacks {
+ fn config(&mut self, config: &mut interface::Config) {
+ // If a --prints=... option has been given, we don't print the "total"
+ // time because it will mess up the --prints output. See #64339.
+ self.time_passes = config.opts.prints.is_empty()
+ && (config.opts.debugging_opts.time_passes || config.opts.debugging_opts.time);
+
+ config.opts.cg.panic = Some(PanicStrategy::Abort);
+ config.opts.debugging_opts.panic_abort_tests = true;
+ config.opts.maybe_sysroot = Some(config.opts.maybe_sysroot.clone().unwrap_or_else(|| {
+ std::env::current_exe().unwrap().parent().unwrap().parent().unwrap().to_owned()
+ }));
++
++ config.opts.trimmed_def_paths = TrimmedDefPaths::GoodPath;
+ }
+}
+
+fn main() {
+ let start_time = std::time::Instant::now();
+ let start_rss = get_resident_set_size();
+ rustc_driver::init_rustc_env_logger();
+ let mut callbacks = CraneliftPassesCallbacks::default();
- SyncLazy::force(&DEFAULT_HOOK); // Install ice hook
++ Lazy::force(&DEFAULT_HOOK); // Install ice hook
+ let exit_code = rustc_driver::catch_with_exit_code(|| {
+ let args = std::env::args_os()
+ .enumerate()
+ .map(|(i, arg)| {
+ arg.into_string().unwrap_or_else(|arg| {
+ early_error(
+ ErrorOutputType::default(),
+ &format!("Argument {} is not valid Unicode: {:?}", i, arg),
+ )
+ })
+ })
+ .collect::<Vec<_>>();
+ let mut run_compiler = rustc_driver::RunCompiler::new(&args, &mut callbacks);
+ run_compiler.set_make_codegen_backend(Some(Box::new(move |_| {
+ Box::new(rustc_codegen_cranelift::CraneliftCodegenBackend { config: None })
+ })));
+ run_compiler.run()
+ });
+
+ if callbacks.time_passes {
+ let end_rss = get_resident_set_size();
+ print_time_passes_entry("total", start_time.elapsed(), start_rss, end_rss);
+ }
+
+ std::process::exit(exit_code)
+}
--- /dev/null
- macro builtin_functions($register:ident; $(fn $name:ident($($arg_name:ident: $arg_ty:ty),*) -> $ret_ty:ty;)*) {
- #[cfg(feature = "jit")]
- #[allow(improper_ctypes)]
- extern "C" {
- $(fn $name($($arg_name: $arg_ty),*) -> $ret_ty;)*
- }
++macro_rules! builtin_functions {
++ ($register:ident; $(fn $name:ident($($arg_name:ident: $arg_ty:ty),*) -> $ret_ty:ty;)*) => {
++ #[cfg(feature = "jit")]
++ #[allow(improper_ctypes)]
++ extern "C" {
++ $(fn $name($($arg_name: $arg_ty),*) -> $ret_ty;)*
++ }
+
- #[cfg(feature = "jit")]
- pub(crate) fn $register(builder: &mut cranelift_jit::JITBuilder) {
- for (name, val) in [$((stringify!($name), $name as *const u8)),*] {
- builder.symbol(name, val);
++ #[cfg(feature = "jit")]
++ pub(crate) fn $register(builder: &mut cranelift_jit::JITBuilder) {
++ for (name, val) in [$((stringify!($name), $name as *const u8)),*] {
++ builder.symbol(name, val);
++ }
+ }
- }
++ };
+}
+
+builtin_functions! {
+ register_functions_for_jit;
+
+ // integers
+ fn __multi3(a: i128, b: i128) -> i128;
+ fn __udivti3(n: u128, d: u128) -> u128;
+ fn __divti3(n: i128, d: i128) -> i128;
+ fn __umodti3(n: u128, d: u128) -> u128;
+ fn __modti3(n: i128, d: i128) -> i128;
+ fn __rust_u128_addo(a: u128, b: u128) -> (u128, bool);
+ fn __rust_i128_addo(a: i128, b: i128) -> (i128, bool);
+ fn __rust_u128_subo(a: u128, b: u128) -> (u128, bool);
+ fn __rust_i128_subo(a: i128, b: i128) -> (i128, bool);
+ fn __rust_u128_mulo(a: u128, b: u128) -> (u128, bool);
+ fn __rust_i128_mulo(a: i128, b: i128) -> (i128, bool);
+
+ // floats
+ fn __floattisf(i: i128) -> f32;
+ fn __floattidf(i: i128) -> f64;
+ fn __floatuntisf(i: u128) -> f32;
+ fn __floatuntidf(i: u128) -> f64;
+ fn __fixsfti(f: f32) -> i128;
+ fn __fixdfti(f: f64) -> i128;
+ fn __fixunssfti(f: f32) -> u128;
+ fn __fixunsdfti(f: f64) -> u128;
+}
--- /dev/null
- // FIXME: probably omit this
+//! Line info generation (`.debug_line`)
+
+use std::ffi::OsStr;
+use std::path::{Component, Path};
+
+use crate::prelude::*;
+
+use rustc_span::{
+ FileName, Pos, SourceFile, SourceFileAndLine, SourceFileHash, SourceFileHashAlgorithm,
+};
+
+use cranelift_codegen::binemit::CodeOffset;
+use cranelift_codegen::MachSrcLoc;
+
+use gimli::write::{
+ Address, AttributeValue, FileId, FileInfo, LineProgram, LineString, LineStringTable,
+ UnitEntryId,
+};
+
+// OPTIMIZATION: It is cheaper to do this in one pass than using `.parent()` and `.file_name()`.
+fn split_path_dir_and_file(path: &Path) -> (&Path, &OsStr) {
+ let mut iter = path.components();
+ let file_name = match iter.next_back() {
+ Some(Component::Normal(p)) => p,
+ component => {
+ panic!(
+ "Path component {:?} of path {} is an invalid filename",
+ component,
+ path.display()
+ );
+ }
+ };
+ let parent = iter.as_path();
+ (parent, file_name)
+}
+
+// OPTIMIZATION: Avoid UTF-8 validation on UNIX.
+fn osstr_as_utf8_bytes(path: &OsStr) -> &[u8] {
+ #[cfg(unix)]
+ {
+ use std::os::unix::ffi::OsStrExt;
+ path.as_bytes()
+ }
+ #[cfg(not(unix))]
+ {
+ path.to_str().unwrap().as_bytes()
+ }
+}
+
+pub(crate) const MD5_LEN: usize = 16;
+
+pub(crate) fn make_file_info(hash: SourceFileHash) -> Option<FileInfo> {
+ if hash.kind == SourceFileHashAlgorithm::Md5 {
+ let mut buf = [0u8; MD5_LEN];
+ buf.copy_from_slice(hash.hash_bytes());
+ Some(FileInfo { timestamp: 0, size: 0, md5: buf })
+ } else {
+ None
+ }
+}
+
+fn line_program_add_file(
+ line_program: &mut LineProgram,
+ line_strings: &mut LineStringTable,
+ file: &SourceFile,
+) -> FileId {
+ match &file.name {
+ FileName::Real(path) => {
+ let (dir_path, file_name) = split_path_dir_and_file(path.remapped_path_if_available());
+ let dir_name = osstr_as_utf8_bytes(dir_path.as_os_str());
+ let file_name = osstr_as_utf8_bytes(file_name);
+
+ let dir_id = if !dir_name.is_empty() {
+ let dir_name = LineString::new(dir_name, line_program.encoding(), line_strings);
+ line_program.add_directory(dir_name)
+ } else {
+ line_program.default_directory()
+ };
+ let file_name = LineString::new(file_name, line_program.encoding(), line_strings);
+
+ let info = make_file_info(file.src_hash);
+
+ line_program.file_has_md5 &= info.is_some();
+ line_program.add_file(file_name, dir_id, info)
+ }
+ // FIXME give more appropriate file names
+ filename => {
+ let dir_id = line_program.default_directory();
+ let dummy_file_name = LineString::new(
+ filename.prefer_remapped().to_string().into_bytes(),
+ line_program.encoding(),
+ line_strings,
+ );
+ line_program.add_file(dummy_file_name, dir_id, None)
+ }
+ }
+}
+
+impl<'tcx> DebugContext<'tcx> {
+ pub(super) fn emit_location(&mut self, entry_id: UnitEntryId, span: Span) {
+ let loc = self.tcx.sess.source_map().lookup_char_pos(span.lo());
+
+ let file_id = line_program_add_file(
+ &mut self.dwarf.unit.line_program,
+ &mut self.dwarf.line_strings,
+ &loc.file,
+ );
+
+ let entry = self.dwarf.unit.get_mut(entry_id);
+
+ entry.set(gimli::DW_AT_decl_file, AttributeValue::FileIndex(Some(file_id)));
+ entry.set(gimli::DW_AT_decl_line, AttributeValue::Udata(loc.line as u64));
+ entry.set(gimli::DW_AT_decl_column, AttributeValue::Udata(loc.col.to_usize() as u64));
+ }
+
+ pub(super) fn create_debug_lines(
+ &mut self,
+ symbol: usize,
+ entry_id: UnitEntryId,
+ context: &Context,
+ function_span: Span,
+ source_info_set: &indexmap::IndexSet<SourceInfo>,
+ ) -> CodeOffset {
+ let tcx = self.tcx;
+ let line_program = &mut self.dwarf.unit.line_program;
+
+ let line_strings = &mut self.dwarf.line_strings;
+ let mut last_span = None;
+ let mut last_file = None;
+ let mut create_row_for_span = |line_program: &mut LineProgram, span: Span| {
+ if let Some(last_span) = last_span {
+ if span == last_span {
+ line_program.generate_row();
+ return;
+ }
+ }
+ last_span = Some(span);
+
+ // Based on https://github.com/rust-lang/rust/blob/e369d87b015a84653343032833d65d0545fd3f26/src/librustc_codegen_ssa/mir/mod.rs#L116-L131
+ // In order to have a good line stepping behavior in debugger, we overwrite debug
+ // locations of macro expansions with that of the outermost expansion site
+ // (unless the crate is being compiled with `-Z debug-macros`).
+ let span = if !span.from_expansion() || tcx.sess.opts.debugging_opts.debug_macros {
+ span
+ } else {
+ // Walk up the macro expansion chain until we reach a non-expanded span.
+ // We also stop at the function body level because no line stepping can occur
+ // at the level above that.
+ rustc_span::hygiene::walk_chain(span, function_span.ctxt())
+ };
+
+ let (file, line, col) = match tcx.sess.source_map().lookup_line(span.lo()) {
+ Ok(SourceFileAndLine { sf: file, line }) => {
+ let line_pos = file.line_begin_pos(span.lo());
+
+ (
+ file,
+ u64::try_from(line).unwrap() + 1,
+ u64::from((span.lo() - line_pos).to_u32()) + 1,
+ )
+ }
+ Err(file) => (file, 0, 0),
+ };
+
+ // line_program_add_file is very slow.
+ // Optimize for the common case of the current file not being changed.
+ let current_file_changed = if let Some(last_file) = &last_file {
+ // If the allocations are not equal, then the files may still be equal, but that
+ // is not a problem, as this is just an optimization.
+ !rustc_data_structures::sync::Lrc::ptr_eq(last_file, &file)
+ } else {
+ true
+ };
+ if current_file_changed {
+ let file_id = line_program_add_file(line_program, line_strings, &file);
+ line_program.row().file = file_id;
+ last_file = Some(file);
+ }
+
+ line_program.row().line = line;
+ line_program.row().column = col;
+ line_program.generate_row();
+ };
+
+ line_program.begin_sequence(Some(Address::Symbol { symbol, addend: 0 }));
+
+ let mut func_end = 0;
+
+ let mcr = context.mach_compile_result.as_ref().unwrap();
+ for &MachSrcLoc { start, end, loc } in mcr.buffer.get_srclocs_sorted() {
+ line_program.row().address_offset = u64::from(start);
+ if !loc.is_default() {
+ let source_info = *source_info_set.get_index(loc.bits() as usize).unwrap();
+ create_row_for_span(line_program, source_info.span);
+ } else {
+ create_row_for_span(line_program, function_span);
+ }
+ func_end = end;
+ }
+
+ line_program.end_sequence(u64::from(func_end));
+
+ let func_end = mcr.buffer.total_size();
+
+ assert_ne!(func_end, 0);
+
+ let entry = self.dwarf.unit.get_mut(entry_id);
+ entry.set(
+ gimli::DW_AT_low_pc,
+ AttributeValue::Address(Address::Symbol { symbol, addend: 0 }),
+ );
+ entry.set(gimli::DW_AT_high_pc, AttributeValue::Udata(u64::from(func_end)));
+
+ self.emit_location(entry_id, function_span);
+
+ func_end
+ }
+}
--- /dev/null
- #[allow(unused_variables)]
- let (eh_frame, eh_frame_len, _) = Vec::into_raw_parts(eh_frame);
+//! Unwind info generation (`.eh_frame`)
+
+use crate::prelude::*;
+
+use cranelift_codegen::ir::Endianness;
+use cranelift_codegen::isa::{unwind::UnwindInfo, TargetIsa};
+
+use cranelift_object::ObjectProduct;
+use gimli::write::{Address, CieId, EhFrame, FrameTable, Section};
+use gimli::RunTimeEndian;
+
+use super::object::WriteDebugInfo;
+
+pub(crate) struct UnwindContext {
+ endian: RunTimeEndian,
+ frame_table: FrameTable,
+ cie_id: Option<CieId>,
+}
+
+impl UnwindContext {
+ pub(crate) fn new(isa: &dyn TargetIsa, pic_eh_frame: bool) -> Self {
+ let endian = match isa.endianness() {
+ Endianness::Little => RunTimeEndian::Little,
+ Endianness::Big => RunTimeEndian::Big,
+ };
+ let mut frame_table = FrameTable::default();
+
+ let cie_id = if let Some(mut cie) = isa.create_systemv_cie() {
+ if pic_eh_frame {
+ cie.fde_address_encoding =
+ gimli::DwEhPe(gimli::DW_EH_PE_pcrel.0 | gimli::DW_EH_PE_sdata4.0);
+ }
+ Some(frame_table.add_cie(cie))
+ } else {
+ None
+ };
+
+ UnwindContext { endian, frame_table, cie_id }
+ }
+
+ pub(crate) fn add_function(&mut self, func_id: FuncId, context: &Context, isa: &dyn TargetIsa) {
+ let unwind_info = if let Some(unwind_info) = context.create_unwind_info(isa).unwrap() {
+ unwind_info
+ } else {
+ return;
+ };
+
+ match unwind_info {
+ UnwindInfo::SystemV(unwind_info) => {
+ self.frame_table.add_fde(
+ self.cie_id.unwrap(),
+ unwind_info
+ .to_fde(Address::Symbol { symbol: func_id.as_u32() as usize, addend: 0 }),
+ );
+ }
+ UnwindInfo::WindowsX64(_) => {
+ // FIXME implement this
+ }
+ unwind_info => unimplemented!("{:?}", unwind_info),
+ }
+ }
+
+ pub(crate) fn emit(self, product: &mut ObjectProduct) {
+ let mut eh_frame = EhFrame::from(super::emit::WriterRelocate::new(self.endian));
+ self.frame_table.write_eh_frame(&mut eh_frame).unwrap();
+
+ if !eh_frame.0.writer.slice().is_empty() {
+ let id = eh_frame.id();
+ let section_id = product.add_debug_section(id, eh_frame.0.writer.into_vec());
+ let mut section_map = FxHashMap::default();
+ section_map.insert(id, section_id);
+
+ for reloc in &eh_frame.0.relocs {
+ product.add_debug_reloc(§ion_map, §ion_id, reloc);
+ }
+ }
+ }
+
+ #[cfg(all(feature = "jit", windows))]
+ pub(crate) unsafe fn register_jit(self, _jit_module: &cranelift_jit::JITModule) {}
+
+ #[cfg(all(feature = "jit", not(windows)))]
+ pub(crate) unsafe fn register_jit(self, jit_module: &cranelift_jit::JITModule) {
++ use std::mem::ManuallyDrop;
++
+ let mut eh_frame = EhFrame::from(super::emit::WriterRelocate::new(self.endian));
+ self.frame_table.write_eh_frame(&mut eh_frame).unwrap();
+
+ if eh_frame.0.writer.slice().is_empty() {
+ return;
+ }
+
+ let mut eh_frame = eh_frame.0.relocate_for_jit(jit_module);
+
+ // GCC expects a terminating "empty" length, so write a 0 length at the end of the table.
+ eh_frame.extend(&[0, 0, 0, 0]);
+
+ // FIXME support unregistering unwind tables once cranelift-jit supports deallocating
+ // individual functions
- // Everything after this line up to the end of the file is loosly based on
++ let eh_frame = ManuallyDrop::new(eh_frame);
+
+ // =======================================================================
- let start = eh_frame;
- let end = start.add(eh_frame_len);
++ // Everything after this line up to the end of the file is loosely based on
+ // https://github.com/bytecodealliance/wasmtime/blob/4471a82b0c540ff48960eca6757ccce5b1b5c3e4/crates/jit/src/unwind/systemv.rs
+ #[cfg(target_os = "macos")]
+ {
+ // On macOS, `__register_frame` takes a pointer to a single FDE
- __register_frame(eh_frame);
++ let start = eh_frame.as_ptr();
++ let end = start.add(eh_frame.len());
+ let mut current = start;
+
+ // Walk all of the entries in the frame table and register them
+ while current < end {
+ let len = std::ptr::read::<u32>(current as *const u32) as usize;
+
+ // Skip over the CIE
+ if current != start {
+ __register_frame(current);
+ }
+
+ // Move to the next table entry (+4 because the length itself is not inclusive)
+ current = current.add(len + 4);
+ }
+ }
+ #[cfg(not(target_os = "macos"))]
+ {
+ // On other platforms, `__register_frame` will walk the FDEs until an entry of length 0
++ __register_frame(eh_frame.as_ptr());
+ }
+ }
+}
+
+extern "C" {
+ // libunwind import
+ fn __register_frame(fde: *const u8);
+}
--- /dev/null
- return trap_unreachable_ret_value(
- fx,
- dest_layout,
- "[panic] Tried to get discriminant for uninhabited type.",
- );
+//! Handling of enum discriminants
+//!
+//! Adapted from <https://github.com/rust-lang/rust/blob/d760df5aea483aae041c9a241e7acacf48f75035/src/librustc_codegen_ssa/mir/place.rs>
+
+use rustc_target::abi::{Int, TagEncoding, Variants};
+
+use crate::prelude::*;
+
+pub(crate) fn codegen_set_discriminant<'tcx>(
+ fx: &mut FunctionCx<'_, '_, 'tcx>,
+ place: CPlace<'tcx>,
+ variant_index: VariantIdx,
+) {
+ let layout = place.layout();
+ if layout.for_variant(fx, variant_index).abi.is_uninhabited() {
+ return;
+ }
+ match layout.variants {
+ Variants::Single { index } => {
+ assert_eq!(index, variant_index);
+ }
+ Variants::Multiple {
+ tag: _,
+ tag_field,
+ tag_encoding: TagEncoding::Direct,
+ variants: _,
+ } => {
+ let ptr = place.place_field(fx, mir::Field::new(tag_field));
+ let to = layout.ty.discriminant_for_variant(fx.tcx, variant_index).unwrap().val;
+ let to = if ptr.layout().abi.is_signed() {
+ ty::ScalarInt::try_from_int(
+ ptr.layout().size.sign_extend(to) as i128,
+ ptr.layout().size,
+ )
+ .unwrap()
+ } else {
+ ty::ScalarInt::try_from_uint(to, ptr.layout().size).unwrap()
+ };
+ let discr = CValue::const_val(fx, ptr.layout(), to);
+ ptr.write_cvalue(fx, discr);
+ }
+ Variants::Multiple {
+ tag: _,
+ tag_field,
+ tag_encoding: TagEncoding::Niche { dataful_variant, ref niche_variants, niche_start },
+ variants: _,
+ } => {
+ if variant_index != dataful_variant {
+ let niche = place.place_field(fx, mir::Field::new(tag_field));
+ let niche_value = variant_index.as_u32() - niche_variants.start().as_u32();
+ let niche_value = ty::ScalarInt::try_from_uint(
+ u128::from(niche_value).wrapping_add(niche_start),
+ niche.layout().size,
+ )
+ .unwrap();
+ let niche_llval = CValue::const_val(fx, niche.layout(), niche_value);
+ niche.write_cvalue(fx, niche_llval);
+ }
+ }
+ }
+}
+
+pub(crate) fn codegen_get_discriminant<'tcx>(
+ fx: &mut FunctionCx<'_, '_, 'tcx>,
+ value: CValue<'tcx>,
+ dest_layout: TyAndLayout<'tcx>,
+) -> CValue<'tcx> {
+ let layout = value.layout();
+
+ if layout.abi == Abi::Uninhabited {
++ let true_ = fx.bcx.ins().iconst(types::I32, 1);
++ fx.bcx.ins().trapnz(true_, TrapCode::UnreachableCodeReached);
++ // Return a dummy value
++ return CValue::by_ref(Pointer::const_addr(fx, 0), dest_layout);
+ }
+
+ let (tag_scalar, tag_field, tag_encoding) = match &layout.variants {
+ Variants::Single { index } => {
+ let discr_val = layout
+ .ty
+ .discriminant_for_variant(fx.tcx, *index)
+ .map_or(u128::from(index.as_u32()), |discr| discr.val);
+ let discr_val = if dest_layout.abi.is_signed() {
+ ty::ScalarInt::try_from_int(
+ dest_layout.size.sign_extend(discr_val) as i128,
+ dest_layout.size,
+ )
+ .unwrap()
+ } else {
+ ty::ScalarInt::try_from_uint(discr_val, dest_layout.size).unwrap()
+ };
+ return CValue::const_val(fx, dest_layout, discr_val);
+ }
+ Variants::Multiple { tag, tag_field, tag_encoding, variants: _ } => {
+ (tag, *tag_field, tag_encoding)
+ }
+ };
+
+ let cast_to = fx.clif_type(dest_layout.ty).unwrap();
+
+ // Read the tag/niche-encoded discriminant from memory.
+ let tag = value.value_field(fx, mir::Field::new(tag_field));
+ let tag = tag.load_scalar(fx);
+
+ // Decode the discriminant (specifically if it's niche-encoded).
+ match *tag_encoding {
+ TagEncoding::Direct => {
+ let signed = match tag_scalar.value {
+ Int(_, signed) => signed,
+ _ => false,
+ };
+ let val = clif_intcast(fx, tag, cast_to, signed);
+ CValue::by_val(val, dest_layout)
+ }
+ TagEncoding::Niche { dataful_variant, ref niche_variants, niche_start } => {
+ // Rebase from niche values to discriminants, and check
+ // whether the result is in range for the niche variants.
+
+ // We first compute the "relative discriminant" (wrt `niche_variants`),
+ // that is, if `n = niche_variants.end() - niche_variants.start()`,
+ // we remap `niche_start..=niche_start + n` (which may wrap around)
+ // to (non-wrap-around) `0..=n`, to be able to check whether the
+ // discriminant corresponds to a niche variant with one comparison.
+ // We also can't go directly to the (variant index) discriminant
+ // and check that it is in the range `niche_variants`, because
+ // that might not fit in the same type, on top of needing an extra
+ // comparison (see also the comment on `let niche_discr`).
+ let relative_discr = if niche_start == 0 {
+ tag
+ } else {
+ // FIXME handle niche_start > i64::MAX
+ fx.bcx.ins().iadd_imm(tag, -i64::try_from(niche_start).unwrap())
+ };
+ let relative_max = niche_variants.end().as_u32() - niche_variants.start().as_u32();
+ let is_niche = {
+ codegen_icmp_imm(
+ fx,
+ IntCC::UnsignedLessThanOrEqual,
+ relative_discr,
+ i128::from(relative_max),
+ )
+ };
+
+ // NOTE(eddyb) this addition needs to be performed on the final
+ // type, in case the niche itself can't represent all variant
+ // indices (e.g. `u8` niche with more than `256` variants,
+ // but enough uninhabited variants so that the remaining variants
+ // fit in the niche).
+ // In other words, `niche_variants.end - niche_variants.start`
+ // is representable in the niche, but `niche_variants.end`
+ // might not be, in extreme cases.
+ let niche_discr = {
+ let relative_discr = if relative_max == 0 {
+ // HACK(eddyb) since we have only one niche, we know which
+ // one it is, and we can avoid having a dynamic value here.
+ fx.bcx.ins().iconst(cast_to, 0)
+ } else {
+ clif_intcast(fx, relative_discr, cast_to, false)
+ };
+ fx.bcx.ins().iadd_imm(relative_discr, i64::from(niche_variants.start().as_u32()))
+ };
+
+ let dataful_variant = fx.bcx.ins().iconst(cast_to, i64::from(dataful_variant.as_u32()));
+ let discr = fx.bcx.ins().select(is_niche, niche_discr, dataful_variant);
+ CValue::by_val(discr, dest_layout)
+ }
+ }
+}
--- /dev/null
- use std::lazy::SyncOnceCell;
+//! The JIT driver uses [`cranelift_jit`] to JIT execute programs without writing any object
+//! files.
+
+use std::cell::RefCell;
+use std::ffi::CString;
- static GLOBAL_MESSAGE_SENDER: SyncOnceCell<Mutex<mpsc::Sender<UnsafeMessage>>> =
- SyncOnceCell::new();
+use std::os::raw::{c_char, c_int};
+use std::sync::{mpsc, Mutex};
+
+use rustc_codegen_ssa::CrateInfo;
+use rustc_middle::mir::mono::MonoItem;
+use rustc_session::Session;
+use rustc_span::Symbol;
+
+use cranelift_jit::{JITBuilder, JITModule};
+
++// FIXME use std::lazy::SyncOnceCell once it stabilizes
++use once_cell::sync::OnceCell;
++
+use crate::{prelude::*, BackendConfig};
+use crate::{CodegenCx, CodegenMode};
+
+struct JitState {
+ backend_config: BackendConfig,
+ jit_module: JITModule,
+}
+
+thread_local! {
+ static LAZY_JIT_STATE: RefCell<Option<JitState>> = const { RefCell::new(None) };
+}
+
+/// The Sender owned by the rustc thread
++static GLOBAL_MESSAGE_SENDER: OnceCell<Mutex<mpsc::Sender<UnsafeMessage>>> = OnceCell::new();
+
+/// A message that is sent from the jitted runtime to the rustc thread.
+/// Senders are responsible for upholding `Send` semantics.
+enum UnsafeMessage {
+ /// Request that the specified `Instance` be lazily jitted.
+ ///
+ /// Nothing accessible through `instance_ptr` may be moved or mutated by the sender after
+ /// this message is sent.
+ JitFn {
+ instance_ptr: *const Instance<'static>,
+ trampoline_ptr: *const u8,
+ tx: mpsc::Sender<*const u8>,
+ },
+}
+unsafe impl Send for UnsafeMessage {}
+
+impl UnsafeMessage {
+ /// Send the message.
+ fn send(self) -> Result<(), mpsc::SendError<UnsafeMessage>> {
+ thread_local! {
+ /// The Sender owned by the local thread
+ static LOCAL_MESSAGE_SENDER: mpsc::Sender<UnsafeMessage> =
+ GLOBAL_MESSAGE_SENDER
+ .get().unwrap()
+ .lock().unwrap()
+ .clone();
+ }
+ LOCAL_MESSAGE_SENDER.with(|sender| sender.send(self))
+ }
+}
+
+fn create_jit_module<'tcx>(
+ tcx: TyCtxt<'tcx>,
+ backend_config: &BackendConfig,
+ hotswap: bool,
+) -> (JITModule, CodegenCx<'tcx>) {
+ let crate_info = CrateInfo::new(tcx, "dummy_target_cpu".to_string());
+ let imported_symbols = load_imported_symbols_for_jit(tcx.sess, crate_info);
+
+ let isa = crate::build_isa(tcx.sess, backend_config);
+ let mut jit_builder = JITBuilder::with_isa(isa, cranelift_module::default_libcall_names());
+ jit_builder.hotswap(hotswap);
+ crate::compiler_builtins::register_functions_for_jit(&mut jit_builder);
+ jit_builder.symbols(imported_symbols);
+ let mut jit_module = JITModule::new(jit_builder);
+
+ let mut cx = crate::CodegenCx::new(
+ tcx,
+ backend_config.clone(),
+ jit_module.isa(),
+ false,
+ Symbol::intern("dummy_cgu_name"),
+ );
+
+ crate::allocator::codegen(tcx, &mut jit_module, &mut cx.unwind_context);
+ crate::main_shim::maybe_create_entry_wrapper(
+ tcx,
+ &mut jit_module,
+ &mut cx.unwind_context,
+ true,
+ true,
+ );
+
+ (jit_module, cx)
+}
+
+pub(crate) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! {
+ if !tcx.sess.opts.output_types.should_codegen() {
+ tcx.sess.fatal("JIT mode doesn't work with `cargo check`");
+ }
+
+ if !tcx.sess.crate_types().contains(&rustc_session::config::CrateType::Executable) {
+ tcx.sess.fatal("can't jit non-executable crate");
+ }
+
+ let (mut jit_module, mut cx) = create_jit_module(
+ tcx,
+ &backend_config,
+ matches!(backend_config.codegen_mode, CodegenMode::JitLazy),
+ );
+
+ let (_, cgus) = tcx.collect_and_partition_mono_items(());
+ let mono_items = cgus
+ .iter()
+ .map(|cgu| cgu.items_in_deterministic_order(tcx).into_iter())
+ .flatten()
+ .collect::<FxHashMap<_, (_, _)>>()
+ .into_iter()
+ .collect::<Vec<(_, (_, _))>>();
+
+ super::time(tcx, backend_config.display_cg_time, "codegen mono items", || {
+ super::predefine_mono_items(tcx, &mut jit_module, &mono_items);
+ for (mono_item, _) in mono_items {
+ match mono_item {
+ MonoItem::Fn(inst) => match backend_config.codegen_mode {
+ CodegenMode::Aot => unreachable!(),
+ CodegenMode::Jit => {
+ cx.tcx.sess.time("codegen fn", || {
+ crate::base::codegen_fn(&mut cx, &mut jit_module, inst)
+ });
+ }
+ CodegenMode::JitLazy => codegen_shim(&mut cx, &mut jit_module, inst),
+ },
+ MonoItem::Static(def_id) => {
+ crate::constant::codegen_static(tcx, &mut jit_module, def_id);
+ }
+ MonoItem::GlobalAsm(item_id) => {
+ let item = tcx.hir().item(item_id);
+ tcx.sess.span_fatal(item.span, "Global asm is not supported in JIT mode");
+ }
+ }
+ }
+ });
+
+ if !cx.global_asm.is_empty() {
+ tcx.sess.fatal("Inline asm is not supported in JIT mode");
+ }
+
+ tcx.sess.abort_if_errors();
+
+ jit_module.finalize_definitions();
+ unsafe { cx.unwind_context.register_jit(&jit_module) };
+
+ println!(
+ "Rustc codegen cranelift will JIT run the executable, because -Cllvm-args=mode=jit was passed"
+ );
+
+ let args = std::iter::once(&*tcx.crate_name(LOCAL_CRATE).as_str().to_string())
+ .chain(backend_config.jit_args.iter().map(|arg| &**arg))
+ .map(|arg| CString::new(arg).unwrap())
+ .collect::<Vec<_>>();
+
+ let start_sig = Signature {
+ params: vec![
+ AbiParam::new(jit_module.target_config().pointer_type()),
+ AbiParam::new(jit_module.target_config().pointer_type()),
+ ],
+ returns: vec![AbiParam::new(jit_module.target_config().pointer_type() /*isize*/)],
+ call_conv: jit_module.target_config().default_call_conv,
+ };
+ let start_func_id = jit_module.declare_function("main", Linkage::Import, &start_sig).unwrap();
+ let finalized_start: *const u8 = jit_module.get_finalized_function(start_func_id);
+
+ LAZY_JIT_STATE.with(|lazy_jit_state| {
+ let mut lazy_jit_state = lazy_jit_state.borrow_mut();
+ assert!(lazy_jit_state.is_none());
+ *lazy_jit_state = Some(JitState { backend_config, jit_module });
+ });
+
+ let f: extern "C" fn(c_int, *const *const c_char) -> c_int =
+ unsafe { ::std::mem::transmute(finalized_start) };
+
+ let (tx, rx) = mpsc::channel();
+ GLOBAL_MESSAGE_SENDER.set(Mutex::new(tx)).unwrap();
+
+ // Spawn the jitted runtime in a new thread so that this rustc thread can handle messages
+ // (eg to lazily JIT further functions as required)
+ std::thread::spawn(move || {
+ let mut argv = args.iter().map(|arg| arg.as_ptr()).collect::<Vec<_>>();
+
+ // Push a null pointer as a terminating argument. This is required by POSIX and
+ // useful as some dynamic linkers use it as a marker to jump over.
+ argv.push(std::ptr::null());
+
+ let ret = f(args.len() as c_int, argv.as_ptr());
+ std::process::exit(ret);
+ });
+
+ // Handle messages
+ loop {
+ match rx.recv().unwrap() {
+ // lazy JIT compilation request - compile requested instance and return pointer to result
+ UnsafeMessage::JitFn { instance_ptr, trampoline_ptr, tx } => {
+ tx.send(jit_fn(instance_ptr, trampoline_ptr))
+ .expect("jitted runtime hung up before response to lazy JIT request was sent");
+ }
+ }
+ }
+}
+
+#[no_mangle]
+extern "C" fn __clif_jit_fn(
+ instance_ptr: *const Instance<'static>,
+ trampoline_ptr: *const u8,
+) -> *const u8 {
+ // send the JIT request to the rustc thread, with a channel for the response
+ let (tx, rx) = mpsc::channel();
+ UnsafeMessage::JitFn { instance_ptr, trampoline_ptr, tx }
+ .send()
+ .expect("rustc thread hung up before lazy JIT request was sent");
+
+ // block on JIT compilation result
+ rx.recv().expect("rustc thread hung up before responding to sent lazy JIT request")
+}
+
+fn jit_fn(instance_ptr: *const Instance<'static>, trampoline_ptr: *const u8) -> *const u8 {
+ rustc_middle::ty::tls::with(|tcx| {
+ // lift is used to ensure the correct lifetime for instance.
+ let instance = tcx.lift(unsafe { *instance_ptr }).unwrap();
+
+ LAZY_JIT_STATE.with(|lazy_jit_state| {
+ let mut lazy_jit_state = lazy_jit_state.borrow_mut();
+ let lazy_jit_state = lazy_jit_state.as_mut().unwrap();
+ let jit_module = &mut lazy_jit_state.jit_module;
+ let backend_config = lazy_jit_state.backend_config.clone();
+
+ let name = tcx.symbol_name(instance).name;
+ let sig = crate::abi::get_function_sig(tcx, jit_module.isa().triple(), instance);
+ let func_id = jit_module.declare_function(name, Linkage::Export, &sig).unwrap();
+
+ let current_ptr = jit_module.read_got_entry(func_id);
+
+ // If the function's GOT entry has already been updated to point at something other
+ // than the shim trampoline, don't re-jit but just return the new pointer instead.
+ // This does not need synchronization as this code is executed only by a sole rustc
+ // thread.
+ if current_ptr != trampoline_ptr {
+ return current_ptr;
+ }
+
+ jit_module.prepare_for_function_redefine(func_id).unwrap();
+
+ let mut cx = crate::CodegenCx::new(
+ tcx,
+ backend_config,
+ jit_module.isa(),
+ false,
+ Symbol::intern("dummy_cgu_name"),
+ );
+ tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, jit_module, instance));
+
+ assert!(cx.global_asm.is_empty());
+ jit_module.finalize_definitions();
+ unsafe { cx.unwind_context.register_jit(&jit_module) };
+ jit_module.get_finalized_function(func_id)
+ })
+ })
+}
+
+fn load_imported_symbols_for_jit(
+ sess: &Session,
+ crate_info: CrateInfo,
+) -> Vec<(String, *const u8)> {
+ use rustc_middle::middle::dependency_format::Linkage;
+
+ let mut dylib_paths = Vec::new();
+
+ let data = &crate_info
+ .dependency_formats
+ .iter()
+ .find(|(crate_type, _data)| *crate_type == rustc_session::config::CrateType::Executable)
+ .unwrap()
+ .1;
+ for &cnum in &crate_info.used_crates {
+ let src = &crate_info.used_crate_source[&cnum];
+ match data[cnum.as_usize() - 1] {
+ Linkage::NotLinked | Linkage::IncludedFromDylib => {}
+ Linkage::Static => {
+ let name = &crate_info.crate_name[&cnum];
+ let mut err = sess.struct_err(&format!("Can't load static lib {}", name.as_str()));
+ err.note("rustc_codegen_cranelift can only load dylibs in JIT mode.");
+ err.emit();
+ }
+ Linkage::Dynamic => {
+ dylib_paths.push(src.dylib.as_ref().unwrap().0.clone());
+ }
+ }
+ }
+
+ let mut imported_symbols = Vec::new();
+ for path in dylib_paths {
+ use object::{Object, ObjectSymbol};
+ let lib = libloading::Library::new(&path).unwrap();
+ let obj = std::fs::read(path).unwrap();
+ let obj = object::File::parse(&*obj).unwrap();
+ imported_symbols.extend(obj.dynamic_symbols().filter_map(|symbol| {
+ let name = symbol.name().unwrap().to_string();
+ if name.is_empty() || !symbol.is_global() || symbol.is_undefined() {
+ return None;
+ }
+ if name.starts_with("rust_metadata_") {
+ // The metadata is part of a section that is not loaded by the dynamic linker in
+ // case of cg_llvm.
+ return None;
+ }
+ let dlsym_name = if cfg!(target_os = "macos") {
+ // On macOS `dlsym` expects the name without leading `_`.
+ assert!(name.starts_with('_'), "{:?}", name);
+ &name[1..]
+ } else {
+ &name
+ };
+ let symbol: libloading::Symbol<'_, *const u8> =
+ unsafe { lib.get(dlsym_name.as_bytes()) }.unwrap();
+ Some((name, *symbol))
+ }));
+ std::mem::forget(lib)
+ }
+
+ sess.abort_if_errors();
+
+ imported_symbols
+}
+
+fn codegen_shim<'tcx>(cx: &mut CodegenCx<'tcx>, module: &mut JITModule, inst: Instance<'tcx>) {
+ let tcx = cx.tcx;
+
+ let pointer_type = module.target_config().pointer_type();
+
+ let name = tcx.symbol_name(inst).name;
+ let sig = crate::abi::get_function_sig(tcx, module.isa().triple(), inst);
+ let func_id = module.declare_function(name, Linkage::Export, &sig).unwrap();
+
+ let instance_ptr = Box::into_raw(Box::new(inst));
+
+ let jit_fn = module
+ .declare_function(
+ "__clif_jit_fn",
+ Linkage::Import,
+ &Signature {
+ call_conv: module.target_config().default_call_conv,
+ params: vec![AbiParam::new(pointer_type), AbiParam::new(pointer_type)],
+ returns: vec![AbiParam::new(pointer_type)],
+ },
+ )
+ .unwrap();
+
+ cx.cached_context.clear();
+ let trampoline = &mut cx.cached_context.func;
+ trampoline.signature = sig.clone();
+
+ let mut builder_ctx = FunctionBuilderContext::new();
+ let mut trampoline_builder = FunctionBuilder::new(trampoline, &mut builder_ctx);
+
+ let trampoline_fn = module.declare_func_in_func(func_id, trampoline_builder.func);
+ let jit_fn = module.declare_func_in_func(jit_fn, trampoline_builder.func);
+ let sig_ref = trampoline_builder.func.import_signature(sig);
+
+ let entry_block = trampoline_builder.create_block();
+ trampoline_builder.append_block_params_for_function_params(entry_block);
+ let fn_args = trampoline_builder.func.dfg.block_params(entry_block).to_vec();
+
+ trampoline_builder.switch_to_block(entry_block);
+ let instance_ptr = trampoline_builder.ins().iconst(pointer_type, instance_ptr as u64 as i64);
+ let trampoline_ptr = trampoline_builder.ins().func_addr(pointer_type, trampoline_fn);
+ let jitted_fn = trampoline_builder.ins().call(jit_fn, &[instance_ptr, trampoline_ptr]);
+ let jitted_fn = trampoline_builder.func.dfg.inst_results(jitted_fn)[0];
+ let call_inst = trampoline_builder.ins().call_indirect(sig_ref, jitted_fn, &fn_args);
+ let ret_vals = trampoline_builder.func.dfg.inst_results(call_inst).to_vec();
+ trampoline_builder.ins().return_(&ret_vals);
+
+ module.define_function(func_id, &mut cx.cached_context).unwrap();
+}
--- /dev/null
- if let Some((_, dest)) = destination {
- let ret_block = fx.get_block(dest);
- fx.bcx.ins().jump(ret_block, &[]);
- } else {
- trap_unreachable(fx, "[corruption] Diverging intrinsic returned.");
- }
+//! Emulate LLVM intrinsics
+
+use crate::intrinsics::*;
+use crate::prelude::*;
+
+use rustc_middle::ty::subst::SubstsRef;
+
+pub(crate) fn codegen_llvm_intrinsic_call<'tcx>(
+ fx: &mut FunctionCx<'_, '_, 'tcx>,
+ intrinsic: &str,
+ _substs: SubstsRef<'tcx>,
+ args: &[mir::Operand<'tcx>],
+ destination: Option<(CPlace<'tcx>, BasicBlock)>,
+) {
+ let ret = destination.unwrap().0;
+
+ intrinsic_match! {
+ fx, intrinsic, args,
+ _ => {
+ fx.tcx.sess.warn(&format!("unsupported llvm intrinsic {}; replacing with trap", intrinsic));
+ crate::trap::trap_unimplemented(fx, intrinsic);
+ };
+
+ // Used by `_mm_movemask_epi8` and `_mm256_movemask_epi8`
+ "llvm.x86.sse2.pmovmskb.128" | "llvm.x86.avx2.pmovmskb" | "llvm.x86.sse2.movmsk.pd", (c a) {
+ let (lane_count, lane_ty) = a.layout().ty.simd_size_and_type(fx.tcx);
+ let lane_ty = fx.clif_type(lane_ty).unwrap();
+ assert!(lane_count <= 32);
+
+ let mut res = fx.bcx.ins().iconst(types::I32, 0);
+
+ for lane in (0..lane_count).rev() {
+ let a_lane = a.value_field(fx, mir::Field::new(lane.try_into().unwrap())).load_scalar(fx);
+
+ // cast float to int
+ let a_lane = match lane_ty {
+ types::F32 => fx.bcx.ins().bitcast(types::I32, a_lane),
+ types::F64 => fx.bcx.ins().bitcast(types::I64, a_lane),
+ _ => a_lane,
+ };
+
+ // extract sign bit of an int
+ let a_lane_sign = fx.bcx.ins().ushr_imm(a_lane, i64::from(lane_ty.bits() - 1));
+
+ // shift sign bit into result
+ let a_lane_sign = clif_intcast(fx, a_lane_sign, types::I32, false);
+ res = fx.bcx.ins().ishl_imm(res, 1);
+ res = fx.bcx.ins().bor(res, a_lane_sign);
+ }
+
+ let res = CValue::by_val(res, fx.layout_of(fx.tcx.types.i32));
+ ret.write_cvalue(fx, res);
+ };
+ "llvm.x86.sse2.cmp.ps" | "llvm.x86.sse2.cmp.pd", (c x, c y, o kind) {
+ let kind = crate::constant::mir_operand_get_const_val(fx, kind).expect("llvm.x86.sse2.cmp.* kind not const");
+ let flt_cc = match kind.try_to_bits(Size::from_bytes(1)).unwrap_or_else(|| panic!("kind not scalar: {:?}", kind)) {
+ 0 => FloatCC::Equal,
+ 1 => FloatCC::LessThan,
+ 2 => FloatCC::LessThanOrEqual,
+ 7 => {
+ unimplemented!("Compares corresponding elements in `a` and `b` to see if neither is `NaN`.");
+ }
+ 3 => {
+ unimplemented!("Compares corresponding elements in `a` and `b` to see if either is `NaN`.");
+ }
+ 4 => FloatCC::NotEqual,
+ 5 => {
+ unimplemented!("not less than");
+ }
+ 6 => {
+ unimplemented!("not less than or equal");
+ }
+ kind => unreachable!("kind {:?}", kind),
+ };
+
+ simd_pair_for_each_lane(fx, x, y, ret, &|fx, lane_ty, res_lane_ty, x_lane, y_lane| {
+ let res_lane = match lane_ty.kind() {
+ ty::Float(_) => fx.bcx.ins().fcmp(flt_cc, x_lane, y_lane),
+ _ => unreachable!("{:?}", lane_ty),
+ };
+ bool_to_zero_or_max_uint(fx, res_lane_ty, res_lane)
+ });
+ };
+ "llvm.x86.sse2.psrli.d", (c a, o imm8) {
+ let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8).expect("llvm.x86.sse2.psrli.d imm8 not const");
+ simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| {
+ match imm8.try_to_bits(Size::from_bytes(4)).unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) {
+ imm8 if imm8 < 32 => fx.bcx.ins().ushr_imm(lane, i64::from(imm8 as u8)),
+ _ => fx.bcx.ins().iconst(types::I32, 0),
+ }
+ });
+ };
+ "llvm.x86.sse2.pslli.d", (c a, o imm8) {
+ let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8).expect("llvm.x86.sse2.psrli.d imm8 not const");
+ simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| {
+ match imm8.try_to_bits(Size::from_bytes(4)).unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) {
+ imm8 if imm8 < 32 => fx.bcx.ins().ishl_imm(lane, i64::from(imm8 as u8)),
+ _ => fx.bcx.ins().iconst(types::I32, 0),
+ }
+ });
+ };
+ "llvm.x86.sse2.storeu.dq", (v mem_addr, c a) {
+ // FIXME correctly handle the unalignment
+ let dest = CPlace::for_ptr(Pointer::new(mem_addr), a.layout());
+ dest.write_cvalue(fx, a);
+ };
+ "llvm.x86.addcarry.64", (v c_in, c a, c b) {
+ llvm_add_sub(
+ fx,
+ BinOp::Add,
+ ret,
+ c_in,
+ a,
+ b
+ );
+ };
+ "llvm.x86.subborrow.64", (v b_in, c a, c b) {
+ llvm_add_sub(
+ fx,
+ BinOp::Sub,
+ ret,
+ b_in,
+ a,
+ b
+ );
+ };
+ }
+
++ let dest = destination.expect("all llvm intrinsics used by stdlib should return").1;
++ let ret_block = fx.get_block(dest);
++ fx.bcx.ins().jump(ret_block, &[]);
+}
+
+// llvm.x86.avx2.vperm2i128
+// llvm.x86.ssse3.pshuf.b.128
+// llvm.x86.avx2.pshuf.b
+// llvm.x86.avx2.psrli.w
+// llvm.x86.sse2.psrli.w
+
+fn llvm_add_sub<'tcx>(
+ fx: &mut FunctionCx<'_, '_, 'tcx>,
+ bin_op: BinOp,
+ ret: CPlace<'tcx>,
+ cb_in: Value,
+ a: CValue<'tcx>,
+ b: CValue<'tcx>,
+) {
+ assert_eq!(
+ a.layout().ty,
+ fx.tcx.types.u64,
+ "llvm.x86.addcarry.64/llvm.x86.subborrow.64 second operand must be u64"
+ );
+ assert_eq!(
+ b.layout().ty,
+ fx.tcx.types.u64,
+ "llvm.x86.addcarry.64/llvm.x86.subborrow.64 third operand must be u64"
+ );
+
+ // c + carry -> c + first intermediate carry or borrow respectively
+ let int0 = crate::num::codegen_checked_int_binop(fx, bin_op, a, b);
+ let c = int0.value_field(fx, mir::Field::new(0));
+ let cb0 = int0.value_field(fx, mir::Field::new(1)).load_scalar(fx);
+
+ // c + carry -> c + second intermediate carry or borrow respectively
+ let cb_in_as_u64 = fx.bcx.ins().uextend(types::I64, cb_in);
+ let cb_in_as_u64 = CValue::by_val(cb_in_as_u64, fx.layout_of(fx.tcx.types.u64));
+ let int1 = crate::num::codegen_checked_int_binop(fx, bin_op, c, cb_in_as_u64);
+ let (c, cb1) = int1.load_scalar_pair(fx);
+
+ // carry0 | carry1 -> carry or borrow respectively
+ let cb_out = fx.bcx.ins().bor(cb0, cb1);
+
+ let layout = fx.layout_of(fx.tcx.mk_tup([fx.tcx.types.u8, fx.tcx.types.u64].iter()));
+ let val = CValue::by_val_pair(cb_out, c, layout);
+ ret.write_cvalue(fx, val);
+}
--- /dev/null
- mod cpuid;
- mod llvm;
- mod simd;
-
- pub(crate) use cpuid::codegen_cpuid_call;
- pub(crate) use llvm::codegen_llvm_intrinsic_call;
-
- use rustc_middle::ty::print::with_no_trimmed_paths;
- use rustc_middle::ty::subst::SubstsRef;
- use rustc_span::symbol::{kw, sym, Symbol};
-
- use crate::prelude::*;
- use cranelift_codegen::ir::AtomicRmwOp;
-
- macro intrinsic_pat {
+//! Codegen of intrinsics. This includes `extern "rust-intrinsic"`, `extern "platform-intrinsic"`
+//! and LLVM intrinsics that have symbol names starting with `llvm.`.
+
- },
++macro_rules! intrinsic_pat {
+ (_) => {
+ _
- },
++ };
+ ($name:ident) => {
+ sym::$name
- },
++ };
+ (kw.$name:ident) => {
+ kw::$name
- },
++ };
+ ($name:literal) => {
+ $name
- macro intrinsic_arg {
- (o $fx:expr, $arg:ident) => {},
++ };
+}
+
- },
++macro_rules! intrinsic_arg {
++ (o $fx:expr, $arg:ident) => {};
+ (c $fx:expr, $arg:ident) => {
+ let $arg = codegen_operand($fx, $arg);
- }
++ };
+ (v $fx:expr, $arg:ident) => {
+ let $arg = codegen_operand($fx, $arg).load_scalar($fx);
- macro intrinsic_match {
++ };
+}
+
- trap_abort(fx, "Called intrinsic::abort.");
++macro_rules! intrinsic_match {
+ ($fx:expr, $intrinsic:expr, $args:expr,
+ _ => $unknown:block;
+ $(
+ $($($name:tt).*)|+ $(if $cond:expr)?, ($($a:ident $arg:ident),*) $content:block;
+ )*) => {
+ match $intrinsic {
+ $(
+ $(intrinsic_pat!($($name).*))|* $(if $cond)? => {
+ if let [$($arg),*] = $args {
+ $(intrinsic_arg!($a $fx, $arg);)*
+ $content
+ } else {
+ bug!("wrong number of args for intrinsic {:?}", $intrinsic);
+ }
+ }
+ )*
+ _ => $unknown,
+ }
+ }
+}
+
++mod cpuid;
++mod llvm;
++mod simd;
++
++pub(crate) use cpuid::codegen_cpuid_call;
++pub(crate) use llvm::codegen_llvm_intrinsic_call;
++
++use rustc_middle::ty::print::with_no_trimmed_paths;
++use rustc_middle::ty::subst::SubstsRef;
++use rustc_span::symbol::{kw, sym, Symbol};
++
++use crate::prelude::*;
++use cranelift_codegen::ir::AtomicRmwOp;
++
+fn report_atomic_type_validation_error<'tcx>(
+ fx: &mut FunctionCx<'_, '_, 'tcx>,
+ intrinsic: Symbol,
+ span: Span,
+ ty: Ty<'tcx>,
+) {
+ fx.tcx.sess.span_err(
+ span,
+ &format!(
+ "`{}` intrinsic: expected basic integer or raw pointer type, found `{:?}`",
+ intrinsic, ty
+ ),
+ );
+ // Prevent verifier error
+ crate::trap::trap_unreachable(fx, "compilation should not have succeeded");
+}
+
+pub(crate) fn clif_vector_type<'tcx>(tcx: TyCtxt<'tcx>, layout: TyAndLayout<'tcx>) -> Option<Type> {
+ let (element, count) = match layout.abi {
+ Abi::Vector { element, count } => (element, count),
+ _ => unreachable!(),
+ };
+
+ match scalar_to_clif_type(tcx, element).by(u16::try_from(count).unwrap()) {
+ // Cranelift currently only implements icmp for 128bit vectors.
+ Some(vector_ty) if vector_ty.bits() == 128 => Some(vector_ty),
+ _ => None,
+ }
+}
+
+fn simd_for_each_lane<'tcx>(
+ fx: &mut FunctionCx<'_, '_, 'tcx>,
+ val: CValue<'tcx>,
+ ret: CPlace<'tcx>,
+ f: &dyn Fn(&mut FunctionCx<'_, '_, 'tcx>, Ty<'tcx>, Ty<'tcx>, Value) -> Value,
+) {
+ let layout = val.layout();
+
+ let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx);
+ let lane_layout = fx.layout_of(lane_ty);
+ let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx);
+ let ret_lane_layout = fx.layout_of(ret_lane_ty);
+ assert_eq!(lane_count, ret_lane_count);
+
+ for lane_idx in 0..lane_count {
+ let lane = val.value_lane(fx, lane_idx).load_scalar(fx);
+
+ let res_lane = f(fx, lane_layout.ty, ret_lane_layout.ty, lane);
+ let res_lane = CValue::by_val(res_lane, ret_lane_layout);
+
+ ret.place_lane(fx, lane_idx).write_cvalue(fx, res_lane);
+ }
+}
+
+fn simd_pair_for_each_lane<'tcx>(
+ fx: &mut FunctionCx<'_, '_, 'tcx>,
+ x: CValue<'tcx>,
+ y: CValue<'tcx>,
+ ret: CPlace<'tcx>,
+ f: &dyn Fn(&mut FunctionCx<'_, '_, 'tcx>, Ty<'tcx>, Ty<'tcx>, Value, Value) -> Value,
+) {
+ assert_eq!(x.layout(), y.layout());
+ let layout = x.layout();
+
+ let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx);
+ let lane_layout = fx.layout_of(lane_ty);
+ let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx);
+ let ret_lane_layout = fx.layout_of(ret_lane_ty);
+ assert_eq!(lane_count, ret_lane_count);
+
+ for lane_idx in 0..lane_count {
+ let x_lane = x.value_lane(fx, lane_idx).load_scalar(fx);
+ let y_lane = y.value_lane(fx, lane_idx).load_scalar(fx);
+
+ let res_lane = f(fx, lane_layout.ty, ret_lane_layout.ty, x_lane, y_lane);
+ let res_lane = CValue::by_val(res_lane, ret_lane_layout);
+
+ ret.place_lane(fx, lane_idx).write_cvalue(fx, res_lane);
+ }
+}
+
+fn simd_reduce<'tcx>(
+ fx: &mut FunctionCx<'_, '_, 'tcx>,
+ val: CValue<'tcx>,
+ acc: Option<Value>,
+ ret: CPlace<'tcx>,
+ f: &dyn Fn(&mut FunctionCx<'_, '_, 'tcx>, Ty<'tcx>, Value, Value) -> Value,
+) {
+ let (lane_count, lane_ty) = val.layout().ty.simd_size_and_type(fx.tcx);
+ let lane_layout = fx.layout_of(lane_ty);
+ assert_eq!(lane_layout, ret.layout());
+
+ let (mut res_val, start_lane) =
+ if let Some(acc) = acc { (acc, 0) } else { (val.value_lane(fx, 0).load_scalar(fx), 1) };
+ for lane_idx in start_lane..lane_count {
+ let lane = val.value_lane(fx, lane_idx).load_scalar(fx);
+ res_val = f(fx, lane_layout.ty, res_val, lane);
+ }
+ let res = CValue::by_val(res_val, lane_layout);
+ ret.write_cvalue(fx, res);
+}
+
+// FIXME move all uses to `simd_reduce`
+fn simd_reduce_bool<'tcx>(
+ fx: &mut FunctionCx<'_, '_, 'tcx>,
+ val: CValue<'tcx>,
+ ret: CPlace<'tcx>,
+ f: &dyn Fn(&mut FunctionCx<'_, '_, 'tcx>, Value, Value) -> Value,
+) {
+ let (lane_count, _lane_ty) = val.layout().ty.simd_size_and_type(fx.tcx);
+ assert!(ret.layout().ty.is_bool());
+
+ let res_val = val.value_lane(fx, 0).load_scalar(fx);
+ let mut res_val = fx.bcx.ins().band_imm(res_val, 1); // mask to boolean
+ for lane_idx in 1..lane_count {
+ let lane = val.value_lane(fx, lane_idx).load_scalar(fx);
+ let lane = fx.bcx.ins().band_imm(lane, 1); // mask to boolean
+ res_val = f(fx, res_val, lane);
+ }
+ let res_val = if fx.bcx.func.dfg.value_type(res_val) != types::I8 {
+ fx.bcx.ins().ireduce(types::I8, res_val)
+ } else {
+ res_val
+ };
+ let res = CValue::by_val(res_val, ret.layout());
+ ret.write_cvalue(fx, res);
+}
+
+fn bool_to_zero_or_max_uint<'tcx>(
+ fx: &mut FunctionCx<'_, '_, 'tcx>,
+ ty: Ty<'tcx>,
+ val: Value,
+) -> Value {
+ let ty = fx.clif_type(ty).unwrap();
+
+ let int_ty = match ty {
+ types::F32 => types::I32,
+ types::F64 => types::I64,
+ ty => ty,
+ };
+
+ let val = fx.bcx.ins().bint(int_ty, val);
+ let mut res = fx.bcx.ins().ineg(val);
+
+ if ty.is_float() {
+ res = fx.bcx.ins().bitcast(ty, res);
+ }
+
+ res
+}
+
+pub(crate) fn codegen_intrinsic_call<'tcx>(
+ fx: &mut FunctionCx<'_, '_, 'tcx>,
+ instance: Instance<'tcx>,
+ args: &[mir::Operand<'tcx>],
+ destination: Option<(CPlace<'tcx>, BasicBlock)>,
+ span: Span,
+) {
+ let intrinsic = fx.tcx.item_name(instance.def_id());
+ let substs = instance.substs;
+
+ let ret = match destination {
+ Some((place, _)) => place,
+ None => {
+ // Insert non returning intrinsics here
+ match intrinsic {
+ sym::abort => {
- if let Some((_, dest)) = destination {
- let ret_block = fx.get_block(dest);
- fx.bcx.ins().jump(ret_block, &[]);
- } else {
- trap_unreachable(fx, "[corruption] Diverging intrinsic returned.");
- }
++ fx.bcx.ins().trap(TrapCode::User(0));
+ }
+ sym::transmute => {
+ crate::base::codegen_panic(fx, "Transmuting to uninhabited type.", span);
+ }
+ _ => unimplemented!("unsupported instrinsic {}", intrinsic),
+ }
+ return;
+ }
+ };
+
+ if intrinsic.as_str().starts_with("simd_") {
+ self::simd::codegen_simd_intrinsic_call(fx, intrinsic, substs, args, ret, span);
+ let ret_block = fx.get_block(destination.expect("SIMD intrinsics don't diverge").1);
+ fx.bcx.ins().jump(ret_block, &[]);
+ } else if codegen_float_intrinsic_call(fx, intrinsic, args, ret) {
+ let ret_block = fx.get_block(destination.expect("Float intrinsics don't diverge").1);
+ fx.bcx.ins().jump(ret_block, &[]);
+ } else {
+ codegen_regular_intrinsic_call(
+ fx,
+ instance,
+ intrinsic,
+ substs,
+ args,
+ ret,
+ span,
+ destination,
+ );
+ }
+}
+
+fn codegen_float_intrinsic_call<'tcx>(
+ fx: &mut FunctionCx<'_, '_, 'tcx>,
+ intrinsic: Symbol,
+ args: &[mir::Operand<'tcx>],
+ ret: CPlace<'tcx>,
+) -> bool {
+ let (name, arg_count, ty) = match intrinsic {
+ sym::expf32 => ("expf", 1, fx.tcx.types.f32),
+ sym::expf64 => ("exp", 1, fx.tcx.types.f64),
+ sym::exp2f32 => ("exp2f", 1, fx.tcx.types.f32),
+ sym::exp2f64 => ("exp2", 1, fx.tcx.types.f64),
+ sym::sqrtf32 => ("sqrtf", 1, fx.tcx.types.f32),
+ sym::sqrtf64 => ("sqrt", 1, fx.tcx.types.f64),
+ sym::powif32 => ("__powisf2", 2, fx.tcx.types.f32), // compiler-builtins
+ sym::powif64 => ("__powidf2", 2, fx.tcx.types.f64), // compiler-builtins
+ sym::powf32 => ("powf", 2, fx.tcx.types.f32),
+ sym::powf64 => ("pow", 2, fx.tcx.types.f64),
+ sym::logf32 => ("logf", 1, fx.tcx.types.f32),
+ sym::logf64 => ("log", 1, fx.tcx.types.f64),
+ sym::log2f32 => ("log2f", 1, fx.tcx.types.f32),
+ sym::log2f64 => ("log2", 1, fx.tcx.types.f64),
+ sym::log10f32 => ("log10f", 1, fx.tcx.types.f32),
+ sym::log10f64 => ("log10", 1, fx.tcx.types.f64),
+ sym::fabsf32 => ("fabsf", 1, fx.tcx.types.f32),
+ sym::fabsf64 => ("fabs", 1, fx.tcx.types.f64),
+ sym::fmaf32 => ("fmaf", 3, fx.tcx.types.f32),
+ sym::fmaf64 => ("fma", 3, fx.tcx.types.f64),
+ sym::copysignf32 => ("copysignf", 2, fx.tcx.types.f32),
+ sym::copysignf64 => ("copysign", 2, fx.tcx.types.f64),
+ sym::floorf32 => ("floorf", 1, fx.tcx.types.f32),
+ sym::floorf64 => ("floor", 1, fx.tcx.types.f64),
+ sym::ceilf32 => ("ceilf", 1, fx.tcx.types.f32),
+ sym::ceilf64 => ("ceil", 1, fx.tcx.types.f64),
+ sym::truncf32 => ("truncf", 1, fx.tcx.types.f32),
+ sym::truncf64 => ("trunc", 1, fx.tcx.types.f64),
+ sym::roundf32 => ("roundf", 1, fx.tcx.types.f32),
+ sym::roundf64 => ("round", 1, fx.tcx.types.f64),
+ sym::sinf32 => ("sinf", 1, fx.tcx.types.f32),
+ sym::sinf64 => ("sin", 1, fx.tcx.types.f64),
+ sym::cosf32 => ("cosf", 1, fx.tcx.types.f32),
+ sym::cosf64 => ("cos", 1, fx.tcx.types.f64),
+ _ => return false,
+ };
+
+ if args.len() != arg_count {
+ bug!("wrong number of args for intrinsic {:?}", intrinsic);
+ }
+
+ let (a, b, c);
+ let args = match args {
+ [x] => {
+ a = [codegen_operand(fx, x)];
+ &a as &[_]
+ }
+ [x, y] => {
+ b = [codegen_operand(fx, x), codegen_operand(fx, y)];
+ &b
+ }
+ [x, y, z] => {
+ c = [codegen_operand(fx, x), codegen_operand(fx, y), codegen_operand(fx, z)];
+ &c
+ }
+ _ => unreachable!(),
+ };
+
+ let res = fx.easy_call(name, &args, ty);
+ ret.write_cvalue(fx, res);
+
+ true
+}
+
+fn codegen_regular_intrinsic_call<'tcx>(
+ fx: &mut FunctionCx<'_, '_, 'tcx>,
+ instance: Instance<'tcx>,
+ intrinsic: Symbol,
+ substs: SubstsRef<'tcx>,
+ args: &[mir::Operand<'tcx>],
+ ret: CPlace<'tcx>,
+ span: Span,
+ destination: Option<(CPlace<'tcx>, BasicBlock)>,
+) {
+ let usize_layout = fx.layout_of(fx.tcx.types.usize);
+
+ intrinsic_match! {
+ fx, intrinsic, args,
+ _ => {
+ fx.tcx.sess.span_fatal(span, &format!("unsupported intrinsic {}", intrinsic));
+ };
+
+ assume, (c _a) {};
+ likely | unlikely, (c a) {
+ ret.write_cvalue(fx, a);
+ };
+ breakpoint, () {
+ fx.bcx.ins().debugtrap();
+ };
+ copy | copy_nonoverlapping, (v src, v dst, v count) {
+ let elem_ty = substs.type_at(0);
+ let elem_size: u64 = fx.layout_of(elem_ty).size.bytes();
+ assert_eq!(args.len(), 3);
+ let byte_amount = if elem_size != 1 {
+ fx.bcx.ins().imul_imm(count, elem_size as i64)
+ } else {
+ count
+ };
+
+ if intrinsic == sym::copy_nonoverlapping {
+ // FIXME emit_small_memcpy
+ fx.bcx.call_memcpy(fx.target_config, dst, src, byte_amount);
+ } else {
+ // FIXME emit_small_memmove
+ fx.bcx.call_memmove(fx.target_config, dst, src, byte_amount);
+ }
+ };
+ // NOTE: the volatile variants have src and dst swapped
+ volatile_copy_memory | volatile_copy_nonoverlapping_memory, (v dst, v src, v count) {
+ let elem_ty = substs.type_at(0);
+ let elem_size: u64 = fx.layout_of(elem_ty).size.bytes();
+ assert_eq!(args.len(), 3);
+ let byte_amount = if elem_size != 1 {
+ fx.bcx.ins().imul_imm(count, elem_size as i64)
+ } else {
+ count
+ };
+
+ // FIXME make the copy actually volatile when using emit_small_mem{cpy,move}
+ if intrinsic == sym::volatile_copy_nonoverlapping_memory {
+ // FIXME emit_small_memcpy
+ fx.bcx.call_memcpy(fx.target_config, dst, src, byte_amount);
+ } else {
+ // FIXME emit_small_memmove
+ fx.bcx.call_memmove(fx.target_config, dst, src, byte_amount);
+ }
+ };
+ size_of_val, (c ptr) {
+ let layout = fx.layout_of(substs.type_at(0));
+ let size = if layout.is_unsized() {
+ let (_ptr, info) = ptr.load_scalar_pair(fx);
+ let (size, _align) = crate::unsize::size_and_align_of_dst(fx, layout, info);
+ size
+ } else {
+ fx
+ .bcx
+ .ins()
+ .iconst(fx.pointer_type, layout.size.bytes() as i64)
+ };
+ ret.write_cvalue(fx, CValue::by_val(size, usize_layout));
+ };
+ min_align_of_val, (c ptr) {
+ let layout = fx.layout_of(substs.type_at(0));
+ let align = if layout.is_unsized() {
+ let (_ptr, info) = ptr.load_scalar_pair(fx);
+ let (_size, align) = crate::unsize::size_and_align_of_dst(fx, layout, info);
+ align
+ } else {
+ fx
+ .bcx
+ .ins()
+ .iconst(fx.pointer_type, layout.align.abi.bytes() as i64)
+ };
+ ret.write_cvalue(fx, CValue::by_val(align, usize_layout));
+ };
+
+ unchecked_add | unchecked_sub | unchecked_mul | unchecked_div | exact_div | unchecked_rem
+ | unchecked_shl | unchecked_shr, (c x, c y) {
+ // FIXME trap on overflow
+ let bin_op = match intrinsic {
+ sym::unchecked_add => BinOp::Add,
+ sym::unchecked_sub => BinOp::Sub,
+ sym::unchecked_mul => BinOp::Mul,
+ sym::unchecked_div | sym::exact_div => BinOp::Div,
+ sym::unchecked_rem => BinOp::Rem,
+ sym::unchecked_shl => BinOp::Shl,
+ sym::unchecked_shr => BinOp::Shr,
+ _ => unreachable!(),
+ };
+ let res = crate::num::codegen_int_binop(fx, bin_op, x, y);
+ ret.write_cvalue(fx, res);
+ };
+ add_with_overflow | sub_with_overflow | mul_with_overflow, (c x, c y) {
+ assert_eq!(x.layout().ty, y.layout().ty);
+ let bin_op = match intrinsic {
+ sym::add_with_overflow => BinOp::Add,
+ sym::sub_with_overflow => BinOp::Sub,
+ sym::mul_with_overflow => BinOp::Mul,
+ _ => unreachable!(),
+ };
+
+ let res = crate::num::codegen_checked_int_binop(
+ fx,
+ bin_op,
+ x,
+ y,
+ );
+ ret.write_cvalue(fx, res);
+ };
+ saturating_add | saturating_sub, (c lhs, c rhs) {
+ assert_eq!(lhs.layout().ty, rhs.layout().ty);
+ let bin_op = match intrinsic {
+ sym::saturating_add => BinOp::Add,
+ sym::saturating_sub => BinOp::Sub,
+ _ => unreachable!(),
+ };
+
+ let signed = type_sign(lhs.layout().ty);
+
+ let checked_res = crate::num::codegen_checked_int_binop(
+ fx,
+ bin_op,
+ lhs,
+ rhs,
+ );
+
+ let (val, has_overflow) = checked_res.load_scalar_pair(fx);
+ let clif_ty = fx.clif_type(lhs.layout().ty).unwrap();
+
+ let (min, max) = type_min_max_value(&mut fx.bcx, clif_ty, signed);
+
+ let val = match (intrinsic, signed) {
+ (sym::saturating_add, false) => fx.bcx.ins().select(has_overflow, max, val),
+ (sym::saturating_sub, false) => fx.bcx.ins().select(has_overflow, min, val),
+ (sym::saturating_add, true) => {
+ let rhs = rhs.load_scalar(fx);
+ let rhs_ge_zero = fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThanOrEqual, rhs, 0);
+ let sat_val = fx.bcx.ins().select(rhs_ge_zero, max, min);
+ fx.bcx.ins().select(has_overflow, sat_val, val)
+ }
+ (sym::saturating_sub, true) => {
+ let rhs = rhs.load_scalar(fx);
+ let rhs_ge_zero = fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThanOrEqual, rhs, 0);
+ let sat_val = fx.bcx.ins().select(rhs_ge_zero, min, max);
+ fx.bcx.ins().select(has_overflow, sat_val, val)
+ }
+ _ => unreachable!(),
+ };
+
+ let res = CValue::by_val(val, lhs.layout());
+
+ ret.write_cvalue(fx, res);
+ };
+ rotate_left, (c x, v y) {
+ let layout = x.layout();
+ let x = x.load_scalar(fx);
+ let res = fx.bcx.ins().rotl(x, y);
+ ret.write_cvalue(fx, CValue::by_val(res, layout));
+ };
+ rotate_right, (c x, v y) {
+ let layout = x.layout();
+ let x = x.load_scalar(fx);
+ let res = fx.bcx.ins().rotr(x, y);
+ ret.write_cvalue(fx, CValue::by_val(res, layout));
+ };
+
+ // The only difference between offset and arith_offset is regarding UB. Because Cranelift
+ // doesn't have UB both are codegen'ed the same way
+ offset | arith_offset, (c base, v offset) {
+ let pointee_ty = base.layout().ty.builtin_deref(true).unwrap().ty;
+ let pointee_size = fx.layout_of(pointee_ty).size.bytes();
+ let ptr_diff = if pointee_size != 1 {
+ fx.bcx.ins().imul_imm(offset, pointee_size as i64)
+ } else {
+ offset
+ };
+ let base_val = base.load_scalar(fx);
+ let res = fx.bcx.ins().iadd(base_val, ptr_diff);
+ ret.write_cvalue(fx, CValue::by_val(res, base.layout()));
+ };
+
+ transmute, (c from) {
+ ret.write_cvalue_transmute(fx, from);
+ };
+ write_bytes | volatile_set_memory, (c dst, v val, v count) {
+ let pointee_ty = dst.layout().ty.builtin_deref(true).unwrap().ty;
+ let pointee_size = fx.layout_of(pointee_ty).size.bytes();
+ let count = if pointee_size != 1 {
+ fx.bcx.ins().imul_imm(count, pointee_size as i64)
+ } else {
+ count
+ };
+ let dst_ptr = dst.load_scalar(fx);
+ // FIXME make the memset actually volatile when switching to emit_small_memset
+ // FIXME use emit_small_memset
+ fx.bcx.call_memset(fx.target_config, dst_ptr, val, count);
+ };
+ ctlz | ctlz_nonzero, (c arg) {
+ let val = arg.load_scalar(fx);
+ // FIXME trap on `ctlz_nonzero` with zero arg.
+ let res = fx.bcx.ins().clz(val);
+ let res = CValue::by_val(res, arg.layout());
+ ret.write_cvalue(fx, res);
+ };
+ cttz | cttz_nonzero, (c arg) {
+ let val = arg.load_scalar(fx);
+ // FIXME trap on `cttz_nonzero` with zero arg.
+ let res = fx.bcx.ins().ctz(val);
+ let res = CValue::by_val(res, arg.layout());
+ ret.write_cvalue(fx, res);
+ };
+ ctpop, (c arg) {
+ let val = arg.load_scalar(fx);
+ let res = fx.bcx.ins().popcnt(val);
+ let res = CValue::by_val(res, arg.layout());
+ ret.write_cvalue(fx, res);
+ };
+ bitreverse, (c arg) {
+ let val = arg.load_scalar(fx);
+ let res = fx.bcx.ins().bitrev(val);
+ let res = CValue::by_val(res, arg.layout());
+ ret.write_cvalue(fx, res);
+ };
+ bswap, (c arg) {
+ // FIXME(CraneStation/cranelift#794) add bswap instruction to cranelift
+ fn swap(bcx: &mut FunctionBuilder<'_>, v: Value) -> Value {
+ match bcx.func.dfg.value_type(v) {
+ types::I8 => v,
+
+ // https://code.woboq.org/gcc/include/bits/byteswap.h.html
+ types::I16 => {
+ let tmp1 = bcx.ins().ishl_imm(v, 8);
+ let n1 = bcx.ins().band_imm(tmp1, 0xFF00);
+
+ let tmp2 = bcx.ins().ushr_imm(v, 8);
+ let n2 = bcx.ins().band_imm(tmp2, 0x00FF);
+
+ bcx.ins().bor(n1, n2)
+ }
+ types::I32 => {
+ let tmp1 = bcx.ins().ishl_imm(v, 24);
+ let n1 = bcx.ins().band_imm(tmp1, 0xFF00_0000);
+
+ let tmp2 = bcx.ins().ishl_imm(v, 8);
+ let n2 = bcx.ins().band_imm(tmp2, 0x00FF_0000);
+
+ let tmp3 = bcx.ins().ushr_imm(v, 8);
+ let n3 = bcx.ins().band_imm(tmp3, 0x0000_FF00);
+
+ let tmp4 = bcx.ins().ushr_imm(v, 24);
+ let n4 = bcx.ins().band_imm(tmp4, 0x0000_00FF);
+
+ let or_tmp1 = bcx.ins().bor(n1, n2);
+ let or_tmp2 = bcx.ins().bor(n3, n4);
+ bcx.ins().bor(or_tmp1, or_tmp2)
+ }
+ types::I64 => {
+ let tmp1 = bcx.ins().ishl_imm(v, 56);
+ let n1 = bcx.ins().band_imm(tmp1, 0xFF00_0000_0000_0000u64 as i64);
+
+ let tmp2 = bcx.ins().ishl_imm(v, 40);
+ let n2 = bcx.ins().band_imm(tmp2, 0x00FF_0000_0000_0000u64 as i64);
+
+ let tmp3 = bcx.ins().ishl_imm(v, 24);
+ let n3 = bcx.ins().band_imm(tmp3, 0x0000_FF00_0000_0000u64 as i64);
+
+ let tmp4 = bcx.ins().ishl_imm(v, 8);
+ let n4 = bcx.ins().band_imm(tmp4, 0x0000_00FF_0000_0000u64 as i64);
+
+ let tmp5 = bcx.ins().ushr_imm(v, 8);
+ let n5 = bcx.ins().band_imm(tmp5, 0x0000_0000_FF00_0000u64 as i64);
+
+ let tmp6 = bcx.ins().ushr_imm(v, 24);
+ let n6 = bcx.ins().band_imm(tmp6, 0x0000_0000_00FF_0000u64 as i64);
+
+ let tmp7 = bcx.ins().ushr_imm(v, 40);
+ let n7 = bcx.ins().band_imm(tmp7, 0x0000_0000_0000_FF00u64 as i64);
+
+ let tmp8 = bcx.ins().ushr_imm(v, 56);
+ let n8 = bcx.ins().band_imm(tmp8, 0x0000_0000_0000_00FFu64 as i64);
+
+ let or_tmp1 = bcx.ins().bor(n1, n2);
+ let or_tmp2 = bcx.ins().bor(n3, n4);
+ let or_tmp3 = bcx.ins().bor(n5, n6);
+ let or_tmp4 = bcx.ins().bor(n7, n8);
+
+ let or_tmp5 = bcx.ins().bor(or_tmp1, or_tmp2);
+ let or_tmp6 = bcx.ins().bor(or_tmp3, or_tmp4);
+ bcx.ins().bor(or_tmp5, or_tmp6)
+ }
+ types::I128 => {
+ let (lo, hi) = bcx.ins().isplit(v);
+ let lo = swap(bcx, lo);
+ let hi = swap(bcx, hi);
+ bcx.ins().iconcat(hi, lo)
+ }
+ ty => unreachable!("bswap {}", ty),
+ }
+ }
+ let val = arg.load_scalar(fx);
+ let res = CValue::by_val(swap(&mut fx.bcx, val), arg.layout());
+ ret.write_cvalue(fx, res);
+ };
+ assert_inhabited | assert_zero_valid | assert_uninit_valid, () {
+ let layout = fx.layout_of(substs.type_at(0));
+ if layout.abi.is_uninhabited() {
+ with_no_trimmed_paths!({
+ crate::base::codegen_panic(
+ fx,
+ &format!("attempted to instantiate uninhabited type `{}`", layout.ty),
+ span,
+ )
+ });
+ return;
+ }
+
+ if intrinsic == sym::assert_zero_valid && !layout.might_permit_raw_init(fx, /*zero:*/ true) {
+ with_no_trimmed_paths!({
+ crate::base::codegen_panic(
+ fx,
+ &format!("attempted to zero-initialize type `{}`, which is invalid", layout.ty),
+ span,
+ );
+ });
+ return;
+ }
+
+ if intrinsic == sym::assert_uninit_valid && !layout.might_permit_raw_init(fx, /*zero:*/ false) {
+ with_no_trimmed_paths!({
+ crate::base::codegen_panic(
+ fx,
+ &format!("attempted to leave type `{}` uninitialized, which is invalid", layout.ty),
+ span,
+ )
+ });
+ return;
+ }
+ };
+
+ volatile_load | unaligned_volatile_load, (c ptr) {
+ // Cranelift treats loads as volatile by default
+ // FIXME correctly handle unaligned_volatile_load
+ let inner_layout =
+ fx.layout_of(ptr.layout().ty.builtin_deref(true).unwrap().ty);
+ let val = CValue::by_ref(Pointer::new(ptr.load_scalar(fx)), inner_layout);
+ ret.write_cvalue(fx, val);
+ };
+ volatile_store | unaligned_volatile_store, (v ptr, c val) {
+ // Cranelift treats stores as volatile by default
+ // FIXME correctly handle unaligned_volatile_store
+ let dest = CPlace::for_ptr(Pointer::new(ptr), val.layout());
+ dest.write_cvalue(fx, val);
+ };
+
+ pref_align_of | needs_drop | type_id | type_name | variant_count, () {
+ let const_val =
+ fx.tcx.const_eval_instance(ParamEnv::reveal_all(), instance, None).unwrap();
+ let val = crate::constant::codegen_const_value(
+ fx,
+ const_val,
+ ret.layout().ty,
+ );
+ ret.write_cvalue(fx, val);
+ };
+
+ ptr_offset_from, (v ptr, v base) {
+ let ty = substs.type_at(0);
+ let isize_layout = fx.layout_of(fx.tcx.types.isize);
+
+ let pointee_size: u64 = fx.layout_of(ty).size.bytes();
+ let diff = fx.bcx.ins().isub(ptr, base);
+ // FIXME this can be an exact division.
+ let val = CValue::by_val(fx.bcx.ins().sdiv_imm(diff, pointee_size as i64), isize_layout);
+ ret.write_cvalue(fx, val);
+ };
+
+ ptr_guaranteed_eq, (c a, c b) {
+ let val = crate::num::codegen_ptr_binop(fx, BinOp::Eq, a, b);
+ ret.write_cvalue(fx, val);
+ };
+
+ ptr_guaranteed_ne, (c a, c b) {
+ let val = crate::num::codegen_ptr_binop(fx, BinOp::Ne, a, b);
+ ret.write_cvalue(fx, val);
+ };
+
+ caller_location, () {
+ let caller_location = fx.get_caller_location(span);
+ ret.write_cvalue(fx, caller_location);
+ };
+
+ _ if intrinsic.as_str().starts_with("atomic_fence"), () {
+ fx.bcx.ins().fence();
+ };
+ _ if intrinsic.as_str().starts_with("atomic_singlethreadfence"), () {
+ // FIXME use a compiler fence once Cranelift supports it
+ fx.bcx.ins().fence();
+ };
+ _ if intrinsic.as_str().starts_with("atomic_load"), (v ptr) {
+ let ty = substs.type_at(0);
+ match ty.kind() {
++ ty::Uint(UintTy::U128) | ty::Int(IntTy::I128) => {
++ // FIXME implement 128bit atomics
++ if fx.tcx.is_compiler_builtins(LOCAL_CRATE) {
++ // special case for compiler-builtins to avoid having to patch it
++ crate::trap::trap_unimplemented(fx, "128bit atomics not yet supported");
++ let ret_block = fx.get_block(destination.unwrap().1);
++ fx.bcx.ins().jump(ret_block, &[]);
++ return;
++ } else {
++ fx.tcx.sess.span_fatal(span, "128bit atomics not yet supported");
++ }
++ }
+ ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
+ _ => {
+ report_atomic_type_validation_error(fx, intrinsic, span, ty);
+ return;
+ }
+ }
+ let clif_ty = fx.clif_type(ty).unwrap();
+
+ let val = fx.bcx.ins().atomic_load(clif_ty, MemFlags::trusted(), ptr);
+
+ let val = CValue::by_val(val, fx.layout_of(ty));
+ ret.write_cvalue(fx, val);
+ };
+ _ if intrinsic.as_str().starts_with("atomic_store"), (v ptr, c val) {
+ let ty = substs.type_at(0);
+ match ty.kind() {
++ ty::Uint(UintTy::U128) | ty::Int(IntTy::I128) => {
++ // FIXME implement 128bit atomics
++ if fx.tcx.is_compiler_builtins(LOCAL_CRATE) {
++ // special case for compiler-builtins to avoid having to patch it
++ crate::trap::trap_unimplemented(fx, "128bit atomics not yet supported");
++ let ret_block = fx.get_block(destination.unwrap().1);
++ fx.bcx.ins().jump(ret_block, &[]);
++ return;
++ } else {
++ fx.tcx.sess.span_fatal(span, "128bit atomics not yet supported");
++ }
++ }
+ ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
+ _ => {
+ report_atomic_type_validation_error(fx, intrinsic, span, ty);
+ return;
+ }
+ }
+
+ let val = val.load_scalar(fx);
+
+ fx.bcx.ins().atomic_store(MemFlags::trusted(), val, ptr);
+ };
+ _ if intrinsic.as_str().starts_with("atomic_xchg"), (v ptr, c new) {
+ let layout = new.layout();
+ match layout.ty.kind() {
+ ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
+ _ => {
+ report_atomic_type_validation_error(fx, intrinsic, span, layout.ty);
+ return;
+ }
+ }
+ let ty = fx.clif_type(layout.ty).unwrap();
+
+ let new = new.load_scalar(fx);
+
+ let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Xchg, ptr, new);
+
+ let old = CValue::by_val(old, layout);
+ ret.write_cvalue(fx, old);
+ };
+ _ if intrinsic.as_str().starts_with("atomic_cxchg"), (v ptr, c test_old, c new) { // both atomic_cxchg_* and atomic_cxchgweak_*
+ let layout = new.layout();
+ match layout.ty.kind() {
+ ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
+ _ => {
+ report_atomic_type_validation_error(fx, intrinsic, span, layout.ty);
+ return;
+ }
+ }
+
+ let test_old = test_old.load_scalar(fx);
+ let new = new.load_scalar(fx);
+
+ let old = fx.bcx.ins().atomic_cas(MemFlags::trusted(), ptr, test_old, new);
+ let is_eq = fx.bcx.ins().icmp(IntCC::Equal, old, test_old);
+
+ let ret_val = CValue::by_val_pair(old, fx.bcx.ins().bint(types::I8, is_eq), ret.layout());
+ ret.write_cvalue(fx, ret_val)
+ };
+
+ _ if intrinsic.as_str().starts_with("atomic_xadd"), (v ptr, c amount) {
+ let layout = amount.layout();
+ match layout.ty.kind() {
+ ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
+ _ => {
+ report_atomic_type_validation_error(fx, intrinsic, span, layout.ty);
+ return;
+ }
+ }
+ let ty = fx.clif_type(layout.ty).unwrap();
+
+ let amount = amount.load_scalar(fx);
+
+ let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Add, ptr, amount);
+
+ let old = CValue::by_val(old, layout);
+ ret.write_cvalue(fx, old);
+ };
+ _ if intrinsic.as_str().starts_with("atomic_xsub"), (v ptr, c amount) {
+ let layout = amount.layout();
+ match layout.ty.kind() {
+ ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
+ _ => {
+ report_atomic_type_validation_error(fx, intrinsic, span, layout.ty);
+ return;
+ }
+ }
+ let ty = fx.clif_type(layout.ty).unwrap();
+
+ let amount = amount.load_scalar(fx);
+
+ let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Sub, ptr, amount);
+
+ let old = CValue::by_val(old, layout);
+ ret.write_cvalue(fx, old);
+ };
+ _ if intrinsic.as_str().starts_with("atomic_and"), (v ptr, c src) {
+ let layout = src.layout();
+ match layout.ty.kind() {
+ ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
+ _ => {
+ report_atomic_type_validation_error(fx, intrinsic, span, layout.ty);
+ return;
+ }
+ }
+ let ty = fx.clif_type(layout.ty).unwrap();
+
+ let src = src.load_scalar(fx);
+
+ let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::And, ptr, src);
+
+ let old = CValue::by_val(old, layout);
+ ret.write_cvalue(fx, old);
+ };
+ _ if intrinsic.as_str().starts_with("atomic_or"), (v ptr, c src) {
+ let layout = src.layout();
+ match layout.ty.kind() {
+ ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
+ _ => {
+ report_atomic_type_validation_error(fx, intrinsic, span, layout.ty);
+ return;
+ }
+ }
+ let ty = fx.clif_type(layout.ty).unwrap();
+
+ let src = src.load_scalar(fx);
+
+ let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Or, ptr, src);
+
+ let old = CValue::by_val(old, layout);
+ ret.write_cvalue(fx, old);
+ };
+ _ if intrinsic.as_str().starts_with("atomic_xor"), (v ptr, c src) {
+ let layout = src.layout();
+ match layout.ty.kind() {
+ ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
+ _ => {
+ report_atomic_type_validation_error(fx, intrinsic, span, layout.ty);
+ return;
+ }
+ }
+ let ty = fx.clif_type(layout.ty).unwrap();
+
+ let src = src.load_scalar(fx);
+
+ let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Xor, ptr, src);
+
+ let old = CValue::by_val(old, layout);
+ ret.write_cvalue(fx, old);
+ };
+ _ if intrinsic.as_str().starts_with("atomic_nand"), (v ptr, c src) {
+ let layout = src.layout();
+ match layout.ty.kind() {
+ ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
+ _ => {
+ report_atomic_type_validation_error(fx, intrinsic, span, layout.ty);
+ return;
+ }
+ }
+ let ty = fx.clif_type(layout.ty).unwrap();
+
+ let src = src.load_scalar(fx);
+
+ let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Nand, ptr, src);
+
+ let old = CValue::by_val(old, layout);
+ ret.write_cvalue(fx, old);
+ };
+ _ if intrinsic.as_str().starts_with("atomic_max"), (v ptr, c src) {
+ let layout = src.layout();
+ match layout.ty.kind() {
+ ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
+ _ => {
+ report_atomic_type_validation_error(fx, intrinsic, span, layout.ty);
+ return;
+ }
+ }
+ let ty = fx.clif_type(layout.ty).unwrap();
+
+ let src = src.load_scalar(fx);
+
+ let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Smax, ptr, src);
+
+ let old = CValue::by_val(old, layout);
+ ret.write_cvalue(fx, old);
+ };
+ _ if intrinsic.as_str().starts_with("atomic_umax"), (v ptr, c src) {
+ let layout = src.layout();
+ match layout.ty.kind() {
+ ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
+ _ => {
+ report_atomic_type_validation_error(fx, intrinsic, span, layout.ty);
+ return;
+ }
+ }
+ let ty = fx.clif_type(layout.ty).unwrap();
+
+ let src = src.load_scalar(fx);
+
+ let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Umax, ptr, src);
+
+ let old = CValue::by_val(old, layout);
+ ret.write_cvalue(fx, old);
+ };
+ _ if intrinsic.as_str().starts_with("atomic_min"), (v ptr, c src) {
+ let layout = src.layout();
+ match layout.ty.kind() {
+ ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
+ _ => {
+ report_atomic_type_validation_error(fx, intrinsic, span, layout.ty);
+ return;
+ }
+ }
+ let ty = fx.clif_type(layout.ty).unwrap();
+
+ let src = src.load_scalar(fx);
+
+ let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Smin, ptr, src);
+
+ let old = CValue::by_val(old, layout);
+ ret.write_cvalue(fx, old);
+ };
+ _ if intrinsic.as_str().starts_with("atomic_umin"), (v ptr, c src) {
+ let layout = src.layout();
+ match layout.ty.kind() {
+ ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
+ _ => {
+ report_atomic_type_validation_error(fx, intrinsic, span, layout.ty);
+ return;
+ }
+ }
+ let ty = fx.clif_type(layout.ty).unwrap();
+
+ let src = src.load_scalar(fx);
+
+ let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Umin, ptr, src);
+
+ let old = CValue::by_val(old, layout);
+ ret.write_cvalue(fx, old);
+ };
+
+ // In Rust floating point min and max don't propagate NaN. In Cranelift they do however.
+ // For this reason it is necessary to use `a.is_nan() ? b : (a >= b ? b : a)` for `minnumf*`
+ // and `a.is_nan() ? b : (a <= b ? b : a)` for `maxnumf*`. NaN checks are done by comparing
+ // a float against itself. Only in case of NaN is it not equal to itself.
+ minnumf32, (v a, v b) {
+ let a_is_nan = fx.bcx.ins().fcmp(FloatCC::NotEqual, a, a);
+ let a_ge_b = fx.bcx.ins().fcmp(FloatCC::GreaterThanOrEqual, a, b);
+ let temp = fx.bcx.ins().select(a_ge_b, b, a);
+ let val = fx.bcx.ins().select(a_is_nan, b, temp);
+ let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f32));
+ ret.write_cvalue(fx, val);
+ };
+ minnumf64, (v a, v b) {
+ let a_is_nan = fx.bcx.ins().fcmp(FloatCC::NotEqual, a, a);
+ let a_ge_b = fx.bcx.ins().fcmp(FloatCC::GreaterThanOrEqual, a, b);
+ let temp = fx.bcx.ins().select(a_ge_b, b, a);
+ let val = fx.bcx.ins().select(a_is_nan, b, temp);
+ let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f64));
+ ret.write_cvalue(fx, val);
+ };
+ maxnumf32, (v a, v b) {
+ let a_is_nan = fx.bcx.ins().fcmp(FloatCC::NotEqual, a, a);
+ let a_le_b = fx.bcx.ins().fcmp(FloatCC::LessThanOrEqual, a, b);
+ let temp = fx.bcx.ins().select(a_le_b, b, a);
+ let val = fx.bcx.ins().select(a_is_nan, b, temp);
+ let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f32));
+ ret.write_cvalue(fx, val);
+ };
+ maxnumf64, (v a, v b) {
+ let a_is_nan = fx.bcx.ins().fcmp(FloatCC::NotEqual, a, a);
+ let a_le_b = fx.bcx.ins().fcmp(FloatCC::LessThanOrEqual, a, b);
+ let temp = fx.bcx.ins().select(a_le_b, b, a);
+ let val = fx.bcx.ins().select(a_is_nan, b, temp);
+ let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f64));
+ ret.write_cvalue(fx, val);
+ };
+
+ kw.Try, (v f, v data, v _catch_fn) {
+ // FIXME once unwinding is supported, change this to actually catch panics
+ let f_sig = fx.bcx.func.import_signature(Signature {
+ call_conv: fx.target_config.default_call_conv,
+ params: vec![AbiParam::new(fx.bcx.func.dfg.value_type(data))],
+ returns: vec![],
+ });
+
+ fx.bcx.ins().call_indirect(f_sig, f, &[data]);
+
+ let layout = ret.layout();
+ let ret_val = CValue::const_val(fx, layout, ty::ScalarInt::null(layout.size));
+ ret.write_cvalue(fx, ret_val);
+ };
+
+ fadd_fast | fsub_fast | fmul_fast | fdiv_fast | frem_fast, (c x, c y) {
+ let res = crate::num::codegen_float_binop(fx, match intrinsic {
+ sym::fadd_fast => BinOp::Add,
+ sym::fsub_fast => BinOp::Sub,
+ sym::fmul_fast => BinOp::Mul,
+ sym::fdiv_fast => BinOp::Div,
+ sym::frem_fast => BinOp::Rem,
+ _ => unreachable!(),
+ }, x, y);
+ ret.write_cvalue(fx, res);
+ };
+ float_to_int_unchecked, (v f) {
+ let res = crate::cast::clif_int_or_float_cast(
+ fx,
+ f,
+ false,
+ fx.clif_type(ret.layout().ty).unwrap(),
+ type_sign(ret.layout().ty),
+ );
+ ret.write_cvalue(fx, CValue::by_val(res, ret.layout()));
+ };
+
+ raw_eq, (v lhs_ref, v rhs_ref) {
+ let size = fx.layout_of(substs.type_at(0)).layout.size();
+ // FIXME add and use emit_small_memcmp
+ let is_eq_value =
+ if size == Size::ZERO {
+ // No bytes means they're trivially equal
+ fx.bcx.ins().iconst(types::I8, 1)
+ } else if let Some(clty) = size.bits().try_into().ok().and_then(Type::int) {
+ // Can't use `trusted` for these loads; they could be unaligned.
+ let mut flags = MemFlags::new();
+ flags.set_notrap();
+ let lhs_val = fx.bcx.ins().load(clty, flags, lhs_ref, 0);
+ let rhs_val = fx.bcx.ins().load(clty, flags, rhs_ref, 0);
+ let eq = fx.bcx.ins().icmp(IntCC::Equal, lhs_val, rhs_val);
+ fx.bcx.ins().bint(types::I8, eq)
+ } else {
+ // Just call `memcmp` (like slices do in core) when the
+ // size is too large or it's not a power-of-two.
+ let signed_bytes = i64::try_from(size.bytes()).unwrap();
+ let bytes_val = fx.bcx.ins().iconst(fx.pointer_type, signed_bytes);
+ let params = vec![AbiParam::new(fx.pointer_type); 3];
+ let returns = vec![AbiParam::new(types::I32)];
+ let args = &[lhs_ref, rhs_ref, bytes_val];
+ let cmp = fx.lib_call("memcmp", params, returns, args)[0];
+ let eq = fx.bcx.ins().icmp_imm(IntCC::Equal, cmp, 0);
+ fx.bcx.ins().bint(types::I8, eq)
+ };
+ ret.write_cvalue(fx, CValue::by_val(is_eq_value, ret.layout()));
+ };
+
+ const_allocate, (c _size, c _align) {
+ // returns a null pointer at runtime.
+ let null = fx.bcx.ins().iconst(fx.pointer_type, 0);
+ ret.write_cvalue(fx, CValue::by_val(null, ret.layout()));
+ };
+
+ const_deallocate, (c _ptr, c _size, c _align) {
+ // nop at runtime.
+ };
+
+ black_box, (c a) {
+ // FIXME implement black_box semantics
+ ret.write_cvalue(fx, a);
+ };
+ }
+
++ let ret_block = fx.get_block(destination.unwrap().1);
++ fx.bcx.ins().jump(ret_block, &[]);
+}
--- /dev/null
+//! Codegen `extern "platform-intrinsic"` intrinsics.
+
+use rustc_middle::ty::subst::SubstsRef;
+use rustc_span::Symbol;
+
+use super::*;
+use crate::prelude::*;
+
+fn report_simd_type_validation_error(
+ fx: &mut FunctionCx<'_, '_, '_>,
+ intrinsic: Symbol,
+ span: Span,
+ ty: Ty<'_>,
+) {
+ fx.tcx.sess.span_err(span, &format!("invalid monomorphization of `{}` intrinsic: expected SIMD input type, found non-SIMD `{}`", intrinsic, ty));
+ // Prevent verifier error
+ crate::trap::trap_unreachable(fx, "compilation should not have succeeded");
+}
+
+pub(super) fn codegen_simd_intrinsic_call<'tcx>(
+ fx: &mut FunctionCx<'_, '_, 'tcx>,
+ intrinsic: Symbol,
+ _substs: SubstsRef<'tcx>,
+ args: &[mir::Operand<'tcx>],
+ ret: CPlace<'tcx>,
+ span: Span,
+) {
+ intrinsic_match! {
+ fx, intrinsic, args,
+ _ => {
+ fx.tcx.sess.span_fatal(span, &format!("Unknown SIMD intrinsic {}", intrinsic));
+ };
+
+ simd_cast, (c a) {
+ if !a.layout().ty.is_simd() {
+ report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
+ return;
+ }
+
+ simd_for_each_lane(fx, a, ret, &|fx, lane_ty, ret_lane_ty, lane| {
+ let ret_lane_clif_ty = fx.clif_type(ret_lane_ty).unwrap();
+
+ let from_signed = type_sign(lane_ty);
+ let to_signed = type_sign(ret_lane_ty);
+
+ clif_int_or_float_cast(fx, lane, from_signed, ret_lane_clif_ty, to_signed)
+ });
+ };
+
+ simd_eq | simd_ne | simd_lt | simd_le | simd_gt | simd_ge, (c x, c y) {
+ if !x.layout().ty.is_simd() {
+ report_simd_type_validation_error(fx, intrinsic, span, x.layout().ty);
+ return;
+ }
+
+ // FIXME use vector instructions when possible
+ simd_pair_for_each_lane(fx, x, y, ret, &|fx, lane_ty, res_lane_ty, x_lane, y_lane| {
+ let res_lane = match (lane_ty.kind(), intrinsic) {
+ (ty::Uint(_), sym::simd_eq) => fx.bcx.ins().icmp(IntCC::Equal, x_lane, y_lane),
+ (ty::Uint(_), sym::simd_ne) => fx.bcx.ins().icmp(IntCC::NotEqual, x_lane, y_lane),
+ (ty::Uint(_), sym::simd_lt) => {
+ fx.bcx.ins().icmp(IntCC::UnsignedLessThan, x_lane, y_lane)
+ }
+ (ty::Uint(_), sym::simd_le) => {
+ fx.bcx.ins().icmp(IntCC::UnsignedLessThanOrEqual, x_lane, y_lane)
+ }
+ (ty::Uint(_), sym::simd_gt) => {
+ fx.bcx.ins().icmp(IntCC::UnsignedGreaterThan, x_lane, y_lane)
+ }
+ (ty::Uint(_), sym::simd_ge) => {
+ fx.bcx.ins().icmp(IntCC::UnsignedGreaterThanOrEqual, x_lane, y_lane)
+ }
+
+ (ty::Int(_), sym::simd_eq) => fx.bcx.ins().icmp(IntCC::Equal, x_lane, y_lane),
+ (ty::Int(_), sym::simd_ne) => fx.bcx.ins().icmp(IntCC::NotEqual, x_lane, y_lane),
+ (ty::Int(_), sym::simd_lt) => fx.bcx.ins().icmp(IntCC::SignedLessThan, x_lane, y_lane),
+ (ty::Int(_), sym::simd_le) => {
+ fx.bcx.ins().icmp(IntCC::SignedLessThanOrEqual, x_lane, y_lane)
+ }
+ (ty::Int(_), sym::simd_gt) => {
+ fx.bcx.ins().icmp(IntCC::SignedGreaterThan, x_lane, y_lane)
+ }
+ (ty::Int(_), sym::simd_ge) => {
+ fx.bcx.ins().icmp(IntCC::SignedGreaterThanOrEqual, x_lane, y_lane)
+ }
+
+ (ty::Float(_), sym::simd_eq) => fx.bcx.ins().fcmp(FloatCC::Equal, x_lane, y_lane),
+ (ty::Float(_), sym::simd_ne) => fx.bcx.ins().fcmp(FloatCC::NotEqual, x_lane, y_lane),
+ (ty::Float(_), sym::simd_lt) => fx.bcx.ins().fcmp(FloatCC::LessThan, x_lane, y_lane),
+ (ty::Float(_), sym::simd_le) => {
+ fx.bcx.ins().fcmp(FloatCC::LessThanOrEqual, x_lane, y_lane)
+ }
+ (ty::Float(_), sym::simd_gt) => fx.bcx.ins().fcmp(FloatCC::GreaterThan, x_lane, y_lane),
+ (ty::Float(_), sym::simd_ge) => {
+ fx.bcx.ins().fcmp(FloatCC::GreaterThanOrEqual, x_lane, y_lane)
+ }
+
+ _ => unreachable!(),
+ };
+
+ let ty = fx.clif_type(res_lane_ty).unwrap();
+
+ let res_lane = fx.bcx.ins().bint(ty, res_lane);
+ fx.bcx.ins().ineg(res_lane)
+ });
+ };
+
+ // simd_shuffle32<T, U>(x: T, y: T, idx: [u32; 32]) -> U
+ _ if intrinsic.as_str().starts_with("simd_shuffle"), (c x, c y, o idx) {
+ if !x.layout().ty.is_simd() {
+ report_simd_type_validation_error(fx, intrinsic, span, x.layout().ty);
+ return;
+ }
+
+ // If this intrinsic is the older "simd_shuffleN" form, simply parse the integer.
+ // If there is no suffix, use the index array length.
+ let n: u16 = if intrinsic == sym::simd_shuffle {
+ // Make sure this is actually an array, since typeck only checks the length-suffixed
+ // version of this intrinsic.
+ let idx_ty = fx.monomorphize(idx.ty(fx.mir, fx.tcx));
+ match idx_ty.kind() {
+ ty::Array(ty, len) if matches!(ty.kind(), ty::Uint(ty::UintTy::U32)) => {
+ len.try_eval_usize(fx.tcx, ty::ParamEnv::reveal_all()).unwrap_or_else(|| {
+ span_bug!(span, "could not evaluate shuffle index array length")
+ }).try_into().unwrap()
+ }
+ _ => {
+ fx.tcx.sess.span_err(
+ span,
+ &format!(
+ "simd_shuffle index must be an array of `u32`, got `{}`",
+ idx_ty,
+ ),
+ );
+ // Prevent verifier error
+ crate::trap::trap_unreachable(fx, "compilation should not have succeeded");
+ return;
+ }
+ }
+ } else {
+ intrinsic.as_str()["simd_shuffle".len()..].parse().unwrap()
+ };
+
+ assert_eq!(x.layout(), y.layout());
+ let layout = x.layout();
+
+ let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx);
+ let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx);
+
+ assert_eq!(lane_ty, ret_lane_ty);
+ assert_eq!(u64::from(n), ret_lane_count);
+
+ let total_len = lane_count * 2;
+
+ let indexes = {
+ use rustc_middle::mir::interpret::*;
+ let idx_const = crate::constant::mir_operand_get_const_val(fx, idx).expect("simd_shuffle* idx not const");
+
+ let idx_bytes = match idx_const {
+ ConstValue::ByRef { alloc, offset } => {
+ let size = Size::from_bytes(4 * ret_lane_count /* size_of([u32; ret_lane_count]) */);
+ alloc.inner().get_bytes(fx, alloc_range(offset, size)).unwrap()
+ }
+ _ => unreachable!("{:?}", idx_const),
+ };
+
+ (0..ret_lane_count).map(|i| {
+ let i = usize::try_from(i).unwrap();
+ let idx = rustc_middle::mir::interpret::read_target_uint(
+ fx.tcx.data_layout.endian,
+ &idx_bytes[4*i.. 4*i + 4],
+ ).expect("read_target_uint");
+ u16::try_from(idx).expect("try_from u32")
+ }).collect::<Vec<u16>>()
+ };
+
+ for &idx in &indexes {
+ assert!(u64::from(idx) < total_len, "idx {} out of range 0..{}", idx, total_len);
+ }
+
+ for (out_idx, in_idx) in indexes.into_iter().enumerate() {
+ let in_lane = if u64::from(in_idx) < lane_count {
+ x.value_lane(fx, in_idx.into())
+ } else {
+ y.value_lane(fx, u64::from(in_idx) - lane_count)
+ };
+ let out_lane = ret.place_lane(fx, u64::try_from(out_idx).unwrap());
+ out_lane.write_cvalue(fx, in_lane);
+ }
+ };
+
+ simd_insert, (c base, o idx, c val) {
+ // FIXME validate
+ let idx_const = if let Some(idx_const) = crate::constant::mir_operand_get_const_val(fx, idx) {
+ idx_const
+ } else {
+ fx.tcx.sess.span_fatal(
+ span,
+ "Index argument for `simd_insert` is not a constant",
+ );
+ };
+
+ let idx = idx_const.try_to_bits(Size::from_bytes(4 /* u32*/)).unwrap_or_else(|| panic!("kind not scalar: {:?}", idx_const));
+ let (lane_count, _lane_ty) = base.layout().ty.simd_size_and_type(fx.tcx);
+ if idx >= lane_count.into() {
+ fx.tcx.sess.span_fatal(fx.mir.span, &format!("[simd_insert] idx {} >= lane_count {}", idx, lane_count));
+ }
+
+ ret.write_cvalue(fx, base);
+ let ret_lane = ret.place_field(fx, mir::Field::new(idx.try_into().unwrap()));
+ ret_lane.write_cvalue(fx, val);
+ };
+
+ simd_extract, (c v, o idx) {
+ if !v.layout().ty.is_simd() {
+ report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
+ return;
+ }
+
+ let idx_const = if let Some(idx_const) = crate::constant::mir_operand_get_const_val(fx, idx) {
+ idx_const
+ } else {
+ fx.tcx.sess.span_warn(
+ span,
+ "Index argument for `simd_extract` is not a constant",
+ );
+ let res = crate::trap::trap_unimplemented_ret_value(
+ fx,
+ ret.layout(),
+ "Index argument for `simd_extract` is not a constant",
+ );
+ ret.write_cvalue(fx, res);
+ return;
+ };
+
+ let idx = idx_const.try_to_bits(Size::from_bytes(4 /* u32*/)).unwrap_or_else(|| panic!("kind not scalar: {:?}", idx_const));
+ let (lane_count, _lane_ty) = v.layout().ty.simd_size_and_type(fx.tcx);
+ if idx >= lane_count.into() {
+ fx.tcx.sess.span_fatal(fx.mir.span, &format!("[simd_extract] idx {} >= lane_count {}", idx, lane_count));
+ }
+
+ let ret_lane = v.value_lane(fx, idx.try_into().unwrap());
+ ret.write_cvalue(fx, ret_lane);
+ };
+
+ simd_neg, (c a) {
+ if !a.layout().ty.is_simd() {
+ report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
+ return;
+ }
+
+ simd_for_each_lane(fx, a, ret, &|fx, lane_ty, _ret_lane_ty, lane| {
+ match lane_ty.kind() {
+ ty::Int(_) => fx.bcx.ins().ineg(lane),
+ ty::Float(_) => fx.bcx.ins().fneg(lane),
+ _ => unreachable!(),
+ }
+ });
+ };
+
+ simd_add | simd_sub | simd_mul | simd_div | simd_rem
+ | simd_shl | simd_shr | simd_and | simd_or | simd_xor, (c x, c y) {
+ if !x.layout().ty.is_simd() {
+ report_simd_type_validation_error(fx, intrinsic, span, x.layout().ty);
+ return;
+ }
+
+ // FIXME use vector instructions when possible
+ simd_pair_for_each_lane(fx, x, y, ret, &|fx, lane_ty, _ret_lane_ty, x_lane, y_lane| match (
+ lane_ty.kind(),
+ intrinsic,
+ ) {
+ (ty::Uint(_), sym::simd_add) => fx.bcx.ins().iadd(x_lane, y_lane),
+ (ty::Uint(_), sym::simd_sub) => fx.bcx.ins().isub(x_lane, y_lane),
+ (ty::Uint(_), sym::simd_mul) => fx.bcx.ins().imul(x_lane, y_lane),
+ (ty::Uint(_), sym::simd_div) => fx.bcx.ins().udiv(x_lane, y_lane),
+ (ty::Uint(_), sym::simd_rem) => fx.bcx.ins().urem(x_lane, y_lane),
+
+ (ty::Int(_), sym::simd_add) => fx.bcx.ins().iadd(x_lane, y_lane),
+ (ty::Int(_), sym::simd_sub) => fx.bcx.ins().isub(x_lane, y_lane),
+ (ty::Int(_), sym::simd_mul) => fx.bcx.ins().imul(x_lane, y_lane),
+ (ty::Int(_), sym::simd_div) => fx.bcx.ins().sdiv(x_lane, y_lane),
+ (ty::Int(_), sym::simd_rem) => fx.bcx.ins().srem(x_lane, y_lane),
+
+ (ty::Float(_), sym::simd_add) => fx.bcx.ins().fadd(x_lane, y_lane),
+ (ty::Float(_), sym::simd_sub) => fx.bcx.ins().fsub(x_lane, y_lane),
+ (ty::Float(_), sym::simd_mul) => fx.bcx.ins().fmul(x_lane, y_lane),
+ (ty::Float(_), sym::simd_div) => fx.bcx.ins().fdiv(x_lane, y_lane),
+ (ty::Float(FloatTy::F32), sym::simd_rem) => fx.lib_call(
+ "fmodf",
+ vec![AbiParam::new(types::F32), AbiParam::new(types::F32)],
+ vec![AbiParam::new(types::F32)],
+ &[x_lane, y_lane],
+ )[0],
+ (ty::Float(FloatTy::F64), sym::simd_rem) => fx.lib_call(
+ "fmod",
+ vec![AbiParam::new(types::F64), AbiParam::new(types::F64)],
+ vec![AbiParam::new(types::F64)],
+ &[x_lane, y_lane],
+ )[0],
+
+ (ty::Uint(_), sym::simd_shl) => fx.bcx.ins().ishl(x_lane, y_lane),
+ (ty::Uint(_), sym::simd_shr) => fx.bcx.ins().ushr(x_lane, y_lane),
+ (ty::Uint(_), sym::simd_and) => fx.bcx.ins().band(x_lane, y_lane),
+ (ty::Uint(_), sym::simd_or) => fx.bcx.ins().bor(x_lane, y_lane),
+ (ty::Uint(_), sym::simd_xor) => fx.bcx.ins().bxor(x_lane, y_lane),
+
+ (ty::Int(_), sym::simd_shl) => fx.bcx.ins().ishl(x_lane, y_lane),
+ (ty::Int(_), sym::simd_shr) => fx.bcx.ins().sshr(x_lane, y_lane),
+ (ty::Int(_), sym::simd_and) => fx.bcx.ins().band(x_lane, y_lane),
+ (ty::Int(_), sym::simd_or) => fx.bcx.ins().bor(x_lane, y_lane),
+ (ty::Int(_), sym::simd_xor) => fx.bcx.ins().bxor(x_lane, y_lane),
+
+ _ => unreachable!(),
+ });
+ };
+
+ simd_fma, (c a, c b, c c) {
+ if !a.layout().ty.is_simd() {
+ report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
+ return;
+ }
+ assert_eq!(a.layout(), b.layout());
+ assert_eq!(a.layout(), c.layout());
+ let layout = a.layout();
+
+ let (lane_count, _lane_ty) = layout.ty.simd_size_and_type(fx.tcx);
+ let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx);
+ assert_eq!(lane_count, ret_lane_count);
+ let ret_lane_layout = fx.layout_of(ret_lane_ty);
+
+ for lane in 0..lane_count {
+ let a_lane = a.value_lane(fx, lane).load_scalar(fx);
+ let b_lane = b.value_lane(fx, lane).load_scalar(fx);
+ let c_lane = c.value_lane(fx, lane).load_scalar(fx);
+
+ let mul_lane = fx.bcx.ins().fmul(a_lane, b_lane);
+ let res_lane = CValue::by_val(fx.bcx.ins().fadd(mul_lane, c_lane), ret_lane_layout);
+
+ ret.place_lane(fx, lane).write_cvalue(fx, res_lane);
+ }
+ };
+
+ simd_fmin | simd_fmax, (c x, c y) {
+ if !x.layout().ty.is_simd() {
+ report_simd_type_validation_error(fx, intrinsic, span, x.layout().ty);
+ return;
+ }
+
+ // FIXME use vector instructions when possible
+ simd_pair_for_each_lane(fx, x, y, ret, &|fx, lane_ty, _ret_lane_ty, x_lane, y_lane| {
+ match lane_ty.kind() {
+ ty::Float(_) => {},
+ _ => unreachable!("{:?}", lane_ty),
+ }
+ match intrinsic {
+ sym::simd_fmin => fx.bcx.ins().fmin(x_lane, y_lane),
+ sym::simd_fmax => fx.bcx.ins().fmax(x_lane, y_lane),
+ _ => unreachable!(),
+ }
+ });
+ };
+
+ simd_round, (c a) {
+ if !a.layout().ty.is_simd() {
+ report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
+ return;
+ }
+
+ simd_for_each_lane(fx, a, ret, &|fx, lane_ty, _ret_lane_ty, lane| {
+ match lane_ty.kind() {
+ ty::Float(FloatTy::F32) => fx.lib_call(
+ "roundf",
+ vec![AbiParam::new(types::F32)],
+ vec![AbiParam::new(types::F32)],
+ &[lane],
+ )[0],
+ ty::Float(FloatTy::F64) => fx.lib_call(
+ "round",
+ vec![AbiParam::new(types::F64)],
+ vec![AbiParam::new(types::F64)],
+ &[lane],
+ )[0],
+ _ => unreachable!("{:?}", lane_ty),
+ }
+ });
+ };
+
+ simd_fabs | simd_fsqrt | simd_ceil | simd_floor | simd_trunc, (c a) {
+ if !a.layout().ty.is_simd() {
+ report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
+ return;
+ }
+
+ simd_for_each_lane(fx, a, ret, &|fx, lane_ty, _ret_lane_ty, lane| {
+ match lane_ty.kind() {
+ ty::Float(_) => {},
+ _ => unreachable!("{:?}", lane_ty),
+ }
+ match intrinsic {
+ sym::simd_fabs => fx.bcx.ins().fabs(lane),
+ sym::simd_fsqrt => fx.bcx.ins().sqrt(lane),
+ sym::simd_ceil => fx.bcx.ins().ceil(lane),
+ sym::simd_floor => fx.bcx.ins().floor(lane),
+ sym::simd_trunc => fx.bcx.ins().trunc(lane),
+ _ => unreachable!(),
+ }
+ });
+ };
+
+ simd_reduce_add_ordered | simd_reduce_add_unordered, (c v, v acc) {
++ // FIXME there must be no acc param for integer vectors
+ if !v.layout().ty.is_simd() {
+ report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
+ return;
+ }
+
+ simd_reduce(fx, v, Some(acc), ret, &|fx, lane_ty, a, b| {
+ if lane_ty.is_floating_point() {
+ fx.bcx.ins().fadd(a, b)
+ } else {
+ fx.bcx.ins().iadd(a, b)
+ }
+ });
+ };
+
+ simd_reduce_mul_ordered | simd_reduce_mul_unordered, (c v, v acc) {
++ // FIXME there must be no acc param for integer vectors
+ if !v.layout().ty.is_simd() {
+ report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
+ return;
+ }
+
+ simd_reduce(fx, v, Some(acc), ret, &|fx, lane_ty, a, b| {
+ if lane_ty.is_floating_point() {
+ fx.bcx.ins().fmul(a, b)
+ } else {
+ fx.bcx.ins().imul(a, b)
+ }
+ });
+ };
+
+ simd_reduce_all, (c v) {
+ if !v.layout().ty.is_simd() {
+ report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
+ return;
+ }
+
+ simd_reduce_bool(fx, v, ret, &|fx, a, b| fx.bcx.ins().band(a, b));
+ };
+
+ simd_reduce_any, (c v) {
+ if !v.layout().ty.is_simd() {
+ report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
+ return;
+ }
+
+ simd_reduce_bool(fx, v, ret, &|fx, a, b| fx.bcx.ins().bor(a, b));
+ };
+
+ simd_reduce_and, (c v) {
+ if !v.layout().ty.is_simd() {
+ report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
+ return;
+ }
+
+ simd_reduce(fx, v, None, ret, &|fx, _ty, a, b| fx.bcx.ins().band(a, b));
+ };
+
+ simd_reduce_or, (c v) {
+ if !v.layout().ty.is_simd() {
+ report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
+ return;
+ }
+
+ simd_reduce(fx, v, None, ret, &|fx, _ty, a, b| fx.bcx.ins().bor(a, b));
+ };
+
+ simd_reduce_xor, (c v) {
+ if !v.layout().ty.is_simd() {
+ report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
+ return;
+ }
+
+ simd_reduce(fx, v, None, ret, &|fx, _ty, a, b| fx.bcx.ins().bxor(a, b));
+ };
+
+ simd_reduce_min, (c v) {
+ if !v.layout().ty.is_simd() {
+ report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
+ return;
+ }
+
+ simd_reduce(fx, v, None, ret, &|fx, ty, a, b| {
+ let lt = match ty.kind() {
+ ty::Int(_) => fx.bcx.ins().icmp(IntCC::SignedLessThan, a, b),
+ ty::Uint(_) => fx.bcx.ins().icmp(IntCC::UnsignedLessThan, a, b),
+ ty::Float(_) => fx.bcx.ins().fcmp(FloatCC::LessThan, a, b),
+ _ => unreachable!(),
+ };
+ fx.bcx.ins().select(lt, a, b)
+ });
+ };
+
+ simd_reduce_max, (c v) {
+ if !v.layout().ty.is_simd() {
+ report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
+ return;
+ }
+
+ simd_reduce(fx, v, None, ret, &|fx, ty, a, b| {
+ let gt = match ty.kind() {
+ ty::Int(_) => fx.bcx.ins().icmp(IntCC::SignedGreaterThan, a, b),
+ ty::Uint(_) => fx.bcx.ins().icmp(IntCC::UnsignedGreaterThan, a, b),
+ ty::Float(_) => fx.bcx.ins().fcmp(FloatCC::GreaterThan, a, b),
+ _ => unreachable!(),
+ };
+ fx.bcx.ins().select(gt, a, b)
+ });
+ };
+
+ simd_select, (c m, c a, c b) {
+ if !m.layout().ty.is_simd() {
+ report_simd_type_validation_error(fx, intrinsic, span, m.layout().ty);
+ return;
+ }
+ if !a.layout().ty.is_simd() {
+ report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
+ return;
+ }
+ assert_eq!(a.layout(), b.layout());
+
+ let (lane_count, lane_ty) = a.layout().ty.simd_size_and_type(fx.tcx);
+ let lane_layout = fx.layout_of(lane_ty);
+
+ for lane in 0..lane_count {
+ let m_lane = m.value_lane(fx, lane).load_scalar(fx);
+ let a_lane = a.value_lane(fx, lane).load_scalar(fx);
+ let b_lane = b.value_lane(fx, lane).load_scalar(fx);
+
+ let m_lane = fx.bcx.ins().icmp_imm(IntCC::Equal, m_lane, 0);
+ let res_lane = CValue::by_val(fx.bcx.ins().select(m_lane, b_lane, a_lane), lane_layout);
+
+ ret.place_lane(fx, lane).write_cvalue(fx, res_lane);
+ }
+ };
+
+ // simd_saturating_*
+ // simd_bitmask
+ // simd_scatter
+ // simd_gather
+ }
+}
--- /dev/null
- #![feature(rustc_private, decl_macro)]
- #![cfg_attr(feature = "jit", feature(never_type, vec_into_raw_parts, once_cell))]
++#![feature(rustc_private)]
++// Note: please avoid adding other feature gates where possible
+#![warn(rust_2018_idioms)]
+#![warn(unused_lifetimes)]
+#![warn(unreachable_pub)]
+
+#[macro_use]
+extern crate rustc_middle;
+extern crate rustc_ast;
+extern crate rustc_codegen_ssa;
+extern crate rustc_data_structures;
+extern crate rustc_errors;
+extern crate rustc_fs_util;
+extern crate rustc_hir;
+extern crate rustc_incremental;
+extern crate rustc_index;
+extern crate rustc_interface;
+extern crate rustc_metadata;
+extern crate rustc_session;
+extern crate rustc_span;
+extern crate rustc_target;
+
+// This prevents duplicating functions and statics that are already part of the host rustc process.
+#[allow(unused_extern_crates)]
+extern crate rustc_driver;
+
+use std::any::Any;
+use std::cell::Cell;
+
+use rustc_codegen_ssa::traits::CodegenBackend;
+use rustc_codegen_ssa::CodegenResults;
+use rustc_errors::ErrorGuaranteed;
+use rustc_metadata::EncodedMetadata;
+use rustc_middle::dep_graph::{WorkProduct, WorkProductId};
+use rustc_session::config::OutputFilenames;
+use rustc_session::Session;
+use rustc_span::Symbol;
+
+use cranelift_codegen::isa::TargetIsa;
+use cranelift_codegen::settings::{self, Configurable};
+
+pub use crate::config::*;
+use crate::prelude::*;
+
+mod abi;
+mod allocator;
+mod analyze;
+mod archive;
+mod base;
+mod cast;
+mod codegen_i128;
+mod common;
+mod compiler_builtins;
+mod config;
+mod constant;
+mod debuginfo;
+mod discriminant;
+mod driver;
+mod inline_asm;
+mod intrinsics;
+mod linkage;
+mod main_shim;
+mod num;
+mod optimize;
+mod pointer;
+mod pretty_clif;
+mod toolchain;
+mod trap;
+mod unsize;
+mod value_and_place;
+mod vtable;
+
+mod prelude {
+ pub(crate) use rustc_span::{FileNameDisplayPreference, Span};
+
+ pub(crate) use rustc_hir::def_id::{DefId, LOCAL_CRATE};
+ pub(crate) use rustc_middle::bug;
+ pub(crate) use rustc_middle::mir::{self, *};
+ pub(crate) use rustc_middle::ty::layout::{self, LayoutOf, TyAndLayout};
+ pub(crate) use rustc_middle::ty::{
+ self, FloatTy, Instance, InstanceDef, IntTy, ParamEnv, Ty, TyCtxt, TypeAndMut,
+ TypeFoldable, UintTy,
+ };
+ pub(crate) use rustc_target::abi::{Abi, Scalar, Size, VariantIdx};
+
+ pub(crate) use rustc_data_structures::fx::FxHashMap;
+
+ pub(crate) use rustc_index::vec::Idx;
+
+ pub(crate) use cranelift_codegen::ir::condcodes::{FloatCC, IntCC};
+ pub(crate) use cranelift_codegen::ir::function::Function;
+ pub(crate) use cranelift_codegen::ir::types;
+ pub(crate) use cranelift_codegen::ir::{
+ AbiParam, Block, ExternalName, FuncRef, Inst, InstBuilder, MemFlags, Signature, SourceLoc,
+ StackSlot, StackSlotData, StackSlotKind, TrapCode, Type, Value,
+ };
+ pub(crate) use cranelift_codegen::isa::{self, CallConv};
+ pub(crate) use cranelift_codegen::Context;
+ pub(crate) use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext, Variable};
+ pub(crate) use cranelift_module::{self, DataContext, FuncId, Linkage, Module};
+
+ pub(crate) use crate::abi::*;
+ pub(crate) use crate::base::{codegen_operand, codegen_place};
+ pub(crate) use crate::cast::*;
+ pub(crate) use crate::common::*;
+ pub(crate) use crate::debuginfo::{DebugContext, UnwindContext};
+ pub(crate) use crate::pointer::Pointer;
- pub(crate) use crate::trap::*;
+ pub(crate) use crate::value_and_place::{CPlace, CPlaceInner, CValue};
+}
+
+struct PrintOnPanic<F: Fn() -> String>(F);
+impl<F: Fn() -> String> Drop for PrintOnPanic<F> {
+ fn drop(&mut self) {
+ if ::std::thread::panicking() {
+ println!("{}", (self.0)());
+ }
+ }
+}
+
+/// The codegen context holds any information shared between the codegen of individual functions
+/// inside a single codegen unit with the exception of the Cranelift [`Module`](cranelift_module::Module).
+struct CodegenCx<'tcx> {
+ tcx: TyCtxt<'tcx>,
+ global_asm: String,
+ inline_asm_index: Cell<usize>,
+ cached_context: Context,
+ debug_context: Option<DebugContext<'tcx>>,
+ unwind_context: UnwindContext,
+ cgu_name: Symbol,
+}
+
+impl<'tcx> CodegenCx<'tcx> {
+ fn new(
+ tcx: TyCtxt<'tcx>,
+ backend_config: BackendConfig,
+ isa: &dyn TargetIsa,
+ debug_info: bool,
+ cgu_name: Symbol,
+ ) -> Self {
+ assert_eq!(pointer_ty(tcx), isa.pointer_type());
+
+ let unwind_context =
+ UnwindContext::new(isa, matches!(backend_config.codegen_mode, CodegenMode::Aot));
+ let debug_context = if debug_info { Some(DebugContext::new(tcx, isa)) } else { None };
+ CodegenCx {
+ tcx,
+ global_asm: String::new(),
+ inline_asm_index: Cell::new(0),
+ cached_context: Context::new(),
+ debug_context,
+ unwind_context,
+ cgu_name,
+ }
+ }
+}
+
+pub struct CraneliftCodegenBackend {
+ pub config: Option<BackendConfig>,
+}
+
+impl CodegenBackend for CraneliftCodegenBackend {
+ fn init(&self, sess: &Session) {
+ use rustc_session::config::Lto;
+ match sess.lto() {
+ Lto::No | Lto::ThinLocal => {}
+ Lto::Thin | Lto::Fat => sess.warn("LTO is not supported. You may get a linker error."),
+ }
+ }
+
+ fn target_features(&self, _sess: &Session) -> Vec<rustc_span::Symbol> {
+ vec![]
+ }
+
+ fn print_version(&self) {
+ println!("Cranelift version: {}", cranelift_codegen::VERSION);
+ }
+
+ fn codegen_crate(
+ &self,
+ tcx: TyCtxt<'_>,
+ metadata: EncodedMetadata,
+ need_metadata_module: bool,
+ ) -> Box<dyn Any> {
+ tcx.sess.abort_if_errors();
+ let config = if let Some(config) = self.config.clone() {
+ config
+ } else {
+ if !tcx.sess.unstable_options() && !tcx.sess.opts.cg.llvm_args.is_empty() {
+ tcx.sess.fatal("`-Z unstable-options` must be passed to allow configuring cg_clif");
+ }
+ BackendConfig::from_opts(&tcx.sess.opts.cg.llvm_args)
+ .unwrap_or_else(|err| tcx.sess.fatal(&err))
+ };
+ match config.codegen_mode {
+ CodegenMode::Aot => driver::aot::run_aot(tcx, config, metadata, need_metadata_module),
+ CodegenMode::Jit | CodegenMode::JitLazy => {
+ #[cfg(feature = "jit")]
- let _: ! = driver::jit::run_jit(tcx, config);
++ driver::jit::run_jit(tcx, config);
+
+ #[cfg(not(feature = "jit"))]
+ tcx.sess.fatal("jit support was disabled when compiling rustc_codegen_cranelift");
+ }
+ }
+ }
+
+ fn join_codegen(
+ &self,
+ ongoing_codegen: Box<dyn Any>,
+ _sess: &Session,
+ _outputs: &OutputFilenames,
+ ) -> Result<(CodegenResults, FxHashMap<WorkProductId, WorkProduct>), ErrorGuaranteed> {
+ Ok(*ongoing_codegen
+ .downcast::<(CodegenResults, FxHashMap<WorkProductId, WorkProduct>)>()
+ .unwrap())
+ }
+
+ fn link(
+ &self,
+ sess: &Session,
+ codegen_results: CodegenResults,
+ outputs: &OutputFilenames,
+ ) -> Result<(), ErrorGuaranteed> {
+ use rustc_codegen_ssa::back::link::link_binary;
+
+ link_binary::<crate::archive::ArArchiveBuilder<'_>>(sess, &codegen_results, outputs)
+ }
+}
+
+fn target_triple(sess: &Session) -> target_lexicon::Triple {
+ match sess.target.llvm_target.parse() {
+ Ok(triple) => triple,
+ Err(err) => sess.fatal(&format!("target not recognized: {}", err)),
+ }
+}
+
+fn build_isa(sess: &Session, backend_config: &BackendConfig) -> Box<dyn isa::TargetIsa + 'static> {
+ use target_lexicon::BinaryFormat;
+
+ let target_triple = crate::target_triple(sess);
+
+ let mut flags_builder = settings::builder();
+ flags_builder.enable("is_pic").unwrap();
+ flags_builder.set("enable_probestack", "false").unwrap(); // __cranelift_probestack is not provided
+ let enable_verifier = if backend_config.enable_verifier { "true" } else { "false" };
+ flags_builder.set("enable_verifier", enable_verifier).unwrap();
+
+ let tls_model = match target_triple.binary_format {
+ BinaryFormat::Elf => "elf_gd",
+ BinaryFormat::Macho => "macho",
+ BinaryFormat::Coff => "coff",
+ _ => "none",
+ };
+ flags_builder.set("tls_model", tls_model).unwrap();
+
+ flags_builder.set("enable_simd", "true").unwrap();
+
+ flags_builder.set("enable_llvm_abi_extensions", "true").unwrap();
+
+ flags_builder.set("regalloc", &backend_config.regalloc).unwrap();
+
+ use rustc_session::config::OptLevel;
+ match sess.opts.optimize {
+ OptLevel::No => {
+ flags_builder.set("opt_level", "none").unwrap();
+ }
+ OptLevel::Less | OptLevel::Default => {}
+ OptLevel::Size | OptLevel::SizeMin | OptLevel::Aggressive => {
+ flags_builder.set("opt_level", "speed_and_size").unwrap();
+ }
+ }
+
+ let flags = settings::Flags::new(flags_builder);
+
+ let isa_builder = match sess.opts.cg.target_cpu.as_deref() {
+ Some("native") => {
+ let builder = cranelift_native::builder_with_options(true).unwrap();
+ builder
+ }
+ Some(value) => {
+ let mut builder =
+ cranelift_codegen::isa::lookup(target_triple.clone()).unwrap_or_else(|err| {
+ sess.fatal(&format!("can't compile for {}: {}", target_triple, err));
+ });
+ if let Err(_) = builder.enable(value) {
+ sess.fatal("the specified target cpu isn't currently supported by Cranelift.");
+ }
+ builder
+ }
+ None => {
+ let mut builder =
+ cranelift_codegen::isa::lookup(target_triple.clone()).unwrap_or_else(|err| {
+ sess.fatal(&format!("can't compile for {}: {}", target_triple, err));
+ });
+ if target_triple.architecture == target_lexicon::Architecture::X86_64 {
+ // Don't use "haswell" as the default, as it implies `has_lzcnt`.
+ // macOS CI is still at Ivy Bridge EP, so `lzcnt` is interpreted as `bsr`.
+ builder.enable("nehalem").unwrap();
+ }
+ builder
+ }
+ };
+
- isa_builder.finish(flags)
++ match isa_builder.finish(flags) {
++ Ok(target_isa) => target_isa,
++ Err(err) => sess.fatal(&format!("failed to build TargetIsa: {}", err)),
++ }
+}
+
+/// This is the entrypoint for a hot plugged rustc_codegen_cranelift
+#[no_mangle]
+pub fn __rustc_codegen_backend() -> Box<dyn CodegenBackend> {
+ Box::new(CraneliftCodegenBackend { config: None })
+}
--- /dev/null
- /// Trap code: user1
- pub(crate) fn trap_abort(fx: &mut FunctionCx<'_, '_, '_>, msg: impl AsRef<str>) {
- codegen_print(fx, msg.as_ref());
- fx.bcx.ins().trap(TrapCode::User(1));
- }
-
+//! Helpers used to print a message and abort in case of certain panics and some detected UB.
+
+use crate::prelude::*;
+
+fn codegen_print(fx: &mut FunctionCx<'_, '_, '_>, msg: &str) {
+ let puts = fx
+ .module
+ .declare_function(
+ "puts",
+ Linkage::Import,
+ &Signature {
+ call_conv: fx.target_config.default_call_conv,
+ params: vec![AbiParam::new(fx.pointer_type)],
+ returns: vec![AbiParam::new(types::I32)],
+ },
+ )
+ .unwrap();
+ let puts = fx.module.declare_func_in_func(puts, &mut fx.bcx.func);
+ if fx.clif_comments.enabled() {
+ fx.add_comment(puts, "puts");
+ }
+
+ let real_msg = format!("trap at {:?} ({}): {}\0", fx.instance, fx.symbol_name, msg);
+ let msg_ptr = fx.anonymous_str(&real_msg);
+ fx.bcx.ins().call(puts, &[msg_ptr]);
+}
+
-
- /// Like `trap_unreachable` but returns a fake value of the specified type.
- ///
- /// Trap code: user65535
- pub(crate) fn trap_unreachable_ret_value<'tcx>(
- fx: &mut FunctionCx<'_, '_, 'tcx>,
- dest_layout: TyAndLayout<'tcx>,
- msg: impl AsRef<str>,
- ) -> CValue<'tcx> {
- codegen_print(fx, msg.as_ref());
- let true_ = fx.bcx.ins().iconst(types::I32, 1);
- fx.bcx.ins().trapnz(true_, TrapCode::UnreachableCodeReached);
- CValue::by_ref(Pointer::const_addr(fx, 0), dest_layout)
- }
-
+/// Use this for example when a function call should never return. This will fill the current block,
+/// so you can **not** add instructions to it afterwards.
+///
+/// Trap code: user65535
+pub(crate) fn trap_unreachable(fx: &mut FunctionCx<'_, '_, '_>, msg: impl AsRef<str>) {
+ codegen_print(fx, msg.as_ref());
+ fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
+}
+/// Use this when something is unimplemented, but `libcore` or `libstd` requires it to codegen.
+/// Unlike `trap_unreachable` this will not fill the current block, so you **must** add instructions
+/// to it afterwards.
+///
+/// Trap code: user65535
+pub(crate) fn trap_unimplemented(fx: &mut FunctionCx<'_, '_, '_>, msg: impl AsRef<str>) {
+ codegen_print(fx, msg.as_ref());
+ let true_ = fx.bcx.ins().iconst(types::I32, 1);
+ fx.bcx.ins().trapnz(true_, TrapCode::User(!0));
+}
+
+/// Like `trap_unimplemented` but returns a fake value of the specified type.
+///
+/// Trap code: user65535
+pub(crate) fn trap_unimplemented_ret_value<'tcx>(
+ fx: &mut FunctionCx<'_, '_, 'tcx>,
+ dest_layout: TyAndLayout<'tcx>,
+ msg: impl AsRef<str>,
+) -> CValue<'tcx> {
+ trap_unimplemented(fx, msg);
+ CValue::by_ref(Pointer::const_addr(fx, 0), dest_layout)
+}
--- /dev/null
- #![allow()] /*This line is ignored by bash
+#!/usr/bin/env bash
- rustc $0 -o ${0/.rs/.bin} -g
++#![deny(unsafe_code)] /*This line is ignored by bash
+# This block is ignored by rustc
+set -e
+echo "[BUILD] y.rs" 1>&2
++rustc $0 -o ${0/.rs/.bin} -Cdebuginfo=1
+exec ${0/.rs/.bin} $@
+*/
+
+//! The build system for cg_clif
+//!
+//! # Manual compilation
+//!
+//! If your system doesn't support shell scripts you can manually compile and run this file using
+//! for example:
+//!
+//! ```shell
+//! $ rustc y.rs -o y.bin
+//! $ ./y.bin
+//! ```
+//!
+//! # Naming
+//!
+//! The name `y.rs` was chosen to not conflict with rustc's `x.py`.
+
+#[path = "build_system/mod.rs"]
+mod build_system;
+
+fn main() {
+ build_system::main();
+}
projects / rust.git / commitdiff