Vendor dependencies (#2883)

This allows us to simplify the logic which ensures that we never depend
on crate versions which are incompatible with our MSRV. It also avoids a
performance bottleneck on our MSRV in which updating the crates.io index
is very slow.

Closes #1595

gherrit-pr-id: Gd8dc83951469fd1467ddb65d2ac524b709fe9503

Author: joshlf

.cargo/config.toml

@@ -0,0 +1,6 @@
+[source.crates-io]
+replace-with = "vendored-sources"
+
+[source.vendored-sources]
+directory = "vendor"
+

.github/workflows/ci.yml

@@ -261,19 +261,6 @@ jobs:
     - name: Populate cache
       uses: ./.github/actions/cache
 
-    # Ensure that Cargo resolves the minimum possible syn version so that if we
-    # accidentally make a change which depends upon features added in more
-    # recent versions of syn, we'll catch it in CI.
-    #
-    # TODO(#1595): Debug why this step is still necessary after #1564 and maybe
-    # remove it.
-    - name: Pin syn dependency
-      run: |
-        set -eo pipefail
-        # Override the exising `syn` dependency with one which requires an exact
-        # version.
-        cargo add -p zerocopy-derive 'syn@=2.0.46'
-
     - name: Configure environment variables
       env:
         TOOLCHAIN: ${{ matrix.toolchain }}
@@ -460,6 +447,15 @@ jobs:
       run: |
         set -eo pipefail
 
+        # FIXME(#2906): We do this because `cargo vendor` doesn't currently
+        # support vendoring std's dependencies (required in order to build std
+        # from source, which Miri does). As a workaround, we simply temporarily
+        # remove the cargo config and bypass vendoring altogether. Eventually,
+        # we should get vendoring working for std's dependencies too.
+        #
+        # See also: https://github.com/rust-lang/wg-cargo-std-aware/issues/23
+        mv .cargo/config.toml .cargo/config.toml.bak
+
         # Work around https://github.com/rust-lang/miri/issues/3125
         [ "$TARGET" == "aarch64-unknown-linux-gnu" ] && cargo clean
 
@@ -479,6 +475,8 @@ jobs:
             --target $TARGET \
             $FEATURES
         done
+
+        mv .cargo/config.toml.bak .cargo/config.toml
       # Only nightly has a working Miri, so we skip installing on all other
       # toolchains.
       #
@@ -567,6 +565,18 @@ jobs:
           echo "ZC_SKIP_CARGO_SEMVER_CHECKS=1" >> $GITHUB_ENV
         fi
 
+    # FIXME(#2906): We do this because `cargo semver-checks` fetches the latest
+    # zerocopy from crates.io, but `.cargo/config.toml` causes that to resolve
+    # in our vendor directory, and we don't vendor zerocopy. Removing this file
+    # has the effect of causing the subsequent build to use crates.io rather
+    # than vendored dependencies, which is fine since we only run this on the
+    # stable toolchain. Eventually, we should update this job to use the
+    # `--baseline-rev` option to use a previous git commit as the baseline for
+    # checking compatibility (rather than the most recent published version),
+    # which will make this unnecessary.
+    - name: Remove Cargo config
+      run: rm .cargo/config.toml
+
     # Check semver compatibility with the most recently-published version on
     # crates.io. We do this in the matrix rather than in its own job so that it
     # gets run on different targets. Some of our API is target-specific (e.g.,
@@ -685,7 +695,19 @@ jobs:
             toolchain: ${{ env.ZC_TOOLCHAIN }}
             components: clippy, rust-src
       - name: Check big endian for aarch64_be-unknown-linux-gnu target
-        run: ./cargo.sh +nightly build --target=aarch64_be-unknown-linux-gnu -Zbuild-std --features simd
+        run: |
+          set -eo pipefail
+
+          # FIXME(#2906): We do this because `cargo vendor` doesn't currently
+          # support vendoring std's dependencies (required in order to build
+          # std from source, as we do here). As a workaround, we simply nuke
+          # the cargo config and bypass vendoring altogether. Eventually, we
+          # should get vendoring working for std's dependencies too.
+          #
+          # See also: https://github.com/rust-lang/wg-cargo-std-aware/issues/23
+          rm .cargo/config.toml
+
+          ./cargo.sh +nightly build --target=aarch64_be-unknown-linux-gnu -Zbuild-std --features simd
 
   # We can't use this as part of the build matrix because rustup doesn't support
   # the `avr-none` target.
@@ -716,6 +738,15 @@ jobs:
         with:
             toolchain: ${{ env.ZC_TOOLCHAIN }}
             components: clippy, rust-src
+      # FIXME(#2906): We do this because `cargo vendor` doesn't currently
+      # support vendoring std's dependencies (required in order to build std
+      # from source, as we do here). As a workaround, we simply nuke the cargo
+      # config and bypass vendoring altogether. Eventually, we should get
+      # vendoring working for std's dependencies too.
+      #
+      # See also: https://github.com/rust-lang/wg-cargo-std-aware/issues/23
+      - name: Remove Cargo config
+        run: rm .cargo/config.toml
       # NOTE: We cannot check tests because of a number of different issues (at
       # the time of writing):
       # - No `alloc::sync`
@@ -851,14 +882,6 @@ jobs:
             #
             # [1]  https://stackoverflow.com/a/42139535/836390
 
-            # See comment on "Pin syn dependency" job for why we do this. It needs
-            # to happen before the subsequent `cargo check`, so we don't
-            # background it.
-            #
-            # TODO(#1595): Debug why this step is still necessary after #1564 and
-            # maybe remove it.
-            cargo add -p zerocopy-derive 'syn@=2.0.46' &> /dev/null
-
             cargo check --workspace --tests                         &> /dev/null &
             # On our MSRV toolchain, updating the Cargo index takes a long time, so
             # it is worth specifically caching the MSRV index.
@@ -947,6 +970,8 @@ jobs:
           persist-credentials: false
       - uses: zizmorcore/zizmor-action@e639db99335bc9038abc0e066dfcd72e23d26fb4 # v0.3.0
         with:
+          # Only scan the .github directory to avoid scanning vendored dependencies
+          inputs: .github
           # We don't want to use GitHub Advanced Security because we want to
           # block the merge on zizmor failure, and the only way to do that
           # (without manually configuring a ruleset) is to fail the job, which

.gitignore

@@ -13,3 +13,8 @@ lcov.info
 
 # VSCode workspace files
 *.code-workspace
+
+# Unconditionally commit everything inside vendor directories (overrides
+# previous rules)
+!vendor/**
+!tools/vendor/**

.rgignore

@@ -0,0 +1,3 @@
+vendor/
+tools/vendor/
+

Cargo.toml

@@ -121,19 +121,11 @@ zerocopy-derive = { version = "=0.8.33", path = "zerocopy-derive", optional = tr
 zerocopy-derive = { version = "=0.8.33", path = "zerocopy-derive" }
 
 [dev-dependencies]
-# More recent versions of `either` have an MSRV higher than ours.
-either = "=1.13.0"
 # FIXME(#381) Remove this dependency once we have our own layout gadgets.
 elain = "0.3.0"
-# More recent versions of `glob` have an MSRV higher than ours.
-glob = "=0.3.2"
 itertools = "0.11"
-# More recent versions of `itoa` have an MSRV higher than ours.
-itoa = "=1.0.15"
 rand = { version = "0.8.5", default-features = false, features = ["small_rng"] }
 rustversion = "1.0"
-# More recent versions of `ryu` have an MSRV higher than ours.
-ryu = "=1.0.20"
 static_assertions = "1.1"
 testutil = { path = "testutil" }
 # Pinned to a specific version so that the version used for local development

Cargo.toml.std

@@ -0,0 +1,58 @@
+[package]
+name = "std-deps"
+version = "0.0.0"
+edition = "2021"
+publish = false
+
+# This manifest exists solely to force `cargo vendor` to vendor dependencies
+# required by `std` when building with `-Zbuild-std[=core]`. When running
+# `cargo vendor`, make sure to include this manifest using `--sync`.
+#
+# See: https://github.com/rust-lang/wg-cargo-std-aware/issues/23
+
+[workspace]
+
+[dependencies]
+# Dependencies from std/Cargo.toml and transitive deps from library/Cargo.lock
+addr2line = { version = "=0.25.1", default-features = false }
+adler2 = { version = "=2.0.1", default-features = false }
+cc = { version = "=1.2.0", default-features = false }
+cfg-if = { version = "=1.0.4", default-features = false }
+dlmalloc = { version = "=0.2.11", default-features = false }
+foldhash = { version = "=0.2.0", default-features = false }
+fortanix-sgx-abi = { version = "=0.6.1", default-features = false }
+getopts = { version = "=0.2.24", default-features = false }
+gimli = { version = "=0.32.3", default-features = false }
+hashbrown = { version = "=0.16.1", default-features = false }
+hermit-abi = { version = "=0.5.2", default-features = false }
+libc = { version = "=0.2.178", default-features = false }
+memchr = { version = "=2.7.6", default-features = false }
+miniz_oxide = { version = "=0.8.9", default-features = false }
+moto-rt = { version = "=0.16.0", default-features = false }
+object = { version = "=0.37.3", default-features = false }
+r-efi = { version = "=5.3.0", default-features = false }
+r-efi-alloc = { version = "=2.1.0", default-features = false }
+rand = { version = "=0.9.2", default-features = false }
+rand_core = { version = "=0.9.3", default-features = false }
+rand_xorshift = { version = "=0.4.0", default-features = false }
+rustc-demangle = "=0.1.26"
+rustc-literal-escaper = { version = "=0.0.7", default-features = false }
+shlex = { version = "=1.3.0", default-features = false }
+unwinding = { version = "=0.2.8", default-features = false }
+vex-sdk = { version = "=0.27.1", default-features = false }
+wasi-snapshot-preview1 = { package = "wasi", version = "=0.11.1+wasi-snapshot-preview1", default-features = false }
+wasi-p2 = { package = "wasi", version = "=0.14.4+wasi-0.2.4", default-features = false }
+windows-link = { version = "=0.2.1", default-features = false }
+windows-sys = { version = "=0.60.2", default-features = false }
+windows-targets = { version = "=0.53.5", default-features = false }
+wit-bindgen = { version = "=0.45.1", default-features = false }
+
+# Windows targets dependencies (transitive via windows-targets)
+windows_aarch64_gnullvm = { version = "=0.53.1", default-features = false }
+windows_aarch64_msvc = { version = "=0.53.1", default-features = false }
+windows_i686_gnu = { version = "=0.53.1", default-features = false }
+windows_i686_gnullvm = { version = "=0.53.1", default-features = false }
+windows_i686_msvc = { version = "=0.53.1", default-features = false }
+windows_x86_64_gnu = { version = "=0.53.1", default-features = false }
+windows_x86_64_gnullvm = { version = "=0.53.1", default-features = false }
+windows_x86_64_msvc = { version = "=0.53.1", default-features = false }

cargo.sh

@@ -10,6 +10,6 @@ set -eo pipefail
 
 # Build `cargo-zerocopy` without any RUSTFLAGS or CARGO_TARGET_DIR set in the
 # environment
-env -u RUSTFLAGS -u CARGO_TARGET_DIR cargo +stable build --manifest-path tools/Cargo.toml -p cargo-zerocopy -q
+env -u RUSTFLAGS -u CARGO_TARGET_DIR cargo +stable build --config tools/.cargo/config.toml --manifest-path tools/Cargo.toml -p cargo-zerocopy -q
 # Thin wrapper around the `cargo-zerocopy` binary in `tools/cargo-zerocopy`
 ./tools/target/debug/cargo-zerocopy $@

ci/check_fmt.sh

@@ -9,7 +9,7 @@
 # those terms.
 
 set -eo pipefail
-files=$(find . -iname '*.rs' -type f -not -path './target/*' -not -iname '*.expected.rs')
+files=$(find . -iname '*.rs' -type f -not -path './target/*' -not -iname '*.expected.rs' -not -path './vendor/*' -not -path './tools/vendor/*')
 # check that find succeeded
 if [[ -z $files ]]
 then

ci/check_readme.sh

@@ -15,5 +15,5 @@ set -eo pipefail
 # suppress all errors from it.
 cargo install -q cargo-readme --version 3.2.0
 
-diff <(cargo -q run --manifest-path tools/Cargo.toml -p generate-readme) README.md >&2
+diff <(cargo -q run --config tools/.cargo/config.toml --manifest-path tools/Cargo.toml -p generate-readme) README.md >&2
 exit $?

tools/.cargo/config.toml

@@ -0,0 +1,6 @@
+[source.crates-io]
+replace-with = "vendored-sources"
+
+[source.vendored-sources]
+directory = "vendor"
+