build #9863
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: build | |
| on: | |
| push: | |
| branches: [main, "rust-v*"] | |
| pull_request: | |
| branches: [main, "rust-v*"] | |
| merge_group: | |
| env: | |
| DEFAULT_FEATURES: "azure,datafusion,s3,gcs,glue,hdfs " | |
| # Disable full debug symbol generation to speed up CI build and keep memory down | |
| RUSTFLAGS: -C debuginfo=line-tables-only | |
| # Disable incremental builds by cargo for CI which should save disk space | |
| # and hopefully avoid final link "No space left on device" | |
| CARGO_INCREMENTAL: 0 | |
| SCCACHE_GHA_ENABLED: "true" | |
| RUSTC_WRAPPER: "sccache" | |
| jobs: | |
| # run various build configurations, fmt, and clippy. | |
| build: | |
| strategy: | |
| fail-fast: true | |
| matrix: | |
| os: | |
| - ubuntu-latest | |
| - windows-latest | |
| - macos-latest | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Run sccache-cache | |
| uses: mozilla-actions/[email protected] | |
| - name: Install minimal stable with clippy and rustfmt | |
| uses: actions-rs/toolchain@v1 | |
| with: | |
| profile: default | |
| toolchain: "1.85" | |
| override: true | |
| - name: Format | |
| run: cargo fmt -- --check | |
| - name: Default build | |
| run: (cd crates/deltalake && cargo build --tests) | |
| - name: build and lint with clippy | |
| run: cargo clippy --features ${{ env.DEFAULT_FEATURES }} --tests | |
| - name: Spot-check build for native-tls features | |
| run: cargo clippy --no-default-features --features azure,datafusion,s3-native-tls,gcs,glue --tests | |
| - name: Check no default features (except rustls) | |
| run: cargo check --no-default-features --features rustls | |
| - name: Check docs | |
| run: cargo doc --no-deps --features ${{ env.DEFAULT_FEATURES }} | |
| unit_test: | |
| name: Unit Tests | |
| strategy: | |
| fail-fast: true | |
| matrix: | |
| os: | |
| - ubuntu-latest | |
| - windows-latest | |
| - macos-latest | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Run sccache-cache | |
| uses: mozilla-actions/[email protected] | |
| - name: Install minimal stable with clippy and rustfmt | |
| uses: actions-rs/toolchain@v1 | |
| with: | |
| profile: default | |
| toolchain: "1.85" | |
| override: true | |
| - name: Run tests | |
| run: | | |
| make setup-dat | |
| cargo test --features ${{ env.DEFAULT_FEATURES }} | |
| integration_test: | |
| name: Integration Tests | |
| runs-on: ubuntu-latest | |
| env: | |
| # https://github.com/rust-lang/cargo/issues/10280 | |
| CARGO_NET_GIT_FETCH_WITH_CLI: "true" | |
| AWS_DEFAULT_REGION: "us-east-1" | |
| AWS_ACCESS_KEY_ID: deltalake | |
| AWS_SECRET_ACCESS_KEY: weloverust | |
| AWS_ENDPOINT_URL: http://localhost:4566 | |
| AWS_ALLOW_HTTP: "1" | |
| AZURE_USE_EMULATOR: "1" | |
| AZURE_STORAGE_ALLOW_HTTP: "1" | |
| AZURITE_BLOB_STORAGE_URL: "http://localhost:10000" | |
| AZURE_STORAGE_CONNECTION_STRING: "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://localhost:10000/devstoreaccount1;QueueEndpoint=http://localhost:10001/devstoreaccount1;" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Run sccache-cache | |
| uses: mozilla-actions/[email protected] | |
| - name: Install minimal stable with clippy and rustfmt | |
| uses: actions-rs/toolchain@v1 | |
| with: | |
| profile: default | |
| toolchain: "1.85" | |
| override: true | |
| - name: Install cargo-llvm-cov | |
| uses: taiki-e/install-action@cargo-llvm-cov | |
| - name: Start emulated services | |
| run: docker compose up -d | |
| - name: Run tests with rustls (default) | |
| run: | | |
| gmake setup-dat | |
| cargo llvm-cov \ | |
| --features integration_test,${{ env.DEFAULT_FEATURES }} \ | |
| --workspace \ | |
| --exclude delta-inspect \ | |
| --exclude deltalake-hdfs \ | |
| --exclude deltalake-lakefs \ | |
| --codecov \ | |
| --output-path codecov.json | |
| - name: Upload coverage to Codecov | |
| uses: codecov/codecov-action@v4 | |
| with: | |
| files: codecov.json | |
| fail_ci_if_error: true | |
| env: | |
| CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} | |
| integration_test_native_tls: | |
| name: Integration Tests (Native TLS) | |
| runs-on: ubuntu-latest | |
| env: | |
| # https://github.com/rust-lang/cargo/issues/10280 | |
| CARGO_NET_GIT_FETCH_WITH_CLI: "true" | |
| AWS_DEFAULT_REGION: "us-east-1" | |
| AWS_ACCESS_KEY_ID: deltalake | |
| AWS_SECRET_ACCESS_KEY: weloverust | |
| AWS_ENDPOINT_URL: http://localhost:4566 | |
| AWS_ALLOW_HTTP: "1" | |
| AZURE_USE_EMULATOR: "1" | |
| AZURE_STORAGE_ALLOW_HTTP: "1" | |
| AZURITE_BLOB_STORAGE_URL: "http://localhost:10000" | |
| AZURE_STORAGE_CONNECTION_STRING: "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://localhost:10000/devstoreaccount1;QueueEndpoint=http://localhost:10001/devstoreaccount1;" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Run sccache-cache | |
| uses: mozilla-actions/[email protected] | |
| - name: Install minimal stable with clippy and rustfmt | |
| uses: actions-rs/toolchain@v1 | |
| with: | |
| profile: default | |
| toolchain: "1.85" | |
| override: true | |
| # Install Java and Hadoop for HDFS integration tests | |
| - uses: actions/setup-java@v4 | |
| with: | |
| distribution: "temurin" | |
| java-version: "17" | |
| - name: Download Hadoop | |
| run: | | |
| wget -q https://dlcdn.apache.org/hadoop/common/hadoop-3.4.0/hadoop-3.4.0.tar.gz | |
| tar -xf hadoop-3.4.0.tar.gz -C $GITHUB_WORKSPACE | |
| echo "$GITHUB_WORKSPACE/hadoop-3.4.0/bin" >> $GITHUB_PATH | |
| - name: Start emulated services | |
| run: docker compose up -d | |
| - name: Run tests with native-tls | |
| run: | | |
| gmake setup-dat | |
| cargo test --no-default-features --features integration_test,s3-native-tls,datafusion | |
| integration_test_hdfs: | |
| name: Integration Tests (HDFS) | |
| runs-on: ubuntu-latest | |
| env: | |
| # https://github.com/rust-lang/cargo/issues/10280 | |
| CARGO_NET_GIT_FETCH_WITH_CLI: "true" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Run sccache-cache | |
| uses: mozilla-actions/[email protected] | |
| - name: Install minimal stable with clippy and rustfmt | |
| uses: actions-rs/toolchain@v1 | |
| with: | |
| profile: default | |
| toolchain: "1.85" | |
| override: true | |
| - name: Install cargo-llvm-cov | |
| uses: taiki-e/install-action@cargo-llvm-cov | |
| # Install Java and Hadoop for HDFS integration tests | |
| - uses: actions/setup-java@v4 | |
| with: | |
| distribution: "temurin" | |
| java-version: "17" | |
| - name: Download Hadoop | |
| run: | | |
| wget -q https://dlcdn.apache.org/hadoop/common/hadoop-3.4.0/hadoop-3.4.0.tar.gz | |
| tar -xf hadoop-3.4.0.tar.gz -C $GITHUB_WORKSPACE | |
| echo "$GITHUB_WORKSPACE/hadoop-3.4.0/bin" >> $GITHUB_PATH | |
| - name: Run tests with rustls (default) | |
| run: | | |
| gmake setup-dat | |
| cargo llvm-cov \ | |
| --features integration_test \ | |
| --package deltalake-hdfs \ | |
| --codecov \ | |
| --output-path codecov.json | |
| - name: Upload coverage to Codecov | |
| uses: codecov/codecov-action@v4 | |
| with: | |
| files: codecov.json | |
| fail_ci_if_error: true | |
| env: | |
| CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} | |
| integration_test_lakefs: | |
| name: Integration Tests (LakeFS v1.48) | |
| runs-on: ubuntu-latest | |
| env: | |
| # https://github.com/rust-lang/cargo/issues/10280 | |
| CARGO_NET_GIT_FETCH_WITH_CLI: "true" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Run sccache-cache | |
| uses: mozilla-actions/[email protected] | |
| - name: Install minimal stable with clippy and rustfmt | |
| uses: actions-rs/toolchain@v1 | |
| with: | |
| profile: default | |
| toolchain: "1.85" | |
| override: true | |
| - name: Install cargo-llvm-cov | |
| uses: taiki-e/install-action@cargo-llvm-cov | |
| - name: Download Lakectl | |
| run: | | |
| wget -q https://github.com/treeverse/lakeFS/releases/download/v1.48.1/lakeFS_1.48.1_Linux_x86_64.tar.gz | |
| tar -xf lakeFS_1.48.1_Linux_x86_64.tar.gz -C $GITHUB_WORKSPACE | |
| echo "$GITHUB_WORKSPACE" >> $GITHUB_PATH | |
| - name: Start emulated services | |
| run: docker compose -f docker-compose-lakefs.yml up -d | |
| - name: Run tests with rustls (default) | |
| run: | | |
| gmake setup-dat | |
| cargo llvm-cov \ | |
| --package deltalake-lakefs \ | |
| --features integration_test_lakefs \ | |
| --codecov \ | |
| --output-path codecov.json | |
| - name: Upload coverage to Codecov | |
| uses: codecov/codecov-action@v4 | |
| with: | |
| files: codecov.json | |
| fail_ci_if_error: true | |
| env: | |
| CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} |