Fix flaky test "spark read limit" (#750) #3692
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build and Test | |
| on: [push, pull_request, workflow_dispatch] | |
| jobs: | |
| build-and-test-server: | |
| runs-on: ubuntu-24.04 | |
| env: | |
| SPARK_LOCAL_IP: localhost | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AZURE_TEST_ACCOUNT_KEY: ${{ secrets.AZURE_TEST_ACCOUNT_KEY }} | |
| GOOGLE_APPLICATION_CREDENTIALS: /tmp/google_service_account_key.json | |
| GOOGLE_SERVICE_ACCOUNT_KEY: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_KEY }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v2 | |
| - name: Cache Scala, SBT | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.sbt | |
| ~/.ivy2 | |
| ~/.cache/coursier | |
| key: build-and-test-server | |
| - name: Install Java 8 | |
| uses: actions/setup-java@v3 | |
| with: | |
| distribution: 'zulu' | |
| java-version: '8' | |
| - run: ./build/sbt server/test | |
| build-and-test-client: | |
| runs-on: ubuntu-24.04 | |
| env: | |
| SPARK_LOCAL_IP: localhost | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AZURE_TEST_ACCOUNT_KEY: ${{ secrets.AZURE_TEST_ACCOUNT_KEY }} | |
| GOOGLE_APPLICATION_CREDENTIALS: /tmp/google_service_account_key.json | |
| GOOGLE_SERVICE_ACCOUNT_KEY: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_KEY }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v2 | |
| - name: Cache Scala, SBT | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.sbt | |
| ~/.ivy2 | |
| ~/.cache/coursier | |
| key: build-and-test-client | |
| - name: Install Java 17 | |
| uses: actions/setup-java@v3 | |
| with: | |
| distribution: 'zulu' | |
| java-version: '17' | |
| - run: ./build/sbt client/test spark/test | |
| python: | |
| runs-on: ubuntu-24.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ["3.8", "3.9", "3.10", "3.11"] | |
| include: | |
| - pandas-version: 2.0.3 | |
| pyarrow-version: 16.1.0 | |
| env: | |
| PYTHON_VERSION: ${{ matrix.python-version }} | |
| PANDAS_VERSION: ${{ matrix.pandas-version }} | |
| PYARROW_VERSION: ${{ matrix.pyarrow-version }} | |
| SPARK_LOCAL_IP: localhost | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AZURE_TEST_ACCOUNT_KEY: ${{ secrets.AZURE_TEST_ACCOUNT_KEY }} | |
| GOOGLE_APPLICATION_CREDENTIALS: /tmp/google_service_account_key.json | |
| GOOGLE_SERVICE_ACCOUNT_KEY: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_KEY }} | |
| # Github Actions' default miniconda | |
| CONDA_PREFIX: /usr/share/miniconda | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v2 | |
| - name: Cache Scala, SBT | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.sbt | |
| ~/.ivy2 | |
| ~/.cache/coursier | |
| key: build-and-test-python | |
| - name: Install Java 8 | |
| uses: actions/setup-java@v1 | |
| with: | |
| java-version: '8' | |
| - name: Install dependencies | |
| run: | | |
| # See also https://github.com/conda/conda/issues/7980 | |
| source "$CONDA_PREFIX/etc/profile.d/conda.sh" | |
| conda update -q conda | |
| conda create -c conda-forge -q -n test-environment python=$PYTHON_VERSION | |
| conda activate test-environment | |
| conda config --env --add pinned_packages python=$PYTHON_VERSION | |
| conda config --env --add pinned_packages pandas==$PANDAS_VERSION | |
| conda config --env --add pinned_packages pyarrow==$PYARROW_VERSION | |
| conda install -c conda-forge --yes pandas==$PANDAS_VERSION pyarrow==$PYARROW_VERSION | |
| sed -i -e "/pandas/d" -e "/pyarrow/d" python/requirements-dev.txt | |
| conda install -c conda-forge --yes --file python/requirements-dev.txt | |
| conda list | |
| cd python | |
| cd delta-kernel-rust-sharing-wrapper | |
| maturin develop | |
| - name: Build Server | |
| run: ./build/sbt package | |
| - name: Run tests | |
| run: | | |
| # See also https://github.com/conda/conda/issues/7980 | |
| source "$CONDA_PREFIX/etc/profile.d/conda.sh" | |
| conda activate test-environment | |
| ./python/dev/lint-python | |
| ./python/dev/pytest |