devops: sync up with upstream as of 8/28/24

COMPT-3417
merge latest upstream as of 8/28/24

Author: andrewinnocent

.github/workflows/build_python_runtime.yml

@@ -5,11 +5,21 @@ on:
   workflow_dispatch:
     inputs:
       python_version:
-        description: "The Python version to build, specified as X.Y.Z"
+        description: "Python version (eg: 3.12.0)"
         type: string
         required: true
+      stack:
+        description: "Stack(s)"
+        type: choice
+        options:
+          - auto
+          - heroku-20
+          - heroku-22
+          - heroku-24
+        default: auto
+        required: false
       dry_run:
-        description: "Skip deploying to S3 (e.g. for testing)"
+        description: "Skip uploading to S3 (dry run)"
         type: boolean
         default: false
         required: false
@@ -24,36 +34,73 @@ env:
   S3_BUCKET: "heroku-buildpack-python"
 
 # Unfortunately these jobs cannot be easily written as a matrix since `matrix.exclude` does not
-# support expression syntax, and the `inputs` context is not available inside the job `if` key.
+# support expression syntax, and the `matrix` context is not available inside the job `if` key.
 jobs:
-  build-and-upload-heroku-20:
-    runs-on: pub-hk-ubuntu-22.04-xlarge
+  heroku-20:
+    if: inputs.stack == 'heroku-20' || inputs.stack == 'auto'
+    runs-on: pub-hk-ubuntu-24.04-xlarge
     env:
       STACK_VERSION: "20"
     steps:
       - name: Checkout
         uses: actions/checkout@v4
       - name: Build Docker image
-        run: docker build --pull --tag buildenv --build-arg=STACK_VERSION builds/
-      - name: Build and package Python runtime
-        run: docker run --rm --platform="linux/amd64" --volume="${PWD}/upload:/tmp/upload" buildenv ./build_python_runtime.sh "${{ inputs.python_version }}"
+        run: docker build --platform="linux/amd64" --pull --tag buildenv --build-arg=STACK_VERSION builds/
+      - name: Compile and package Python runtime
+        run: docker run --rm --volume="${PWD}/upload:/tmp/upload" buildenv ./build_python_runtime.sh "${{ inputs.python_version }}"
+      - name: Test Python runtime
+        run: |
+          RUN_IMAGE='heroku/heroku:${{ env.STACK_VERSION }}'
+          ARCHIVE_FILENAME='python-${{ inputs.python_version }}-ubuntu-${{ env.STACK_VERSION }}.04-amd64.tar.zst'
+          docker run --rm --volume="${PWD}/upload:/upload:ro" --volume="${PWD}/builds:/builds:ro" "${RUN_IMAGE}" /builds/test_python_runtime.sh "/upload/${ARCHIVE_FILENAME}"
       - name: Upload Python runtime archive to S3
         if: (!inputs.dry_run)
         run: aws s3 sync ./upload "s3://${S3_BUCKET}"
 
-  build-and-upload-heroku-22:
-    # We only support Python 3.9+ on Heroku-22.
-    if: (!startsWith(inputs.python_version,'3.8.'))
-    runs-on: pub-hk-ubuntu-22.04-xlarge
+  heroku-22:
+    # On Heroku-22 we only support Python 3.9+.
+    if: inputs.stack == 'heroku-22' || (inputs.stack == 'auto' && !startsWith(inputs.python_version,'3.8.'))
+    runs-on: pub-hk-ubuntu-24.04-xlarge
     env:
       STACK_VERSION: "22"
     steps:
       - name: Checkout
         uses: actions/checkout@v4
       - name: Build Docker image
-        run: docker build --pull --tag buildenv --build-arg=STACK_VERSION builds/
-      - name: Build and package Python runtime
-        run: docker run --rm --platform="linux/amd64" --volume="${PWD}/upload:/tmp/upload" buildenv ./build_python_runtime.sh "${{ inputs.python_version }}"
+        run: docker build --platform="linux/amd64" --pull --tag buildenv --build-arg=STACK_VERSION builds/
+      - name: Compile and package Python runtime
+        run: docker run --rm --volume="${PWD}/upload:/tmp/upload" buildenv ./build_python_runtime.sh "${{ inputs.python_version }}"
+      - name: Test Python runtime
+        run: |
+          RUN_IMAGE='heroku/heroku:${{ env.STACK_VERSION }}'
+          ARCHIVE_FILENAME='python-${{ inputs.python_version }}-ubuntu-${{ env.STACK_VERSION }}.04-amd64.tar.zst'
+          docker run --rm --volume="${PWD}/upload:/upload:ro" --volume="${PWD}/builds:/builds:ro" "${RUN_IMAGE}" /builds/test_python_runtime.sh "/upload/${ARCHIVE_FILENAME}"
+      - name: Upload Python runtime archive to S3
+        if: (!inputs.dry_run)
+        run: aws s3 sync ./upload "s3://${S3_BUCKET}"
+
+  heroku-24:
+    # On Heroku-24 we only support Python 3.10+.
+    if: inputs.stack == 'heroku-24' || (inputs.stack == 'auto' && !startsWith(inputs.python_version,'3.8.') && !startsWith(inputs.python_version,'3.9.'))
+    strategy:
+      fail-fast: false
+      matrix:
+        arch: ["amd64", "arm64"]
+    runs-on: ${{ matrix.arch == 'arm64' && 'pub-hk-ubuntu-24.04-arm-xlarge' || 'pub-hk-ubuntu-24.04-xlarge' }}
+    env:
+      STACK_VERSION: "24"
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Build Docker image
+        run: docker build --platform="linux/${{ matrix.arch }}" --pull --tag buildenv --build-arg=STACK_VERSION builds/
+      - name: Compile and package Python runtime
+        run: docker run --rm --volume="${PWD}/upload:/tmp/upload" buildenv ./build_python_runtime.sh "${{ inputs.python_version }}"
+      - name: Test Python runtime
+        run: |
+          RUN_IMAGE='heroku/heroku:${{ env.STACK_VERSION }}'
+          ARCHIVE_FILENAME='python-${{ inputs.python_version }}-ubuntu-${{ env.STACK_VERSION }}.04-${{ matrix.arch }}.tar.zst'
+          docker run --rm --volume="${PWD}/upload:/upload:ro" --volume="${PWD}/builds:/builds:ro" "${RUN_IMAGE}" /builds/test_python_runtime.sh "/upload/${ARCHIVE_FILENAME}"
       - name: Upload Python runtime archive to S3
         if: (!inputs.dry_run)
         run: aws s3 sync ./upload "s3://${S3_BUCKET}"

.github/workflows/check_changelog.yml

@@ -9,7 +9,7 @@ permissions:
 
 jobs:
   check-changelog:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     if: (!contains(github.event.pull_request.labels.*.name, 'skip changelog'))
     steps:
       - name: Checkout

.github/workflows/ci.yml

@@ -12,44 +12,62 @@ permissions:
 
 jobs:
   lint:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout
         uses: actions/checkout@v4
       - name: Install Ruby and dependencies
         uses: ruby/setup-ruby@v1
         with:
           bundler-cache: true
-          ruby-version: "3.1"
+          ruby-version: "3.3"
       - name: Run ShellCheck
         run: make lint-scripts
       - name: Run Rubocop
         run: bundle exec rubocop
 
   integration-test:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     strategy:
       fail-fast: false
       matrix:
-        stack: ["heroku-20", "heroku-22"]
+        stack: ["heroku-20", "heroku-22", "heroku-24"]
     env:
       HATCHET_APP_LIMIT: 200
       HATCHET_DEFAULT_STACK: ${{ matrix.stack }}
-      HATCHET_RETRIES: 2
+      HATCHET_EXPENSIVE_MODE: 1
       HEROKU_API_KEY: ${{ secrets.HEROKU_API_KEY }}
       HEROKU_API_USER: ${{ secrets.HEROKU_API_USER }}
       HEROKU_DISABLE_AUTOUPDATE: 1
       PARALLEL_SPLIT_TEST_PROCESSES: 60
+      RSPEC_RETRY_RETRY_COUNT: 3
     steps:
       - name: Checkout
         uses: actions/checkout@v4
       - name: Install Ruby and dependencies
         uses: ruby/setup-ruby@v1
         with:
           bundler-cache: true
-          ruby-version: "3.1"
+          ruby-version: "3.3"
       - name: Hatchet setup
         run: bundle exec hatchet ci:setup
       - name: Run Hatchet integration tests
         # parallel_split_test runs rspec in parallel, with concurrency equal to PARALLEL_SPLIT_TEST_PROCESSES.
         run: bundle exec parallel_split_test spec/hatchet/
+
+  container-test:
+    runs-on: ubuntu-24.04
+    defaults:
+      run:
+        # Work around lack of TTY causing errors when using `docker run -it`:
+        # https://github.com/actions/runner/issues/241
+        shell: 'script -q -e -c "bash -eo pipefail {0}"'
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      # These test both the local development `make run` workflow and that `bin/report` completes successfully
+      # for both passing and failing builds (since `bin/report` can't easily be tested via Hatchet tests).
+      - name: Run buildpack using default app fixture
+        run: make run
+      - name: Run buildpack using an app fixture that's expected to fail
+        run: make run FIXTURE=spec/fixtures/python_version_invalid/

.github/workflows/hatchet_app_cleaner.yml

@@ -12,7 +12,7 @@ permissions:
 
 jobs:
   hatchet-app-cleaner:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     env:
       HEROKU_API_KEY: ${{ secrets.HEROKU_API_KEY }}
       HEROKU_API_USER: ${{ secrets.HEROKU_API_USER }}
@@ -24,7 +24,7 @@ jobs:
         uses: ruby/setup-ruby@v1
         with:
           bundler-cache: true
-          ruby-version: "3.1"
+          ruby-version: "3.3"
       - name: Run Hatchet destroy
         # Only apps older than 10 minutes are destroyed, to ensure that any
         # in progress CI runs are not interrupted.

.gitignore

@@ -1,5 +1,10 @@
 __pycache__/
 .hatchet/repos/
+.venv/
+.idea/
+# The setup-ruby GitHub Action creates this directory when caching is enabled, so we ignore
+# it here so it does not show up in the output of `git ls-files` for `make lint-scripts`.
+vendor/bundle/
 .DS_Store
 .rspec_status
 .local_temp

.rubocop.yml

@@ -26,3 +26,9 @@ RSpec/Focus:
 
 RSpec/MultipleExpectations:
   Enabled: false
+
+Style/TrailingCommaInArrayLiteral:
+  EnforcedStyleForMultiline: consistent_comma
+
+Style/TrailingCommaInHashLiteral:
+  EnforcedStyleForMultiline: consistent_comma

.shellcheckrc

@@ -0,0 +1,15 @@
+# Enable all checks, including the optional ones that are off by default.
+enable=all
+
+# TODO: Triage and potentially enable more of these optional checks.
+
+# SC2154 (warning): var is referenced but not assigned.
+disable=SC2154
+# SC2250 (style): Prefer putting braces around variable references even when not strictly required.
+disable=SC2250
+# SC2310 (info): This function is invoked in an 'if' condition so set -e will be disabled.
+disable=SC2310
+# SC2311 (info): Bash implicitly disabled set -e for this function invocation because it's inside a command substitution.
+disable=SC2311
+# SC2312 (info): Consider invoking this command separately to avoid masking its return value
+disable=SC2312

CHANGELOG.md

@@ -3,6 +3,73 @@
 ## [Unreleased]
 
 
+## [v255] - 2024-08-07
+
+- Added support for Python 3.12.5. ([#1622](https://github.com/heroku/heroku-buildpack-python/pull/1622))
+- Changed the default Python version for new apps from 3.12.4 to 3.12.5. ([#1622](https://github.com/heroku/heroku-buildpack-python/pull/1622))
+
+## [v254] - 2024-07-16
+
+- Updated setuptools from 69.2.0 to 70.3.0. ([#1614](https://github.com/heroku/heroku-buildpack-python/pull/1614))
+- Updated pipenv from 2023.12.1 to 2024.0.1. ([#1601](https://github.com/heroku/heroku-buildpack-python/pull/1601))
+
+## [v253] - 2024-07-01
+
+- Improved the error messages shown when an app is missing the necessary Python package manager files. ([#1608](https://github.com/heroku/heroku-buildpack-python/pull/1608))
+
+## [v252] - 2024-06-17
+
+- Removed export of `Pipfile.lock` to `requirements.txt` during the build. ([#1593](https://github.com/heroku/heroku-buildpack-python/pull/1593))
+- Removed internal `pipenv-to-pip` script that was unintentionally exposed onto `PATH`. ([#1593](https://github.com/heroku/heroku-buildpack-python/pull/1593))
+- Stopped exposing the internal `BIN_DIR`, `BPLOG_PREFIX`, `EXPORT_PATH` and `PROFILE_PATH` environment variables to `bin/{pre,post}_compile` and other subprocesses. ([#1595](https://github.com/heroku/heroku-buildpack-python/pull/1595) and [#1597](https://github.com/heroku/heroku-buildpack-python/pull/1597))
+- Implemented the `bin/report` build report API and removed log based metrics. ([#1597](https://github.com/heroku/heroku-buildpack-python/pull/1597))
+
+## [v251] - 2024-06-07
+
+- Added support for Python 3.12.4. ([#1591](https://github.com/heroku/heroku-buildpack-python/pull/1591))
+- Changed the default Python version for new apps from 3.12.3 to 3.12.4. ([#1591](https://github.com/heroku/heroku-buildpack-python/pull/1591))
+
+## [v250] - 2024-04-26
+
+- Added support for Heroku-24. ([#1575](https://github.com/heroku/heroku-buildpack-python/pull/1575))
+
+## [v249] - 2024-04-18
+
+- Improved the error message shown for EOL Python versions when using a stack for which those versions were never built. ([#1570](https://github.com/heroku/heroku-buildpack-python/pull/1570))
+- Fixed the "Python security update is available" warning being shown when the requested version is newer than the latest version known to the buildpack. ([#1569](https://github.com/heroku/heroku-buildpack-python/pull/1569))
+- Fixed glibc warnings seen when downgrading the stack version. ([#1568](https://github.com/heroku/heroku-buildpack-python/pull/1568))
+- Changed compression format and S3 URL for Python runtime archives. ([#1567](https://github.com/heroku/heroku-buildpack-python/pull/1567))
+- Adjusted compiler options used to build Python for improved parity with the Docker Hub Python images. ([#1566](https://github.com/heroku/heroku-buildpack-python/pull/1566))
+- Excluded `LD_LIBRARY_PATH` and `PYTHONHOME` app config vars when invoking subprocesses during the build. ([#1565](https://github.com/heroku/heroku-buildpack-python/pull/1565))
+
+## [v248] - 2024-04-09
+
+- Added support for Python 3.12.3. ([#1560](https://github.com/heroku/heroku-buildpack-python/pull/1560))
+- Changed the default Python version for new apps from 3.12.2 to 3.12.3. ([#1560](https://github.com/heroku/heroku-buildpack-python/pull/1560))
+
+## [v247] - 2024-04-08
+
+- Added support for Python 3.11.9. ([#1558](https://github.com/heroku/heroku-buildpack-python/pull/1558))
+
+## [v246] - 2024-03-25
+
+- Updated pip from 23.3.2 to 24.0. ([#1541](https://github.com/heroku/heroku-buildpack-python/pull/1541))
+- Updated setuptools from 68.2.2 to 69.2.0. ([#1553](https://github.com/heroku/heroku-buildpack-python/pull/1553))
+- Updated wheel from 0.42.0 to 0.43.0. ([#1550](https://github.com/heroku/heroku-buildpack-python/pull/1550))
+- Updated pipenv from 2023.11.15 to 2023.12.1. ([#1540](https://github.com/heroku/heroku-buildpack-python/pull/1540))
+
+## [v245] - 2024-03-21
+
+- Added support for Python 3.8.19, 3.9.19 and 3.10.14. ([#1551](https://github.com/heroku/heroku-buildpack-python/pull/1551))
+
+## [v244] - 2024-03-13
+
+- Improved the automatic `WEB_CONCURRENCY` feature: ([#1547](https://github.com/heroku/heroku-buildpack-python/pull/1547))
+  - Switched to a dynamic calculation based on dyno CPU cores and memory instead of a hardcoded mapping.
+  - Decreased default concurrency on `performance-m` / `private-m` / `shield-m` dynos from `8` to `5`.
+  - Increased default concurrency on `performance-l` / `private-l` / `shield-l` dynos from `11` to `17`.
+  - Added logging of memory/CPU/concurrency information to the app logs (for web dynos only).
+
 ## [v243] - 2024-02-07
 
 - Added support for Python 3.11.8 and 3.12.2. ([#1538](https://github.com/heroku/heroku-buildpack-python/pull/1538)).
@@ -958,7 +1025,19 @@ Default Python is now latest 2.7.10. Updated Pip and Distribute.
 - Setuptools updated to v16.0
 - Pip updated to v7.0.1
 
-[unreleased]: https://github.com/heroku/heroku-buildpack-python/compare/v243...main
+[unreleased]: https://github.com/heroku/heroku-buildpack-python/compare/v255...main
+[v255]: https://github.com/heroku/heroku-buildpack-python/compare/v254...v255
+[v254]: https://github.com/heroku/heroku-buildpack-python/compare/v253...v254
+[v253]: https://github.com/heroku/heroku-buildpack-python/compare/v252...v253
+[v252]: https://github.com/heroku/heroku-buildpack-python/compare/v251...v252
+[v251]: https://github.com/heroku/heroku-buildpack-python/compare/v250...v251
+[v250]: https://github.com/heroku/heroku-buildpack-python/compare/v249...v250
+[v249]: https://github.com/heroku/heroku-buildpack-python/compare/v248...v249
+[v248]: https://github.com/heroku/heroku-buildpack-python/compare/v247...v248
+[v247]: https://github.com/heroku/heroku-buildpack-python/compare/v246...v247
+[v246]: https://github.com/heroku/heroku-buildpack-python/compare/v245...v246
+[v245]: https://github.com/heroku/heroku-buildpack-python/compare/v244...v245
+[v244]: https://github.com/heroku/heroku-buildpack-python/compare/v243...v244
 [v243]: https://github.com/heroku/heroku-buildpack-python/compare/v242...v243
 [v242]: https://github.com/heroku/heroku-buildpack-python/compare/v241...v242
 [v241]: https://github.com/heroku/heroku-buildpack-python/compare/v240...v241

Gemfile

@@ -2,13 +2,14 @@
 
 source 'https://rubygems.org'
 
-ruby '>= 3.1', '< 3.3'
+ruby '>= 3.1', '< 3.4'
 
 group :test, :development do
   gem 'heroku_hatchet'
   gem 'parallel_split_test'
   gem 'rspec-core'
   gem 'rspec-expectations'
+  gem 'rspec-retry'
   gem 'rubocop', require: false
   gem 'rubocop-rspec', require: false
 end