Bugfix/fix latest pypi version check (apache#51039)

* Fix version check from Pypi, requires user agent else raises HTTP 406

* Fix version check from Pypi, requires user agent else raises HTTP 406, also other cases in codebase

Author: jscheffl

dev/breeze/src/airflow_breeze/utils/version_utils.py

@@ -31,7 +31,9 @@ def get_latest_helm_chart_version():
 def get_latest_airflow_version():
     import requests
 
-    response = requests.get("https://pypi.org/pypi/apache-airflow/json")
+    response = requests.get(
+        "https://pypi.org/pypi/apache-airflow/json", headers={"User-Agent": "Python requests"}
+    )
     response.raise_for_status()
     latest_released_version = response.json()["info"]["version"]
     return latest_released_version

dev/validate_version_added_fields_in_config.py

@@ -46,7 +46,7 @@
 
 
 def fetch_pypi_versions() -> list[str]:
-    r = requests.get("https://pypi.org/pypi/apache-airflow/json")
+    r = requests.get("https://pypi.org/pypi/apache-airflow/json", headers={"User-Agent": "Python requests"})
     r.raise_for_status()
     all_version = r.json()["releases"].keys()
     released_versions = [d for d in all_version if not (("rc" in d) or ("b" in d))]

docker-tests/tests/docker_tests/test_examples_of_prod_image_building.py

@@ -42,7 +42,9 @@
 
 @cache
 def get_latest_airflow_image():
-    response = requests.get("https://pypi.org/pypi/apache-airflow/json")
+    response = requests.get(
+        "https://pypi.org/pypi/apache-airflow/json", headers={"User-Agent": "Python requests"}
+    )
     response.raise_for_status()
     latest_released_version = response.json()["info"]["version"]
     return f"apache/airflow:{latest_released_version}"

scripts/ci/airflow_version_check.py

@@ -45,9 +45,10 @@ def check_airflow_version(airflow_version: Version) -> tuple[str, bool]:
     returns: tuple containing the version and a boolean indicating if it's latest.
     """
     latest = False
-    url = "https://pypi.org/pypi/apache-airflow/json"
     try:
-        response = requests.get(url)
+        response = requests.get(
+            "https://pypi.org/pypi/apache-airflow/json", headers={"User-Agent": "Python requests"}
+        )
         response.raise_for_status()
         data = response.json()
         latest_version = Version(data["info"]["version"])

scripts/ci/pre_commit/update_installers_and_pre_commit.py

@@ -55,7 +55,9 @@
 
 
 def get_latest_pypi_version(package_name: str) -> str:
-    response = requests.get(f"https://pypi.org/pypi/{package_name}/json")
+    response = requests.get(
+        f"https://pypi.org/pypi/{package_name}/json", headers={"User-Agent": "Python requests"}
+    )
     response.raise_for_status()  # Ensure we got a successful response
     data = response.json()
     latest_version = data["info"]["version"]  # The version info is under the 'info' key