Merge branch 'apache:main' into main

Author: yoeo

airflow-core/tests/unit/always/test_project_structure.py

@@ -61,6 +61,7 @@ def test_providers_modules_should_have_tests(self):
         # We should make sure that one goes to 0
         # TODO(potiuk) - check if that test actually tests something
         OVERLOOKED_TESTS = [
+            "providers/alibaba/tests/unit/alibaba/test_version_compat.py",
             "providers/amazon/tests/unit/amazon/aws/auth_manager/datamodels/test_login.py",
             "providers/amazon/tests/unit/amazon/aws/auth_manager/security_manager/test_aws_security_manager_override.py",
             "providers/amazon/tests/unit/amazon/aws/executors/batch/test_batch_executor_config.py",

docs/spelling_wordlist.txt

@@ -1090,6 +1090,8 @@ masterType
 materializations
 Matomo
 matomo
+MaxCompute
+maxcompute
 Maxime
 MaxRuntimeInSeconds
 mb
@@ -1220,6 +1222,7 @@ objectstorage
 observability
 od
 odbc
+odps
 ok
 Okta
 okta

providers/alibaba/provider.yaml

@@ -78,6 +78,9 @@ operators:
   - integration-name: Alibaba Cloud AnalyticDB Spark
     python-modules:
       - airflow.providers.alibaba.cloud.operators.analyticdb_spark
+  - integration-name: Alibaba Cloud MaxCompute
+    python-modules:
+      - airflow.providers.alibaba.cloud.operators.maxcompute
 
 sensors:
   - integration-name: Alibaba Cloud OSS
@@ -94,13 +97,26 @@ hooks:
   - integration-name: Alibaba Cloud AnalyticDB Spark
     python-modules:
       - airflow.providers.alibaba.cloud.hooks.analyticdb_spark
+  - integration-name: Alibaba Cloud
+    python-modules:
+      - airflow.providers.alibaba.cloud.hooks.base_alibaba
+  - integration-name: Alibaba Cloud MaxCompute
+    python-modules:
+      - airflow.providers.alibaba.cloud.hooks.maxcompute
 
 
 connection-types:
   - hook-class-name: airflow.providers.alibaba.cloud.hooks.oss.OSSHook
     connection-type: oss
   - hook-class-name: airflow.providers.alibaba.cloud.hooks.analyticdb_spark.AnalyticDBSparkHook
     connection-type: adb_spark
+  - hook-class-name: airflow.providers.alibaba.cloud.hooks.base_alibaba.AlibabaBaseHook
+    connection-type: alibaba_cloud
+  - hook-class-name: airflow.providers.alibaba.cloud.hooks.maxcompute.MaxComputeHook
+    connection-type: maxcompute
 
 logging:
   - airflow.providers.alibaba.cloud.log.oss_task_handler.OSSTaskHandler
+
+extra-links:
+  - airflow.providers.alibaba.cloud.links.maxcompute.MaxComputeLogViewLink

providers/alibaba/pyproject.toml

@@ -61,6 +61,7 @@ dependencies = [
     "oss2>=2.14.0",
     "alibabacloud_adb20211201>=1.0.0",
     "alibabacloud_tea_openapi>=0.3.7",
+    "pyodps>=0.12.2.2",
 ]
 
 [dependency-groups]

providers/alibaba/src/airflow/providers/alibaba/cloud/exceptions.py

@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+
+class MaxComputeConfigurationException(Exception):
+    """Raised when MaxCompute project or endpoint is not configured properly."""

providers/alibaba/src/airflow/providers/alibaba/cloud/hooks/base_alibaba.py

@@ -0,0 +1,99 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+from typing import Any, NamedTuple
+
+from airflow.hooks.base import BaseHook
+
+
+class AccessKeyCredentials(NamedTuple):
+    """
+    A NamedTuple to store Alibaba Cloud Access Key credentials.
+
+    :param access_key_id: The Access Key ID for Alibaba Cloud authentication.
+    :param access_key_secret: The Access Key Secret for Alibaba Cloud authentication.
+    """
+
+    access_key_id: str
+    access_key_secret: str
+
+
+class AlibabaBaseHook(BaseHook):
+    """
+    A base hook for Alibaba Cloud-related hooks.
+
+    This hook provides a common interface for authenticating using Alibaba Cloud credentials.
+
+    Supports Access Key-based authentication.
+
+    :param alibaba_cloud_conn_id: The connection ID to use when fetching connection info.
+    """
+
+    conn_name_attr = "alibabacloud_conn_id"
+    default_conn_name = "alibabacloud_default"
+    conn_type = "alibaba_cloud"
+    hook_name = "Alibaba Cloud"
+
+    def __init__(
+        self,
+        alibabacloud_conn_id: str = "alibabacloud_default",
+        **kwargs,
+    ) -> None:
+        super().__init__(**kwargs)
+        self.alibaba_cloud_conn_id = alibabacloud_conn_id
+        self.extras: dict = self.get_connection(self.alibaba_cloud_conn_id).extra_dejson
+
+    @classmethod
+    def get_connection_form_widgets(cls) -> dict[str, Any]:
+        """Return connection widgets to add to connection form."""
+        from flask_appbuilder.fieldwidgets import BS3PasswordFieldWidget
+        from flask_babel import lazy_gettext
+        from wtforms import PasswordField
+
+        return {
+            "access_key_id": PasswordField(lazy_gettext("Access Key ID"), widget=BS3PasswordFieldWidget()),
+            "access_key_secret": PasswordField(
+                lazy_gettext("Access Key Secret"), widget=BS3PasswordFieldWidget()
+            ),
+        }
+
+    @classmethod
+    def get_ui_field_behaviour(cls) -> dict[str, Any]:
+        """Return custom field behaviour."""
+        return super().get_ui_field_behaviour()
+
+    def _get_field(self, field_name: str, default: Any = None) -> Any:
+        """Fetch a field from extras, and returns it."""
+        value = self.extras.get(field_name)
+        return value if value is not None else default
+
+    def get_access_key_credential(self) -> AccessKeyCredentials:
+        """
+        Fetch Access Key Credential for authentication.
+
+        :return: AccessKeyCredentials object containing access_key_id and access_key_secret.
+        """
+        access_key_id = self._get_field("access_key_id", None)
+        access_key_secret = self._get_field("access_key_secret", None)
+        if not access_key_id:
+            raise ValueError("No access_key_id is specified.")
+
+        if not access_key_secret:
+            raise ValueError("No access_key_secret is specified.")
+
+        return AccessKeyCredentials(access_key_id, access_key_secret)