Mouse keyboard adaptation for HarmonyOS Next (#18656)

wanghui187 · web-flow · commit a1243b4251ac · 2025-04-29T17:48:07.000+08:00
diff --git a/native/cocos/platform/openharmony/OpenHarmonyPlatform.cpp b/native/cocos/platform/openharmony/OpenHarmonyPlatform.cpp
@@ -167,6 +167,139 @@ void onSurfaceShowCB(OH_NativeXComponent* component, void* window) {
     sendMsgToWorker(cc::MessageType::WM_XCOMPONENT_SURFACE_SHOW, component, window);
 }
 
+int ohKeyCodeToCocosKeyCode(OH_NativeXComponent_KeyCode ohKeyCode){
+    static const int keyZeroInCocos = 48;
+    static const int keyF1InCocos = 112;
+    static const int keyAInCocos = 65;
+    static std::unordered_map<OH_NativeXComponent_KeyCode, cc::KeyCode> keyCodeMap = {
+        {KEY_ESCAPE, cc::KeyCode::ESCAPE},
+        {KEY_GRAVE, cc::KeyCode::BACKQUOTE},
+        {KEY_MINUS, cc::KeyCode::MINUS},
+        {KEY_EQUALS, cc::KeyCode::EQUAL},
+        {KEY_DEL, cc::KeyCode::BACKSPACE},
+        {KEY_TAB, cc::KeyCode::TAB},
+        {KEY_LEFT_BRACKET, cc::KeyCode::BRACKET_LEFT},
+        {KEY_RIGHT_BRACKET, cc::KeyCode::BRACKET_RIGHT},
+        {KEY_BACKSLASH, cc::KeyCode::BACKSLASH},
+        {KEY_CAPS_LOCK, cc::KeyCode::CAPS_LOCK},
+        {KEY_SEMICOLON, cc::KeyCode::SEMICOLON},
+        {KEY_APOSTROPHE, cc::KeyCode::QUOTE},
+        {KEY_ENTER, cc::KeyCode::ENTER},
+        {KEY_SHIFT_LEFT, cc::KeyCode::SHIFT_LEFT},
+        {KEY_COMMA, cc::KeyCode::COMMA},
+        {KEY_PERIOD, cc::KeyCode::PERIOD},
+        {KEY_SLASH, cc::KeyCode::SLASH},
+        {KEY_SHIFT_RIGHT, cc::KeyCode::SHIFT_RIGHT},
+        {KEY_CTRL_LEFT, cc::KeyCode::CONTROL_LEFT},
+        {KEY_ALT_LEFT, cc::KeyCode::ALT_LEFT},
+        {KEY_SPACE, cc::KeyCode::SPACE},
+        {KEY_ALT_RIGHT, cc::KeyCode::ALT_RIGHT},
+        {KEY_CTRL_RIGHT, cc::KeyCode::CONTROL_RIGHT},
+        {KEY_DPAD_LEFT, cc::KeyCode::ARROW_LEFT},
+        {KEY_DPAD_RIGHT, cc::KeyCode::ARROW_RIGHT},
+        {KEY_DPAD_DOWN, cc::KeyCode::ARROW_DOWN},
+        {KEY_DPAD_UP, cc::KeyCode::ARROW_UP},
+        {KEY_INSERT, cc::KeyCode::INSERT},
+    };
+    if(keyCodeMap.find(ohKeyCode) != keyCodeMap.end()){
+        return int(keyCodeMap[ohKeyCode]);
+    }
+    if(ohKeyCode >= KEY_0 && ohKeyCode <= KEY_9){
+        return keyZeroInCocos + ohKeyCode - KEY_0;
+    }
+    if(ohKeyCode >= KEY_A && ohKeyCode <= KEY_Z){
+        return keyAInCocos + ohKeyCode - KEY_A;
+    }  
+    if(ohKeyCode >= KEY_F1 && ohKeyCode <= KEY_F12){
+        return keyF1InCocos + ohKeyCode - KEY_F1;
+    }  
+    return ohKeyCode;
+}
+
+void dispatchKeyEventCB(OH_NativeXComponent* component, void* window) {
+    OH_NativeXComponent_KeyEvent* keyEvent;
+    if (OH_NativeXComponent_GetKeyEvent(component, &keyEvent) >= 0) {
+        static const int keyCodeUnknownInOH = -1;
+        static const int keyActionUnknownInOH = -1;
+        OH_NativeXComponent_KeyAction action;
+        OH_NativeXComponent_GetKeyEventAction(keyEvent, &action);
+        OH_NativeXComponent_KeyCode code;
+        OH_NativeXComponent_GetKeyEventCode(keyEvent, &code);
+        if (code == keyCodeUnknownInOH || action == keyActionUnknownInOH) {
+            CC_LOG_ERROR("unknown code and action don't callback");
+            return;
+        }
+        cc::KeyboardEvent* ev = new cc::KeyboardEvent;
+        ev->windowId = cc::ISystemWindow::mainWindowId;
+        ev->action = 0 == action ? cc::KeyboardEvent::Action::PRESS : cc::KeyboardEvent::Action::RELEASE;
+
+        ev->key = ohKeyCodeToCocosKeyCode(code);
+        sendMsgToWorker(cc::MessageType::WM_XCOMPONENT_KEY_EVENT, reinterpret_cast<void*>(ev), window);
+    } else {
+        CC_LOG_ERROR("OpenHarmonyPlatform::getKeyEventError");
+    }
+}
+
+
+void dispatchMouseEventCB(OH_NativeXComponent* component, void* window) {
+    OH_NativeXComponent_MouseEvent mouseEvent;
+    int32_t ret = OH_NativeXComponent_GetMouseEvent(component, window, &mouseEvent);
+    if (ret == OH_NATIVEXCOMPONENT_RESULT_SUCCESS) {
+        if (mouseEvent.action == OH_NativeXComponent_MouseEventAction::OH_NATIVEXCOMPONENT_MOUSE_NONE)
+            return;
+        cc::MouseEvent* ev = new cc::MouseEvent;
+        ev->windowId = cc::ISystemWindow::mainWindowId;
+        ev->x = mouseEvent.x;
+        ev->y = mouseEvent.y;
+        switch (mouseEvent.action) {
+            case OH_NativeXComponent_MouseEventAction::OH_NATIVEXCOMPONENT_MOUSE_PRESS:
+                ev->type = cc::MouseEvent::Type::DOWN;
+                break;
+            case OH_NativeXComponent_MouseEventAction::OH_NATIVEXCOMPONENT_MOUSE_RELEASE:
+                ev->type = cc::MouseEvent::Type::UP;
+                break;
+            case OH_NativeXComponent_MouseEventAction::OH_NATIVEXCOMPONENT_MOUSE_MOVE:
+                ev->type = cc::MouseEvent::Type::MOVE;
+                break;          
+            default:
+                ev->type = cc::MouseEvent::Type::UNKNOWN;
+                break;
+        }
+        switch (mouseEvent.button) {
+            case OH_NativeXComponent_MouseEventButton::OH_NATIVEXCOMPONENT_LEFT_BUTTON:
+                ev->button = 0;
+                break;
+            case OH_NativeXComponent_MouseEventButton::OH_NATIVEXCOMPONENT_RIGHT_BUTTON:
+                ev->button = 2;
+                break;
+            case OH_NativeXComponent_MouseEventButton::OH_NATIVEXCOMPONENT_MIDDLE_BUTTON:
+                ev->button = 1;
+                break;
+            case OH_NativeXComponent_MouseEventButton::OH_NATIVEXCOMPONENT_BACK_BUTTON:
+                ev->button = 3;
+                break;
+            case OH_NativeXComponent_MouseEventButton::OH_NATIVEXCOMPONENT_FORWARD_BUTTON:
+                ev->button = 4;
+                break;
+            case OH_NativeXComponent_MouseEventButton::OH_NATIVEXCOMPONENT_NONE_BUTTON:
+                ev->button = -1;
+                break;
+        }
+        if(mouseEvent.action == 1 && mouseEvent.button == 1) {
+            cc::OpenHarmonyPlatform::getInstance()->isMouseLeftActive = true;
+        }
+        if(mouseEvent.action == 2 && mouseEvent.button == 1) {
+            cc::OpenHarmonyPlatform::getInstance()->isMouseLeftActive = false;
+        }
+        sendMsgToWorker(cc::MessageType::WM_XCOMPONENT_MOUSE_EVENT, reinterpret_cast<void*>(ev), window);
+    } else {
+        CC_LOG_ERROR("OpenHarmonyPlatform::getMouseEventError");
+    }
+}
+
+void dispatchHoverEventCB(OH_NativeXComponent* component, bool isHover) {
+    // OpenharmonyPlatform::DispatchHoverEventCB
+}
 
 cc::TouchEvent::Type touchTypeTransform(OH_NativeXComponent_TouchEventType touchType) {
     if (touchType == OH_NATIVEXCOMPONENT_DOWN) {
@@ -267,6 +400,12 @@ void OpenHarmonyPlatform::setNativeXComponent(OH_NativeXComponent* component) {
     OH_NativeXComponent_RegisterCallback(_component, &_callback);
     OH_NativeXComponent_RegisterSurfaceHideCallback(_component, onSurfaceHideCB);
     OH_NativeXComponent_RegisterSurfaceShowCallback(_component, onSurfaceShowCB);
+    // register KeyEvent                                     
+    OH_NativeXComponent_RegisterKeyEventCallback(_component, dispatchKeyEventCB);
+    // register mouseEvent
+    _mouseCallback.DispatchMouseEvent = dispatchMouseEventCB;
+    _mouseCallback.DispatchHoverEvent = dispatchHoverEventCB;
+    OH_NativeXComponent_RegisterMouseEventCallback(_component, &_mouseCallback);
 }
 
 void OpenHarmonyPlatform::enqueue(const WorkerMessageData& msg) {
@@ -338,6 +477,18 @@ void OpenHarmonyPlatform::onMessageCallback(const uv_async_t* /* req */) {
                 events::Touch::broadcast(*ev);
                 delete ev;
                 ev = nullptr;
+            } else if (msgData.type == MessageType::WM_XCOMPONENT_KEY_EVENT) {
+                KeyboardEvent* ev = reinterpret_cast<KeyboardEvent*>(msgData.data);
+                CC_ASSERT(ev != nullptr);
+                events::Keyboard::broadcast(*ev);
+                delete ev;
+                ev = nullptr;
+            } else if (msgData.type == MessageType::WM_XCOMPONENT_MOUSE_EVENT || msgData.type == MessageType::WM_XCOMPONENT_MOUSE_WHEEL_EVENT ) {
+                MouseEvent* ev = reinterpret_cast<MouseEvent*>(msgData.data);
+                CC_ASSERT(ev != nullptr);
+                events::Mouse::broadcast(*ev);
+                delete ev;
+                ev = nullptr;
             } else if (msgData.type == MessageType::WM_XCOMPONENT_SURFACE_CREATED) {
                 CC_LOG_INFO("onMessageCallback WM_XCOMPONENT_SURFACE_CREATED ...");
                 OH_NativeXComponent* nativexcomponet = reinterpret_cast<OH_NativeXComponent*>(msgData.data);
@@ -483,6 +634,23 @@ void OpenHarmonyPlatform::onSurfaceShow(void* window) {
     events::WindowRecreated::broadcast(ISystemWindow::mainWindowId);
 }
 
+void OpenHarmonyPlatform::dispatchMouseWheelCB(std::string eventType, float offsetY) {
+    if(isMouseLeftActive) {
+        return;
+    }
+    if(eventType == "actionUpdate") {
+        float moveScrollY = offsetY - scrollDistance;
+        scrollDistance = offsetY;
+        cc::MouseEvent* ev = new cc::MouseEvent;
+        ev->windowId = cc::ISystemWindow::mainWindowId;
+        ev->type = MouseEvent::Type::WHEEL;
+        ev->x = 0;
+        ev->y = moveScrollY;
+        sendMsgToWorker(MessageType::WM_XCOMPONENT_MOUSE_WHEEL_EVENT, reinterpret_cast<void*>(ev), nullptr);
+    } else {
+        scrollDistance = 0;
+    }
+}
 
 ISystemWindow* OpenHarmonyPlatform::createNativeWindow(uint32_t windowId, void* externalHandle) {
     SystemWindow* window = ccnew SystemWindow(windowId, externalHandle);
diff --git a/native/cocos/platform/openharmony/OpenHarmonyPlatform.h b/native/cocos/platform/openharmony/OpenHarmonyPlatform.h
@@ -73,18 +73,22 @@ class OpenHarmonyPlatform : public UniversalPlatform {
     void onSurfaceDestroyed(OH_NativeXComponent* component, void* window);
     void onSurfaceHide();
     void onSurfaceShow(void* window);
+    void dispatchMouseWheelCB(std::string eventType, float offsetY);
 
     static void onMessageCallback(const uv_async_t* req);
     static void timerCb(uv_timer_t* handle);
 
     OH_NativeXComponent* _component{nullptr};
     OH_NativeXComponent_Callback _callback;
-    uv_timer_t _timerHandle;
+    OH_NativeXComponent_MouseEvent_Callback _mouseCallback{nullptr};
+    uv_timer_t _timerHandle{nullptr};
     uv_loop_t* _workerLoop{nullptr};
     uv_async_t _messageSignal{};
     bool _timerInited{false};
     WorkerMessageQueue _messageQueue;
     //game started
-    bool g_started = false;
+    bool g_started{false};
+    bool isMouseLeftActive{false};
+    float scrollDistance{0};
 };
 } // namespace cc
diff --git a/native/cocos/platform/openharmony/WorkerMessageQueue.h b/native/cocos/platform/openharmony/WorkerMessageQueue.h
@@ -36,6 +36,9 @@ namespace cc {
 enum class MessageType {
     WM_XCOMPONENT_SURFACE_CREATED = 0,
     WM_XCOMPONENT_TOUCH_EVENT,
+    WM_XCOMPONENT_KEY_EVENT,
+    WM_XCOMPONENT_MOUSE_EVENT,
+    WM_XCOMPONENT_MOUSE_WHEEL_EVENT,
     WM_XCOMPONENT_SURFACE_CHANGED,
     WM_XCOMPONENT_SURFACE_HIDE,
     WM_XCOMPONENT_SURFACE_SHOW,
diff --git a/native/cocos/platform/openharmony/napi/NapiHelper.cpp b/native/cocos/platform/openharmony/napi/NapiHelper.cpp
@@ -54,7 +54,8 @@ enum ContextType {
     WEBVIEW_UTILS,
     DISPLAY_UTILS,
     UV_ASYNC_SEND,
-    VIDEO_UTILS
+    VIDEO_UTILS,
+    MOUSE_WHEEL_NAPI
 };
 
 #define KEYCODE_BACK_OH 6
@@ -325,6 +326,16 @@ static void napiOnVideoEvent(const Napi::CallbackInfo &info) {
     }
 }
 
+static void napiOnMouseWheel(const Napi::CallbackInfo &info) {
+    if(info.Length() != 2) {
+        Napi::Error::New(info.Env(), "napiOnMouseWheel , 1 argument expected").ThrowAsJavaScriptException();
+        return;
+    }
+    std::string eventType = info[0].As<Napi::String>().ToString();
+    float offsetY = info[1].As<Napi::Number>().FloatValue();
+    OpenHarmonyPlatform::getInstance()->dispatchMouseWheelCB(eventType, offsetY);
+}
+
 // NAPI Interface
 static Napi::Value getContext(const Napi::CallbackInfo &info) {
     Napi::Env env = info.Env();
@@ -400,6 +411,9 @@ static Napi::Value getContext(const Napi::CallbackInfo &info) {
         case VIDEO_UTILS: {
             exports["onVideoEvent"] = Napi::Function::New(env, napiOnVideoEvent);
         } break;
+        case MOUSE_WHEEL_NAPI: {
+            exports["onMouseWheel"] = Napi::Function::New(env, napiOnMouseWheel);
+        } break;
         default:
             CC_LOG_ERROR("unknown type");
     }
diff --git a/pal/input/native/mouse-input.ts b/pal/input/native/mouse-input.ts
@@ -23,10 +23,12 @@
 */
 
 import { screenAdapter } from 'pal/screen-adapter';
+import { systemInfo } from 'pal/system-info';
 import { EventMouse } from '../../../cocos/input/types';
 import { EventTarget } from '../../../cocos/core/event';
 import { Vec2 } from '../../../cocos/core/math';
 import { InputEventType } from '../../../cocos/input/types/event-enum';
+import { OS } from '../../system-info/enum-type';
 
 export type MouseCallback = (res: EventMouse) => void;
 
@@ -204,8 +206,14 @@ export class MouseInputSource {
         eventMouse.movementX = location.x - this._preMousePos.x;
         eventMouse.movementY = this._preMousePos.y - location.y;
 
-        const matchStandardFactor = 120;
-        eventMouse.setScrollData(mouseEvent.wheelDeltaX * matchStandardFactor, mouseEvent.wheelDeltaY * matchStandardFactor);
+        let matchStandardFactor = 0;
+        if (systemInfo.os === OS.OPENHARMONY) {
+            matchStandardFactor = 5;
+            eventMouse.setScrollData(mouseEvent.wheelDeltaX, mouseEvent.wheelDeltaY * matchStandardFactor);
+        } else {
+            matchStandardFactor = 120;
+            eventMouse.setScrollData(mouseEvent.wheelDeltaX * matchStandardFactor, mouseEvent.wheelDeltaY * matchStandardFactor);
+        }
         // update previous mouse position.
         this._preMousePos.set(location.x, location.y);
         this._eventTarget.emit(eventType, eventMouse);
diff --git a/pal/system-info/native/system-info.ts b/pal/system-info/native/system-info.ts
@@ -115,6 +115,7 @@ class SystemInfo extends EventTarget {
         this.isXR = (typeof xr !== 'undefined' && typeof xr.XrEntry !== 'undefined');
 
         const isHPE: boolean = typeof __supportHPE === 'function' ? __supportHPE() : false;
+        const isHarmonyOSNext = this.platform === Platform.OPENHARMONY;
 
         this._featureMap = {
             [Feature.WEBP]: true,
@@ -126,7 +127,7 @@ class SystemInfo extends EventTarget {
 
             [Feature.INPUT_TOUCH]: this.isMobile,
             [Feature.EVENT_KEYBOARD]: true,
-            [Feature.EVENT_MOUSE]: isHPE || !this.isMobile,
+            [Feature.EVENT_MOUSE]: isHPE || !this.isMobile || isHarmonyOSNext,
             [Feature.EVENT_TOUCH]: true,
             [Feature.EVENT_ACCELEROMETER]: this.isMobile,
             [Feature.EVENT_GAMEPAD]: true,
diff --git a/templates/harmonyos-next/entry/src/main/cpp/types/libcocos/index.d.ts b/templates/harmonyos-next/entry/src/main/cpp/types/libcocos/index.d.ts
@@ -25,7 +25,8 @@ export interface context {
   resourceManagerInit: (resourceManager: resourceManager.ResourceManager) => void;
   writablePathInit: (cacheDir: string) => void;
   onVideoEvent: (param: string) => void;
-  registerFunction: (name:string ,fun:Function) => void;
+  onMouseWheel: (type: string, offsetY: number) => void;
+  registerFunction: (name: string, fun: Function) => void;
 }
 
 export const getContext: (type: ContextType) => context;
diff --git a/templates/harmonyos-next/entry/src/main/ets/common/Constants.ts b/templates/harmonyos-next/entry/src/main/ets/common/Constants.ts
@@ -34,7 +34,8 @@ export enum ContextType {
     WEBVIEW_UTILS,
     DISPLAY_UTILS,
     UV_ASYNC_SEND,
-    VIDEO_UTILS
+    VIDEO_UTILS,
+    MOUSE_WHEEL_NAPI
 }
 
 export class Constants {
diff --git a/templates/harmonyos-next/entry/src/main/ets/pages/index.ets b/templates/harmonyos-next/entry/src/main/ets/pages/index.ets
@@ -36,7 +36,7 @@ import { BusinessError } from '@kit.BasicServicesKit';
 
 const nativePageLifecycle :context = nativerender.getContext(ContextType.JSPAGE_LIFECYCLE);
 const engineUtils :context = nativerender.getContext(ContextType.ENGINE_UTILS);
-
+const mouseWheelNapi :context = nativerender.getContext(ContextType.MOUSE_WHEEL_NAPI);
 
 function executeMethodAsync(nativeFunc: Function, funcData: string, funCb: Function): void {
   nativeFunc && nativeFunc(funcData, funCb);
@@ -107,6 +107,7 @@ struct Index {
   private webViewIndexMap: Map<number, number> = new Map<number, number>();
   private videoIndexMap: Map<number, number> = new Map<number, number>();
   private workPort: PortProxy = new PortProxy(WorkerManager.getInstance().getWorker());
+  private panOption: PanGestureOptions = new PanGestureOptions({ direction: PanDirection.Up | PanDirection.Down});
   dialogController: CustomDialogController = new CustomDialogController({
     builder: EditBoxDialog({
       showInfo: this.showInfo,
@@ -431,13 +432,33 @@ struct Index {
     }
   }
 
+  onMouseWheel(eventType: string, event: PanGestureEvent) {
+      if(event.source == SourceType.Mouse) {
+        mouseWheelNapi.onMouseWheel(eventType, event.offsetY);
+      }
+  }
+
   build() {
     Flex({
       direction: FlexDirection.Column,
       alignItems: ItemAlign.Center,
       justifyContent: FlexAlign.Center
     } as FlexOptions) {
       XComponent({ id: 'xcomponentId', type: 'surface', libraryname: 'cocos' })
+        .focusable(true)
+        .defaultFocus(true)
+        .gesture(
+          PanGesture(this.panOption)
+            .onActionStart((event: GestureEvent) => {
+              this.onMouseWheel("actionStart", event);
+            })
+            .onActionUpdate((event: GestureEvent) => {
+              this.onMouseWheel("actionUpdate", event);
+            })
+            .onActionEnd((event: GestureEvent) => {
+              this.onMouseWheel("actionEnd", event);
+            })
+        )
         .onLoad((context) => {
           // Set the cache directory in the ts layer.
           this.workPort.postMessage("onXCLoad", "XComponent");

Original file line number	Diff line number	Diff line change
`@@ -25,7 +25,8 @@ export interface context {`
`25`	`25`	`resourceManagerInit: (resourceManager: resourceManager.ResourceManager) => void;`
`26`	`26`	`writablePathInit: (cacheDir: string) => void;`
`27`	`27`	`onVideoEvent: (param: string) => void;`
`28`		`- registerFunction: (name:string ,fun:Function) => void;`
	`28`	`+ onMouseWheel: (type: string, offsetY: number) => void;`
	`29`	`+ registerFunction: (name: string, fun: Function) => void;`
`29`	`30`	`}`
`30`	`31`
`31`	`32`	`export const getContext: (type: ContextType) => context;`
Original file line number	Diff line number	Diff line change
`@@ -34,7 +34,8 @@ export enum ContextType {`
`34`	`34`	`WEBVIEW_UTILS,`
`35`	`35`	`DISPLAY_UTILS,`
`36`	`36`	`UV_ASYNC_SEND,`
`37`		`- VIDEO_UTILS`
	`37`	`+ VIDEO_UTILS,`
	`38`	`+ MOUSE_WHEEL_NAPI`
`38`	`39`	`}`
`39`	`40`
`40`	`41`	`export class Constants {`