feat: test

package.json

@@ -37,6 +37,8 @@
   "dependencies": {
     "@ffmpeg/ffmpeg": "^0.12.10",
     "@ffmpeg/util": "^0.12.1",
+    "@nut-tree/nut-js": "^4.0.0",
+    "@nut-tree/shared": "^4.0.0",
     "@vicons/ionicons5": "^0.12.0",
     "@webav/av-recorder": "^0.3.3",
     "axios": "^1.2.1",

src/constant.ts

@@ -20,9 +20,14 @@ export const appBuildInfo =
   // @ts-ignore
   process.env.BilldHtmlWebpackPlugin as BilldHtmlWebpackPluginLog;
 
+// export const WEBSOCKET_URL =
+//   process.env.NODE_ENV === 'development'
+//     ? `ws://192.168.1.102:4300` // `ws://localhost:4300`
+//     : `wss://srs-pull.${prodDomain}`;
+
 export const WEBSOCKET_URL =
   process.env.NODE_ENV === 'development'
-    ? `ws://192.168.1.127:4300` // `ws://localhost:4300`
+    ? `ws://localhost:4300` // `ws://localhost:4300`
     : `wss://srs-pull.${prodDomain}`;
 
 export const AXIOS_BASEURL =

src/hooks/use-pull.ts

@@ -179,6 +179,11 @@ export function usePull(roomId: string) {
             videoElArr.value.push(item.videoEl);
           }
         });
+        nextTick(() => {
+          if (isRemoteDesk.value && videoWrapRef.value) {
+            videoWrapRef.value.style.display = 'inline-block';
+          }
+        });
       }
     },
     {

src/hooks/use-websocket.ts

@@ -39,7 +39,7 @@ import {
   WsMsgTypeEnum,
   WsOfferType,
   WsOtherJoinType,
-  WsRemoteDeskMoveMsgType,
+  WsRemoteDeskBehaviorType,
   WsRoomLivingType,
   WsStartLiveType,
   WsStartRemoteDesk,
@@ -523,11 +523,11 @@ export const useWebsocket = () => {
       }
     );
 
-    // 收到remoteDeskMoveMsg
+    // 收到remoteDeskBehavior
     ws.socketIo.on(
-      WsMsgTypeEnum.remoteDeskMoveMsg,
-      (data: WsRemoteDeskMoveMsgType['data']) => {
-        console.log('收到remoteDeskMoveMsg', data);
+      WsMsgTypeEnum.remoteDeskBehavior,
+      (data: WsRemoteDeskBehaviorType['data']) => {
+        console.log('收到remoteDeskBehavior', data);
       }
     );
 

src/store/app/index.ts

@@ -9,6 +9,8 @@ export type AppRootState = {
   remoteDesk: {
     sender: string;
     startRemoteDesk: boolean;
+    isRemoteing?: boolean;
+    isClose?: boolean;
   };
   playing: boolean;
   videoRatio: number;
@@ -28,7 +30,8 @@ export type AppRootState = {
     stream?: MediaStream;
     streamid?: string;
     trackid?: string;
-    canvasDom?: fabric.Image | fabric.Text;
+    // canvasDom?: fabric.Image | fabric.Text;
+    canvasDom?: any;
     hidden?: boolean;
     muted?: boolean;
     volume?: number;
@@ -72,6 +75,8 @@ export const useAppStore = defineStore('app', {
       remoteDesk: {
         startRemoteDesk: false,
         sender: '',
+        isRemoteing: false,
+        isClose: undefined,
       },
       playing: false,
       videoRatio: 16 / 9,

src/types/websocket.ts

@@ -68,7 +68,7 @@ export enum WsMsgTypeEnum {
   srsCandidate = 'srsCandidate',
 
   startRemoteDesk = 'startRemoteDesk',
-  remoteDeskMoveMsg = 'remoteDeskMoveMsg',
+  remoteDeskBehavior = 'remoteDeskBehavior',
 
   remoteDeskOffer = 'remoteDeskOffer',
   remoteDeskAnswer = 'remoteDeskAnswer',
@@ -128,10 +128,33 @@ export type WsRoomNoLiveType = IWsFormat<{
   live_room: ILiveRoom;
 }>;
 
-export type WsRemoteDeskMoveMsgType = IWsFormat<{
+export enum RemoteDeskBehaviorEnum {
+  move,
+  drag,
+  pressButtonLeft,
+  pressButtonRight,
+  releaseButtonLeft,
+  releaseButtonRight,
+  setPosition,
+  doubleClick,
+  leftClick,
+  rightClick,
+  scrollDown,
+  scrollUp,
+  scrollLeft,
+  scrollRight,
+
+  keyboardType,
+}
+
+export type WsRemoteDeskBehaviorType = IWsFormat<{
   roomId: string;
   sender: string;
   receiver: string;
+  type: RemoteDeskBehaviorEnum;
+  x: number;
+  y: number;
+  keyboardtype: string | number;
 }>;
 
 export interface IDanmu {

src/utils/network/webRTC.ts

@@ -72,6 +72,8 @@ export class WebRTCClass {
   videoEl: HTMLVideoElement;
 
   peerConnection: RTCPeerConnection | null = null;
+  dataChannel: RTCDataChannel | null = null;
+  cbDataChannel: RTCDataChannel | null = null;
 
   /** 最大码率 */
   maxBitrate = -1;
@@ -311,15 +313,15 @@ export class WebRTCClass {
       const stream = event.streams[0];
       this.localStream = stream;
       const appStore = useAppStore();
-      stream.onremovetrack = (event) => {
+      stream.onremovetrack = () => {
         this.prettierLog({ msg: 'onremovetrack事件', type: 'warn' });
-        const res = appStore.allTrack.filter((info) => {
-          if (info.track?.id === event.track.id) {
-            return false;
-          }
-          return true;
-        });
-        appStore.setAllTrack(res);
+        // const res = appStore.allTrack.filter((info) => {
+        //   if (info.track?.id === event.track.id) {
+        //     return false;
+        //   }
+        //   return true;
+        // });
+        // appStore.setAllTrack(res);
       };
 
       const addTrack: AppRootState['allTrack'] = [];
@@ -445,6 +447,7 @@ export class WebRTCClass {
             msg: 'webrtc连接成功！',
             type: 'success',
           });
+          appStore.remoteDesk.isRemoteing = true;
           console.log('sender', this.sender, 'receiver', this.receiver);
           this.update();
         }
@@ -461,6 +464,7 @@ export class WebRTCClass {
             msg: 'iceConnectionState:failed',
             type: 'error',
           });
+          this.close();
         }
         if (iceConnectionState === 'disconnected') {
           // 测试不再活跃，这可能是一个暂时的状态，可以自我恢复。
@@ -468,6 +472,7 @@ export class WebRTCClass {
             msg: 'iceConnectionState:disconnected',
             type: 'error',
           });
+          this.close();
         }
         if (iceConnectionState === 'closed') {
           // ICE 代理关闭，不再应答任何请求。
@@ -514,6 +519,7 @@ export class WebRTCClass {
             msg: 'connectionState:disconnected',
             type: 'error',
           });
+          this.close();
         }
         if (connectionState === 'closed') {
           // 表示 RTCPeerConnection 已关闭。
@@ -528,6 +534,7 @@ export class WebRTCClass {
             msg: 'connectionState:failed',
             type: 'error',
           });
+          this.close();
         }
       }
     );
@@ -544,6 +551,29 @@ export class WebRTCClass {
     });
   };
 
+  dataChannelSend = <T extends unknown>({
+    // 写成<T extends unknown>而不是<T>是为了避免eslint将箭头函数的<T>后面的内容识别成jsx语法
+    msgType,
+    requestId,
+    data,
+  }: {
+    msgType: WsMsgTypeEnum;
+    requestId: string;
+    data?: T;
+  }) => {
+    if (this.dataChannel?.readyState !== 'open') {
+      console.error('dataChannel未连接成功，不发送消息！', msgType, data);
+      return;
+    }
+    this.dataChannel.send(
+      JSON.stringify({
+        msgType,
+        requestId,
+        data,
+      })
+    );
+  };
+
   /** 创建对等连接 */
   createPeerConnection = () => {
     if (!window.RTCPeerConnection) {
@@ -567,6 +597,32 @@ export class WebRTCClass {
       this.peerConnection = new RTCPeerConnection({
         iceServers,
       });
+      this.peerConnection.ondatachannel = (event) => {
+        this.cbDataChannel = event.channel;
+        this.update();
+      };
+      this.dataChannel = this.peerConnection.createDataChannel(
+        'MessageChannel',
+        {
+          // maxRetransmits，用户代理应尝试重新传输在不可靠模式下第一次失败的消息的最大次数。虽然该值是 16 位无符号数，但每个用户代理都可以将其限制为它认为合适的任何最大值。
+          maxRetransmits: 3,
+          // ordered，表示通过 RTCDataChannel 的信息的到达顺序需要和发送顺序一致 (true), 或者到达顺序不需要和发送顺序一致 (false). 默认：true
+          ordered: false,
+          protocol: 'udp',
+        }
+      );
+      this.dataChannel.onopen = () => {
+        this.prettierLog({
+          msg: 'dataChannel连接成功！',
+          type: 'success',
+        });
+      };
+      this.dataChannel.onerror = () => {
+        this.prettierLog({
+          msg: 'dataChannel连接失败！',
+          type: 'error',
+        });
+      };
       this.handleStreamEvent();
       this.handleConnectionEvent();
       this.update();
@@ -583,7 +639,11 @@ export class WebRTCClass {
       this.localStream = null;
       this.peerConnection?.close();
       this.peerConnection = null;
+      this.dataChannel = null;
       this.videoEl.remove();
+      const appStore = useAppStore();
+      appStore.remoteDesk.isClose = true;
+      appStore.remoteDesk.isRemoteing = false;
     } catch (error) {
       this.prettierLog({ msg: '手动关闭webrtc连接失败', type: 'error' });
       console.error(error);

src/views/remoteDesktop/index.vue

@@ -1,76 +1,318 @@
 <template>
   <div>
-    <h1>当前是控制方</h1>
-    <h1>https://github.com/galaxy-s10/billd-desk</h1>
-    <div>
-      房间id：{{ roomId }}，我的id：{{ mySocketId }}，<n-button
-        @click="copyToClipBoard(mySocketId)"
-      >
+    <h1>
+      我的id：{{ mySocketId }}，<n-button @click="copyToClipBoard(mySocketId)">
         复制
       </n-button>
+    </h1>
+    <div>
+      <n-input-group>
+        <n-input-group-label>被控id</n-input-group-label>
+        <n-input
+          :style="{ width: '200px' }"
+          placeholder="请输入被控id"
+          v-model:value="receiverId"
+        />
+
+        <n-button
+          type="error"
+          @click="handleClose"
+          v-if="appStore.remoteDesk.isRemoteing"
+        >
+          结束控制
+        </n-button>
+        <n-button
+          v-else
+          type="primary"
+          @click="handleRemote"
+        >
+          开始远程
+        </n-button>
+      </n-input-group>
     </div>
-    <n-input-group>
-      <n-input-group-label>被控id</n-input-group-label>
-      <n-input
-        :style="{ width: '200px' }"
-        placeholder="请输入被控id"
-        v-model:value="receiverId"
-      />
-      <n-button
-        type="primary"
-        @click="handleRemote"
-      >
-        开始远程
-      </n-button>
-      <n-button @click="handleMoveMouse">移动鼠标</n-button>
-    </n-input-group>
+
     <div
       class="wrap"
       ref="remoteVideoRef"
+      @mousedown="handleMouseDown"
+      @mousemove="handleMouseMove"
+      @mouseup="handleMouseUp"
+      @dblclick="handleDoublelclick"
+      @contextmenu="handleContextmenu"
     ></div>
   </div>
 </template>
 
 <script lang="ts" setup>
+import { Key } from '@nut-tree/shared';
 import { copyToClipBoard, getRandomString } from 'billd-utils';
-import { computed, onMounted, ref } from 'vue';
+import { computed, onMounted, onUnmounted, ref, watch } from 'vue';
 
 import { usePull } from '@/hooks/use-pull';
+import { useTip } from '@/hooks/use-tip';
+import { useAppStore } from '@/store/app';
 import { useNetworkStore } from '@/store/network';
 import {
+  RemoteDeskBehaviorEnum,
   WsMsgTypeEnum,
-  WsRemoteDeskMoveMsgType,
+  WsRemoteDeskBehaviorType,
   WsStartRemoteDesk,
 } from '@/types/websocket';
 
 const num = '123456';
+const appStore = useAppStore();
 const networkStore = useNetworkStore();
 const { videoWrapRef, initPull } = usePull(num);
+
 const roomId = ref(num);
 const receiverId = ref('');
 const remoteVideoRef = ref<HTMLDivElement>();
-
+const isDown = ref(false);
+let clickTimer;
+let isLongClick = false;
 const mySocketId = computed(() => {
   return networkStore.wsMap.get(roomId.value)?.socketIo?.id || '-1';
 });
 
+onUnmounted(() => {
+  window.removeEventListener('keydown', handleKeyDown);
+  handleClose();
+});
+
 onMounted(() => {
   initPull({
     isRemoteDesk: true,
   });
   videoWrapRef.value = remoteVideoRef.value;
+  window.addEventListener('keydown', handleKeyDown);
 });
 
-function handleMoveMouse() {
-  networkStore.wsMap.get(roomId.value)?.send<WsRemoteDeskMoveMsgType['data']>({
-    requestId: getRandomString(8),
-    msgType: WsMsgTypeEnum.remoteDeskMoveMsg,
-    data: {
-      roomId: roomId.value,
-      sender: mySocketId.value,
-      receiver: receiverId.value,
-    },
-  });
+function handleKeyDown(e: KeyboardEvent) {
+  // console.log(e.key, e.code);
+  const keyMap = {
+    Enter: Key.Enter,
+    Space: Key.Space,
+    Backspace: Key.Backspace,
+    ShiftLeft: Key.LeftShift,
+    ShiftRight: Key.RightShift,
+    AltLeft: Key.LeftAlt,
+    AltRight: Key.RightAlt,
+    Tab: Key.Tab,
+    Backquote: Key.Quote,
+    Backslash: Key.Backslash,
+    ArrowUp: Key.Up,
+    ArrowDown: Key.Down,
+    ArrowLeft: Key.Left,
+    ArrowRight: Key.Right,
+    CapsLock: Key.CapsLock,
+    ControlLeft: Key.LeftControl,
+    ControlRight: Key.RightControl,
+    MetaLeft: Key.LeftCmd,
+    LeftWin: Key.LeftCmd,
+    MetaRight: Key.RightCmd,
+    RightWin: Key.RightCmd,
+    Fn: Key.Fn,
+    F1: Key.F1,
+    F2: Key.F2,
+    F3: Key.F3,
+    F4: Key.F4,
+    F5: Key.F5,
+    F6: Key.F6,
+    F7: Key.F7,
+    F8: Key.F8,
+    F9: Key.F9,
+    F10: Key.F10,
+    F11: Key.F11,
+    F12: Key.F12,
+    F13: Key.F13,
+    F14: Key.F14,
+    F15: Key.F15,
+    F16: Key.F16,
+    F17: Key.F17,
+    F18: Key.F18,
+    F19: Key.F19,
+    F20: Key.F20,
+    F21: Key.F21,
+    F22: Key.F22,
+    F23: Key.F23,
+    F24: Key.F24,
+  };
+
+  networkStore.rtcMap
+    .get(receiverId.value)
+    ?.dataChannelSend<WsRemoteDeskBehaviorType['data']>({
+      requestId: getRandomString(8),
+      msgType: WsMsgTypeEnum.remoteDeskBehavior,
+      data: {
+        roomId: roomId.value,
+        sender: mySocketId.value,
+        receiver: receiverId.value,
+        type: RemoteDeskBehaviorEnum.keyboardType,
+        keyboardtype: keyMap[e.code] || e.key,
+        x: 0,
+        y: 0,
+      },
+    });
+}
+
+watch(
+  () => appStore.remoteDesk.isClose,
+  (newval) => {
+    if (newval) {
+      useTip({
+        content: '远程连接断开',
+        hiddenCancel: true,
+        hiddenClose: true,
+      }).catch();
+    }
+  }
+);
+
+function handleDoublelclick() {
+  networkStore.rtcMap
+    .get(receiverId.value)
+    ?.dataChannelSend<WsRemoteDeskBehaviorType['data']>({
+      requestId: getRandomString(8),
+      msgType: WsMsgTypeEnum.remoteDeskBehavior,
+      data: {
+        roomId: roomId.value,
+        sender: mySocketId.value,
+        receiver: receiverId.value,
+        type: RemoteDeskBehaviorEnum.doubleClick,
+        keyboardtype: 0,
+        x: 0,
+        y: 0,
+      },
+    });
+}
+
+function handleContextmenu() {
+  networkStore.rtcMap
+    .get(receiverId.value)
+    ?.dataChannelSend<WsRemoteDeskBehaviorType['data']>({
+      requestId: getRandomString(8),
+      msgType: WsMsgTypeEnum.remoteDeskBehavior,
+      data: {
+        roomId: roomId.value,
+        sender: mySocketId.value,
+        receiver: receiverId.value,
+        type: RemoteDeskBehaviorEnum.rightClick,
+        keyboardtype: 0,
+        x: 0,
+        y: 0,
+      },
+    });
+}
+
+function handleMouseDown(event: MouseEvent) {
+  isDown.value = true;
+  clickTimer = setTimeout(function () {
+    console.log('长按');
+    isLongClick = true;
+    clearTimeout(clickTimer);
+  }, 300);
+  // 获取点击相对于视窗的位置
+  const clickX = event.clientX;
+  const clickY = event.clientY;
+
+  // 获取目标元素的位置和尺寸信息
+  // @ts-ignore
+  const rect: DOMRect = event.target.getBoundingClientRect();
+  // 计算点击位置相对于元素的坐标
+  const xInsideElement = clickX - rect.left;
+  const yInsideElement = clickY - rect.top;
+  const x = (xInsideElement / rect.width) * 1000;
+  const y = (yInsideElement / rect.height) * 1000;
+  console.log('handleMouseDown', x, y, xInsideElement, yInsideElement);
+  networkStore.rtcMap
+    .get(receiverId.value)
+    ?.dataChannelSend<WsRemoteDeskBehaviorType['data']>({
+      requestId: getRandomString(8),
+      msgType: WsMsgTypeEnum.remoteDeskBehavior,
+      data: {
+        roomId: roomId.value,
+        sender: mySocketId.value,
+        receiver: receiverId.value,
+        type: isLongClick
+          ? RemoteDeskBehaviorEnum.pressButtonLeft
+          : RemoteDeskBehaviorEnum.pressButtonLeft,
+        x,
+        y,
+      },
+    });
+}
+
+function handleMouseMove(event: MouseEvent) {
+  // 获取点击相对于视窗的位置
+  const clickX = event.clientX;
+  const clickY = event.clientY;
+
+  // 获取目标元素的位置和尺寸信息
+  // @ts-ignore
+  const rect: DOMRect = event.target.getBoundingClientRect();
+  // 计算点击位置相对于元素的坐标
+  const xInsideElement = clickX - rect.left;
+  const yInsideElement = clickY - rect.top;
+  const x = (xInsideElement / rect.width) * 1000;
+  const y = (yInsideElement / rect.height) * 1000;
+  console.log('handleMouseMove', x, y, xInsideElement, yInsideElement);
+  networkStore.rtcMap
+    .get(receiverId.value)
+    ?.dataChannelSend<WsRemoteDeskBehaviorType['data']>({
+      requestId: getRandomString(8),
+      msgType: WsMsgTypeEnum.remoteDeskBehavior,
+      data: {
+        roomId: roomId.value,
+        sender: mySocketId.value,
+        receiver: receiverId.value,
+        type: RemoteDeskBehaviorEnum.move,
+        keyboardtype: 0,
+        x,
+        y,
+      },
+    });
+}
+
+function handleMouseUp(event: MouseEvent) {
+  if (clickTimer) {
+    clearTimeout(clickTimer);
+  }
+  isDown.value = false;
+  // 获取点击相对于视窗的位置
+  const clickX = event.clientX;
+  const clickY = event.clientY;
+
+  // 获取目标元素的位置和尺寸信息
+  // @ts-ignore
+  const rect: DOMRect = event.target.getBoundingClientRect();
+  // 计算点击位置相对于元素的坐标
+  const xInsideElement = clickX - rect.left;
+  const yInsideElement = clickY - rect.top;
+  const x = (xInsideElement / rect.width) * 1000;
+  const y = (yInsideElement / rect.height) * 1000;
+  console.log('handleMouseUp', x, y, xInsideElement, yInsideElement);
+  networkStore.rtcMap
+    .get(receiverId.value)
+    ?.dataChannelSend<WsRemoteDeskBehaviorType['data']>({
+      requestId: getRandomString(8),
+      msgType: WsMsgTypeEnum.remoteDeskBehavior,
+      data: {
+        roomId: roomId.value,
+        sender: mySocketId.value,
+        receiver: receiverId.value,
+        keyboardtype: '',
+        type: isLongClick
+          ? RemoteDeskBehaviorEnum.releaseButtonLeft
+          : RemoteDeskBehaviorEnum.releaseButtonLeft,
+        x,
+        y,
+      },
+    });
+  isLongClick = false;
+}
+
+function handleClose() {
+  networkStore.rtcMap.get(receiverId.value)?.close();
 }
 
 function handleRemote() {
@@ -88,6 +330,7 @@ function handleRemote() {
 
 <style lang="scss" scoped>
 .wrap {
-  display: inline-block;
+  line-height: 0;
+  cursor: none;
 }
 </style>