Skip to content

Instantly share code, notes, and snippets.

@donghee

donghee/client.js

Last active July 1, 2023 10:34
Show Gist options
  • Save donghee/cb673a575318c3ddeb837c40855ee74d to your computer and use it in GitHub Desktop.
Save donghee/cb673a575318c3ddeb837c40855ee74d to your computer and use it in GitHub Desktop.
Download ZIP
Ground Station: Transmit the joystick command to the robot via WebRTC data channel https://www.youtube.com/watch?v=d-2WEI0uTZA
Raw
client.js
var pc = null;
var localVideo = document.querySelector("video#localVideo");
var serverVideo = document.querySelector("video#serverVideo");
navigator.mediaDevices
.getUserMedia({
video: {
height: 360,
width: 480,
},
})
.then((stream) => {
localVideo.srcObject = stream;
localVideo.addEventListener("loadedmetadata", () => {
localVideo.play();
});
});
function negotiate() {
return pc
.createOffer()
.then(function (offer) {
return pc.setLocalDescription(offer);
})
.then(function () {
// wait for ICE gathering to complete
return new Promise(function (resolve) {
if (pc.iceGatheringState === "complete") {
resolve();
} else {
function checkState() {
if (pc.iceGatheringState === "complete") {
pc.removeEventListener("icegatheringstatechange", checkState);
resolve();
}
}
pc.addEventListener("icegatheringstatechange", checkState);
}
});
})
.then(function () {
var offer = pc.localDescription;
return fetch("/offer", {
body: JSON.stringify({
sdp: offer.sdp,
type: offer.type,
}),
headers: {
"Content-Type": "application/json",
},
method: "POST",
});
})
.then(function (response) {
return response.json();
})
.then(function (answer) {
return pc.setRemoteDescription(answer);
})
.catch(function (e) {
alert(e);
});
}
function start(joystick) {
var config = {
sdpSemantics: "unified-plan",
iceServers: [{ urls: ["stun:stun.l.google.com:19302"] }],
};
pc = new RTCPeerConnection(config);
localVideo.srcObject.getVideoTracks().forEach((track) => {
pc.addTrack(track);
});
pc.addEventListener("track", function (evt) {
console.log("receive server video");
if (evt.track.kind == "video") {
serverVideo.srcObject = evt.streams[0];
}
});
ch = pc.createDataChannel("chat", {
ordered: false,
maxRetransmits: 0,
});
ch.addEventListener("message", function (evt) {
console.log(Date.now() - JSON.parse(evt.data).now);
//joystick.GetX();
//joystick.GetY();
ch.send("X:" + joystick.GetX() + " Y:" + joystick.GetY());
});
document.getElementById("start").style.display = "none";
negotiate();
document.getElementById("stop").style.display = "inline-block";
}
function stop() {
document.getElementById("stop").style.display = "none";
setTimeout(function () {
pc.close();
}, 500);
}
Raw
index.html
<html>
<head>
<meta charset="UTF-8"/>
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>WebRTC webcam</title>
<style>
button {
padding: 8px 16px;
}
video {
width: 70%;
}
.option {
margin-bottom: 8px;
}
#media {
max-width: 1280px;
}
</style>
</head>
<body>
<button id="start" onclick="start(Joy3)">Start</button>
<button id="stop" style="display: none" onclick="stop()">Stop</button>
<div id="media">
<h2>Local</h2>
<video id="localVideo" autoplay="true" playsinline="true"></video>
<h2>Remote</h2>
<video id="serverVideo" autoplay="true" playsinline="true"></video>
</div>
<script src="client.js"></script>
<script src="joy.js"></script>
<div id="joy3Div" style="width:200px;height:200px;margin:50px;position:fixed;bottom:30px;right:0px;"></div>
<div style="position:fixed;bottom:35px;right:550px;">
X :<input id="joy3X" type="text" /></br>
Y :<input id="joy3Y" type="text" />
</div>
<script type="text/javascript">
var joy3Param = { "title": "joystick3" };
var Joy3 = new JoyStick('joy3Div', joy3Param);
var joy3X = document.getElementById("joy3X");
var joy3Y = document.getElementById("joy3Y");
setInterval(function(){ joy3X.value=Joy3.GetX(); }, 50);
setInterval(function(){ joy3Y.value=Joy3.GetY(); }, 50);
</script>
</body>
</html>
Raw
joy.js
/*
* Name : joy.js
* @author : Roberto D'Amico (Bobboteck)
* Last modified : 09.06.2020
* Revision : 1.1.6
*
* Modification History:
* Date Version Modified By Description
* 2020-06-09 1.1.6 Roberto D'Amico Fixed Issue #10 and #11
* 2020-04-20 1.1.5 Roberto D'Amico Correct: Two sticks in a row, thanks to @liamw9534 for the suggestion
* 2020-04-03 Roberto D'Amico Correct: InternalRadius when change the size of canvas, thanks to @vanslipon for the suggestion
* 2020-01-07 1.1.4 Roberto D'Amico Close #6 by implementing a new parameter to set the functionality of auto-return to 0 position
* 2019-11-18 1.1.3 Roberto D'Amico Close #5 correct indication of East direction
* 2019-11-12 1.1.2 Roberto D'Amico Removed Fix #4 incorrectly introduced and restored operation with touch devices
* 2019-11-12 1.1.1 Roberto D'Amico Fixed Issue #4 - Now JoyStick work in any position in the page, not only at 0,0
*
* The MIT License (MIT)
*
* This file is part of the JoyStick Project (https://github.com/bobboteck/JoyStick).
* Copyright (c) 2015 Roberto D'Amico (Bobboteck).
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
/**
* @desc Principal object that draw a joystick, you only need to initialize the object and suggest the HTML container
* @costructor
* @param container {String} - HTML object that contains the Joystick
* @param parameters (optional) - object with following keys:
* title {String} (optional) - The ID of canvas (Default value is 'joystick')
* width {Int} (optional) - The width of canvas, if not specified is setted at width of container object (Default value is the width of container object)
* height {Int} (optional) - The height of canvas, if not specified is setted at height of container object (Default value is the height of container object)
* internalFillColor {String} (optional) - Internal color of Stick (Default value is '#00AA00')
* internalLineWidth {Int} (optional) - Border width of Stick (Default value is 2)
* internalStrokeColor {String}(optional) - Border color of Stick (Default value is '#003300')
* externalLineWidth {Int} (optional) - External reference circonference width (Default value is 2)
* externalStrokeColor {String} (optional) - External reference circonference color (Default value is '#008000')
* autoReturnToCenter {Bool} (optional) - Sets the behavior of the stick, whether or not, it should return to zero position when released (Default value is True and return to zero)
*/
var JoyStick = function (container, parameters) {
parameters = parameters || {};
var title =
typeof parameters.title === "undefined" ? "joystick" : parameters.title,
width = typeof parameters.width === "undefined" ? 0 : parameters.width,
height = typeof parameters.height === "undefined" ? 0 : parameters.height,
internalFillColor =
typeof parameters.internalFillColor === "undefined"
? "#00AA00"
: parameters.internalFillColor,
internalLineWidth =
typeof parameters.internalLineWidth === "undefined"
? 2
: parameters.internalLineWidth,
internalStrokeColor =
typeof parameters.internalStrokeColor === "undefined"
? "#003300"
: parameters.internalStrokeColor,
externalLineWidth =
typeof parameters.externalLineWidth === "undefined"
? 2
: parameters.externalLineWidth,
externalStrokeColor =
typeof parameters.externalStrokeColor === "undefined"
? "#008000"
: parameters.externalStrokeColor,
autoReturnToCenter =
typeof parameters.autoReturnToCenter === "undefined"
? true
: parameters.autoReturnToCenter;
// Create Canvas element and add it in the Container object
var objContainer = document.getElementById(container);
var canvas = document.createElement("canvas");
canvas.id = title;
if (width === 0) {
width = objContainer.clientWidth;
}
if (height === 0) {
height = objContainer.clientHeight;
}
canvas.width = width;
canvas.height = height;
objContainer.appendChild(canvas);
var context = canvas.getContext("2d");
var pressed = 0; // Bool - 1=Yes - 0=No
var circumference = 2 * Math.PI;
var internalRadius = (canvas.width - (canvas.width / 2 + 10)) / 2;
var maxMoveStick = internalRadius + 5;
var externalRadius = internalRadius + 30;
var centerX = canvas.width / 2;
var centerY = canvas.height / 2;
var directionHorizontalLimitPos = canvas.width / 10;
var directionHorizontalLimitNeg = directionHorizontalLimitPos * -1;
var directionVerticalLimitPos = canvas.height / 10;
var directionVerticalLimitNeg = directionVerticalLimitPos * -1;
// Used to save current position of stick
var movedX = centerX;
var movedY = centerY;
// Check if the device support the touch or not
if ("ontouchstart" in document.documentElement) {
canvas.addEventListener("touchstart", onTouchStart, false);
canvas.addEventListener("touchmove", onTouchMove, false);
canvas.addEventListener("touchend", onTouchEnd, false);
} else {
canvas.addEventListener("mousedown", onMouseDown, false);
canvas.addEventListener("mousemove", onMouseMove, false);
canvas.addEventListener("mouseup", onMouseUp, false);
}
// Draw the object
drawExternal();
drawInternal();
/******************************************************
* Private methods
*****************************************************/
/**
* @desc Draw the external circle used as reference position
*/
function drawExternal() {
context.beginPath();
context.arc(centerX, centerY, externalRadius, 0, circumference, false);
context.lineWidth = externalLineWidth;
context.strokeStyle = externalStrokeColor;
context.stroke();
}
/**
* @desc Draw the internal stick in the current position the user have moved it
*/
function drawInternal() {
context.beginPath();
if (movedX < internalRadius) {
movedX = maxMoveStick;
}
if (movedX + internalRadius > canvas.width) {
movedX = canvas.width - maxMoveStick;
}
if (movedY < internalRadius) {
movedY = maxMoveStick;
}
if (movedY + internalRadius > canvas.height) {
movedY = canvas.height - maxMoveStick;
}
context.arc(movedX, movedY, internalRadius, 0, circumference, false);
// create radial gradient
var grd = context.createRadialGradient(
centerX,
centerY,
5,
centerX,
centerY,
200
);
// Light color
grd.addColorStop(0, internalFillColor);
// Dark color
grd.addColorStop(1, internalStrokeColor);
context.fillStyle = grd;
context.fill();
context.lineWidth = internalLineWidth;
context.strokeStyle = internalStrokeColor;
context.stroke();
}
/**
* @desc Events for manage touch
*/
function onTouchStart(event) {
pressed = 1;
}
function onTouchMove(event) {
// Prevent the browser from doing its default thing (scroll, zoom)
event.preventDefault();
if (pressed === 1 && event.targetTouches[0].target === canvas) {
movedX = event.targetTouches[0].pageX;
movedY = event.targetTouches[0].pageY;
// Manage offset
if (canvas.offsetParent.tagName.toUpperCase() === "BODY") {
movedX -= canvas.offsetLeft;
movedY -= canvas.offsetTop;
} else {
movedX -= canvas.offsetParent.offsetLeft;
movedY -= canvas.offsetParent.offsetTop;
}
// Delete canvas
context.clearRect(0, 0, canvas.width, canvas.height);
// Redraw object
drawExternal();
drawInternal();
}
}
function onTouchEnd(event) {
pressed = 0;
// If required reset position store variable
if (autoReturnToCenter) {
movedX = centerX;
movedY = centerY;
}
// Delete canvas
context.clearRect(0, 0, canvas.width, canvas.height);
// Redraw object
drawExternal();
drawInternal();
//canvas.unbind('touchmove');
}
/**
* @desc Events for manage mouse
*/
function onMouseDown(event) {
pressed = 1;
}
function onMouseMove(event) {
if (pressed === 1) {
movedX = event.pageX;
movedY = event.pageY;
// Manage offset
if (canvas.offsetParent.tagName.toUpperCase() === "BODY") {
movedX -= canvas.offsetLeft;
movedY -= canvas.offsetTop;
} else {
movedX -= canvas.offsetParent.offsetLeft;
movedY -= canvas.offsetParent.offsetTop;
}
// Delete canvas
context.clearRect(0, 0, canvas.width, canvas.height);
// Redraw object
drawExternal();
drawInternal();
}
}
function onMouseUp(event) {
pressed = 0;
// If required reset position store variable
if (autoReturnToCenter) {
movedX = centerX;
movedY = centerY;
}
// Delete canvas
context.clearRect(0, 0, canvas.width, canvas.height);
// Redraw object
drawExternal();
drawInternal();
//canvas.unbind('mousemove');
}
/******************************************************
* Public methods
*****************************************************/
/**
* @desc The width of canvas
* @return Number of pixel width
*/
this.GetWidth = function () {
return canvas.width;
};
/**
* @desc The height of canvas
* @return Number of pixel height
*/
this.GetHeight = function () {
return canvas.height;
};
/**
* @desc The X position of the cursor relative to the canvas that contains it and to its dimensions
* @return Number that indicate relative position
*/
this.GetPosX = function () {
return movedX;
};
/**
* @desc The Y position of the cursor relative to the canvas that contains it and to its dimensions
* @return Number that indicate relative position
*/
this.GetPosY = function () {
return movedY;
};
/**
* @desc Normalizzed value of X move of stick
* @return Integer from -100 to +100
*/
this.GetX = function () {
return (100 * ((movedX - centerX) / maxMoveStick)).toFixed();
};
/**
* @desc Normalizzed value of Y move of stick
* @return Integer from -100 to +100
*/
this.GetY = function () {
return (100 * ((movedY - centerY) / maxMoveStick) * -1).toFixed();
};
/**
* @desc Get the direction of the cursor as a string that indicates the cardinal points where this is oriented
* @return String of cardinal point N, NE, E, SE, S, SW, W, NW and C when it is placed in the center
*/
this.GetDir = function () {
var result = "";
var orizontal = movedX - centerX;
var vertical = movedY - centerY;
if (
vertical >= directionVerticalLimitNeg &&
vertical <= directionVerticalLimitPos
) {
result = "C";
}
if (vertical < directionVerticalLimitNeg) {
result = "N";
}
if (vertical > directionVerticalLimitPos) {
result = "S";
}
if (orizontal < directionHorizontalLimitNeg) {
if (result === "C") {
result = "W";
} else {
result += "W";
}
}
if (orizontal > directionHorizontalLimitPos) {
if (result === "C") {
result = "E";
} else {
result += "E";
}
}
return result;
};
};
Raw
main.py
import argparse
import asyncio
import json
import logging
import os
import platform
import ssl
import time
import cv2
from aiohttp import web
from aiortc import (
MediaStreamTrack,
RTCDataChannel,
RTCPeerConnection,
RTCSessionDescription,
VideoStreamTrack,
)
from aiortc.contrib.media import MediaPlayer, MediaRelay
from av import VideoFrame
ROOT = os.path.dirname(__file__)
relay = None
webcam = None
new_video_track = None
async def index(request):
content = open(os.path.join(ROOT, "index.html"), "r").read()
return web.Response(content_type="text/html", text=content)
async def javascript(request):
content = open(os.path.join(ROOT, "client.js"), "r").read()
return web.Response(content_type="application/javascript", text=content)
async def joystick(request):
content = open(os.path.join(ROOT, "joy.js"), "r").read()
return web.Response(content_type="application/javascript", text=content)
async def offer(request):
params = await request.json()
offer = RTCSessionDescription(sdp=params["sdp"], type=params["type"])
pc = RTCPeerConnection()
pcs.add(pc)
await server(pc, offer)
return web.Response(
content_type="application/json",
text=json.dumps(
{"sdp": pc.localDescription.sdp, "type": pc.localDescription.type}
),
)
pcs = set()
async def server(pc, offer):
@pc.on("connectionstatechange")
async def on_connectionstatechange():
print("Connection state is %s" % pc.connectionState)
if pc.connectionState == "failed":
await pc.close()
pcs.discard(pc)
@pc.on("track")
def on_track(track):
if track.kind == "video":
global new_video_track
#new_video_track = FaceSwapper(track)
new_video_track = RobotCamera()
pc.addTrack(new_video_track)
@pc.on("datachannel")
def on_datachannel(channel):
global new_video_track
new_video_track.channel = channel
print("mounted channel")
@channel.on("message")
async def on_message(message):
if isinstance(message, str):
data = message.encode("utf-8")
else:
data = message
print("joystick: ", data)
await pc.setRemoteDescription(offer)
answer = await pc.createAnswer()
await pc.setLocalDescription(answer)
async def on_shutdown(app):
# close peer connections
coros = [pc.close() for pc in pcs]
await asyncio.gather(*coros)
pcs.clear()
class RobotCamera(VideoStreamTrack):
kind = "video"
def __init__(self):
super().__init__()
self.channel = None
self.cap = cv2.VideoCapture(0)
async def recv(self):
pts, time_base = await self.next_timestamp()
ret, frame = self.cap.read()
vf = VideoFrame.from_ndarray(frame)
vf.pts = pts
vf.time_base = time_base
if self.channel is not None:
self.channel.send(
json.dumps(
{
"now": time.time() * 1000,
}
)
)
return vf
class FaceSwapper(VideoStreamTrack):
kind = "video"
def __init__(self, track):
super().__init__()
self.track = track
self.channel = None
async def recv(self):
timestamp, video_timestamp_base = await self.next_timestamp()
frame = await self.track.recv()
if self.channel is not None:
self.channel.send(
json.dumps(
{
"now": time.time() * 1000,
}
)
)
return frame
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="WebRTC webcam demo")
parser.add_argument(
"--host", default="0.0.0.0", help="Host for HTTP server (default: 0.0.0.0)"
)
parser.add_argument(
"--port", type=int, default=8080, help="Port for HTTP server (default: 8080)"
)
args = parser.parse_args()
logging.basicConfig(level=logging.INFO)
app = web.Application()
app.on_shutdown.append(on_shutdown)
app.router.add_get("/", index)
app.router.add_get("/client.js", javascript)
app.router.add_get("/joy.js", joystick)
app.router.add_post("/offer", offer)
web.run_app(app, host=args.host, port=args.port, ssl_context=None)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment