Last active
January 20, 2023 09:53
-
-
Save wizard1066/08046c5aafe91ca9f9f0aca45fb1e1b7 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| // | |
| // ContentView.swift | |
| // ARKit3 | |
| // | |
| // Created by localuser on 25.12.22. | |
| // | |
| import SwiftUI | |
| import ARKit | |
| import Combine | |
| import Foundation | |
| import CoreML | |
| let feedback = PassthroughSubject<CGPoint,Never>() | |
| let screenSize: CGRect = UIScreen.main.bounds | |
| let screenWidth: CGFloat = UIScreen.main.bounds.width | |
| let screenHeight: CGFloat = UIScreen.main.bounds.height | |
| struct Fonts { | |
| static func avenirNextCondensedBold (size: CGFloat) -> Font { | |
| return Font.custom("AvenirNextCondensed-Bold", size: size) | |
| } | |
| static func neutonRegular (size: CGFloat) -> Font { | |
| return Font.custom("Neuton-Regular", size: size) | |
| } | |
| } | |
| class Looker: ObservableObject { | |
| static var shared = Looker() | |
| @Published var paused = false | |
| @Published var gazeX = "" | |
| @Published var gazeY = "" | |
| @Published var gazeA = "" | |
| @Published var outOfRange = false | |
| @Published var leftIn = "0" | |
| @Published var leftOut = "0" | |
| @Published var rightIn = "0" | |
| @Published var rightOut = "0" | |
| @Published var leftUp = "0" | |
| @Published var leftDown = "0" | |
| @Published var rightUp = "0" | |
| @Published var rightDown = "0" | |
| } | |
| extension Text { | |
| init(_ string: String, configure: ((inout AttributedString) -> Void)) { | |
| var attributedString = AttributedString(string) /// create an `AttributedString` | |
| configure(&attributedString) | |
| self.init(attributedString) | |
| } | |
| } | |
| struct ContentView: View { | |
| @State var arview = ARSCNView() | |
| @State var gaze = CGPoint.zero | |
| @ObservedObject var looked = Looker.shared | |
| var body: some View { | |
| ZStack { | |
| CustomARView(view: arview) | |
| Circle() | |
| .stroke(looked.outOfRange ? .red : .green, lineWidth: looked.outOfRange ? 4 : 1) | |
| .frame(width: 312, height: 312) | |
| VStack(spacing:0) { | |
| HStack { | |
| Text("LI \(looked.leftIn)") | |
| .font(Fonts.avenirNextCondensedBold(size: 24)) | |
| .foregroundColor(Double(looked.leftIn)! > 0 ? Color.green: Color.clear) | |
| .background(Double(looked.leftIn)! > 0 ? Color.white: Color.clear) | |
| Text("LO \(looked.leftOut)") | |
| .font(Fonts.avenirNextCondensedBold(size: 24)) | |
| .foregroundColor(Double(looked.leftOut)! > 0 ? Color.green: Color.clear) | |
| .background(Double(looked.leftOut)! > 0 ? Color.white: Color.clear) | |
| Text("RI \(looked.rightIn)") | |
| .font(Fonts.avenirNextCondensedBold(size: 24)) | |
| .foregroundColor(Double(looked.rightIn)! > 0 ? Color.green: Color.clear) | |
| .background(Double(looked.rightIn)! > 0 ? Color.white: Color.clear) | |
| Text("RO \(looked.rightOut)") | |
| .font(Fonts.avenirNextCondensedBold(size: 24)) | |
| .foregroundColor(Double(looked.rightOut)! > 0 ? Color.green: Color.clear) | |
| .background(Double(looked.rightOut)! > 0 ? Color.white: Color.clear) | |
| VStack { | |
| Spacer() | |
| HStack { | |
| Text("\(looked.gazeY)") | |
| .font(Fonts.avenirNextCondensedBold(size: 24)) | |
| .foregroundColor(Color.blue) | |
| .background(Color.white) | |
| Text("\(looked.gazeA)") | |
| .font(Fonts.avenirNextCondensedBold(size: 24)) | |
| .foregroundColor(Color.red) | |
| .background(Color.white) | |
| Text("\(looked.gazeX)") | |
| .font(Fonts.avenirNextCondensedBold(size: 24)) | |
| .foregroundColor(Color.blue) | |
| .background(Color.white) | |
| } | |
| } | |
| Text("LU \(looked.leftUp)") | |
| .font(Fonts.avenirNextCondensedBold(size: 24)) | |
| .foregroundColor(Double(looked.leftUp)! > 0 ? Color.green: Color.clear) | |
| .background(Double(looked.leftUp)! > 0 ? Color.white: Color.clear) | |
| Text("LD \(looked.leftDown)") | |
| .font(Fonts.avenirNextCondensedBold(size: 24)) | |
| .foregroundColor(Double(looked.leftDown)! > 0 ? Color.green: Color.clear) | |
| .background(Double(looked.leftDown)! > 0 ? Color.white: Color.clear) | |
| Text("RU \(looked.rightUp)") | |
| .font(Fonts.avenirNextCondensedBold(size: 24)) | |
| .foregroundColor(Double(looked.rightUp)! > 0 ? Color.green: Color.clear) | |
| .background(Double(looked.rightUp)! > 0 ? Color.white: Color.clear) | |
| Text("RD \(looked.rightDown)") | |
| .font(Fonts.avenirNextCondensedBold(size: 24)) | |
| .foregroundColor(Double(looked.rightDown)! > 0 ? Color.green: Color.clear) | |
| .background(Double(looked.rightDown)! > 0 ? Color.white: Color.clear) | |
| } | |
| } | |
| VStack { | |
| Text("paused") | |
| .font(Fonts.avenirNextCondensedBold(size: 24)) | |
| .foregroundColor(looked.paused ? Color.white:Color.clear) | |
| .background(looked.paused ? Color.red:Color.clear) | |
| .onTapGesture { | |
| looked.paused.toggle() | |
| } | |
| Spacer() | |
| } | |
| // Circle() | |
| // .stroke(Color.green, lineWidth: 2) | |
| // .frame(width: 16, height: 16) | |
| // .position(looked.lookPoint) | |
| // VStack { | |
| // Circle() | |
| // .fill(looked.lookUp ? Color.white: Color.red) | |
| // .frame(width: 32, height: 32) | |
| // .padding(.bottom,36) | |
| // HStack { | |
| // Circle() | |
| // .fill(looked.lookLeft ? Color.white: Color.red) | |
| // .frame(width: 32, height: 32) | |
| // .padding(.trailing, 36) | |
| // Circle() | |
| // .stroke(Color.white, lineWidth: 8) | |
| // .frame(width: 64, height: 64) | |
| // .padding() | |
| // .onTapGesture { | |
| // print("reste") | |
| // looked.lookUp = false | |
| // looked.lookDown = false | |
| // looked.lookRight = false | |
| // looked.lookLeft = false | |
| // } | |
| // Circle() | |
| // .fill(looked.lookRight ? Color.white: Color.red) | |
| // .frame(width: 32, height: 32) | |
| // .padding(.leading, 36) | |
| // } | |
| // Circle() | |
| // .fill(looked.lookDown ? Color.white: Color.red) | |
| // .frame(width: 32, height: 32) | |
| // .padding(.top,36) | |
| // } | |
| } | |
| } | |
| } | |
| struct CustomARView: UIViewRepresentable { | |
| typealias UIViewType = ARSCNView | |
| var view:ARSCNView | |
| var options: [Any] = [] | |
| func makeUIView(context: Context) -> ARSCNView { | |
| view.session.delegate = context.coordinator | |
| return view | |
| } | |
| func updateUIView(_ view: ARSCNView, context: Context) { | |
| } | |
| func makeCoordinator() -> Coordinator { | |
| Coordinator(self.view) | |
| } | |
| } | |
| class Coordinator: NSObject, ARSCNViewDelegate, ARSessionDelegate { | |
| private var trackingView:ARSCNView | |
| private var sphereNode: SCNNode! | |
| private var cubeNode: SCNNode! | |
| private var satelliteNode: SCNNode! | |
| private var satelliteNodeY: SCNNode! | |
| private var blinked = false | |
| private var config = MLModelConfiguration() | |
| @ObservedObject var looked = Looker.shared | |
| init(_ view: ARSCNView) { | |
| self.trackingView = view | |
| super.init() | |
| guard ARFaceTrackingConfiguration.isSupported else { | |
| fatalError("Face tracking not available on this on this device model!") | |
| } | |
| let connectSession = MultipeerSession.shared | |
| let configuration = ARFaceTrackingConfiguration() | |
| self.trackingView.session.run(configuration) | |
| self.trackingView.delegate = self | |
| let geo3 = SCNSphere(radius: 0.5) | |
| geo3.segmentCount = 16 | |
| sphereNode = SCNNode(geometry: geo3) | |
| sphereNode.geometry?.firstMaterial?.diffuse.contents = UIColor.white.withAlphaComponent(1) | |
| sphereNode.geometry?.firstMaterial?.fillMode = .fill | |
| sphereNode.simdPosition = SIMD3(x: 0, y: 1, z: -6) | |
| self.trackingView.scene.rootNode.addChildNode(sphereNode) | |
| let geo = SCNSphere(radius: 0.5) | |
| satelliteNode = SCNNode(geometry: geo) | |
| satelliteNode.geometry?.firstMaterial?.diffuse.contents = UIColor.blue.withAlphaComponent(1) | |
| satelliteNode.position = SCNVector3(x: 0, y: 0, z: 24) | |
| // satelliteNodeY = SCNNode(geometry: geo) | |
| // satelliteNodeY.geometry?.firstMaterial?.diffuse.contents = UIColor.red.withAlphaComponent(0.8) | |
| // satelliteNodeY.position = SCNVector3(x: 0, y: 0, z: -6) | |
| let colorNames:[UIColor] = [UIColor.red, .blue, .green, .purple, .orange, .brown] | |
| var colorMaterials:[SCNMaterial] = [] | |
| for colors in colorNames { | |
| let newMaterial = SCNMaterial() | |
| newMaterial.diffuse.contents = colors | |
| colorMaterials.append(newMaterial) | |
| } | |
| let geo2 = SCNBox(width: 0.25, height: 0.25, length: 0.25, chamferRadius: 0) | |
| cubeNode = SCNNode(geometry: geo2) | |
| // cubeNode.geometry?.materials = colorMaterials | |
| cubeNode.position = SCNVector3(x: 0, y: 0, z: -10) | |
| cubeNode.geometry?.firstMaterial?.diffuse.contents = UIColor.clear | |
| cubeNode.addChildNode(satelliteNode) | |
| // cubeNode.addChildNode(satelliteNodeY) | |
| self.trackingView.scene.rootNode.addChildNode(cubeNode) | |
| } | |
| var xCords:[Float] = [] | |
| var yCords:[Float] = [] | |
| var spawnTime:TimeInterval = 0 | |
| var lastTime:TimeInterval? = nil | |
| var faceAnchor: ARFaceAnchor? = nil | |
| var changeX:Float = 0 | |
| var changeY:Float = 0 | |
| struct Regard { | |
| let direct:String! | |
| let eyeLookIn_L:Double! | |
| let eyeLookOut_R:Double! | |
| let eyeLookIn_R:Double! | |
| let eyeLookOut_L:Double! | |
| let eyeLookDown_L:Double! | |
| let eyeLookDown_R:Double! | |
| let eyeLookUp_L:Double! | |
| let eyeLookUp_R:Double! | |
| let angleX:Double! | |
| let angleY:Double! | |
| } | |
| var looks:[ARFaceAnchor.BlendShapeLocation:Double] = [:] | |
| var log:[Regard] = [] | |
| func renderer(_ renderer: SCNSceneRenderer, updateAtTime time: TimeInterval) { | |
| lastTime = time | |
| if time > spawnTime { | |
| if faceAnchor != nil && !looked.paused { | |
| DispatchQueue.main.async { [self] in | |
| self.cubeNode.simdOrientation = faceAnchor!.leftEyeTransform.orientation | |
| let angleX = rad2deg(Double(self.cubeNode.eulerAngles.x)) | |
| let angleY = rad2deg(Double(self.cubeNode.eulerAngles.y)) | |
| let eyeLookIn_L = (faceAnchor?.blendShapes[.eyeLookInLeft]!.doubleValue)! | |
| let eyeLookIn_R = (faceAnchor?.blendShapes[.eyeLookInRight]?.doubleValue)! | |
| let eyeLookOut_R = (faceAnchor?.blendShapes[.eyeLookOutRight]?.doubleValue)! | |
| let eyeLookOut_L = (faceAnchor?.blendShapes[.eyeLookOutLeft]?.doubleValue)! | |
| let eyeLookDown_L = (faceAnchor?.blendShapes[.eyeLookDownLeft]?.doubleValue)! | |
| let eyeLookUp_L = (faceAnchor?.blendShapes[.eyeLookUpLeft]?.doubleValue)! | |
| let eyeLookDown_R = (faceAnchor?.blendShapes[.eyeLookDownRight]?.doubleValue)! | |
| let eyeLookUp_R = (faceAnchor?.blendShapes[.eyeLookUpRight]?.doubleValue)! | |
| var modelName:String! | |
| do { | |
| let model = try | |
| eyesClassifier48(configuration: config) | |
| let prediction = try model.prediction(eyeLookIn_L: eyeLookIn_L, eyeLookOut_L: eyeLookOut_L, eyeLookIn_R: eyeLookIn_R, eyeLookOut_R: eyeLookOut_R, eyeLookDown_L: eyeLookDown_L, eyeLookDown_R: eyeLookDown_R, eyeLookUp_L: eyeLookUp_L, eyeLookUp_R: eyeLookUp_L, faceX: 0, faceY: 0) | |
| modelName = prediction.direction | |
| } catch { | |
| print("model error") | |
| } | |
| let eyesLookDown = (eyeLookDown_R + eyeLookUp_R) / 2 | |
| let eyesLookUp = (eyeLookUp_L + eyeLookUp_R) / 2 | |
| let eyesLookLeft = (eyeLookOut_L + eyeLookIn_R) / 2 | |
| let eyesLookRight = (eyeLookOut_R + eyeLookIn_L) / 2 | |
| let eyesLookSide = (eyesLookLeft + eyesLookRight) / 2 | |
| let eyesTrue = (faceAnchor?.blendShapes[.browOuterUpLeft]!.doubleValue)! > 0.1 | |
| let talk = Speaking() | |
| if eyesTrue { | |
| let foo = log.filter({$0.direct == "eyesLeft"}) | |
| let foo2 = foo.filter({$0.eyeLookIn_L > 0}) | |
| let minValue = foo2.min { $0.eyeLookIn_L < $1.eyeLookIn_L }?.eyeLookIn_L | |
| let maxValue = foo2.max { $0.eyeLookIn_L < $1.eyeLookIn_L }?.eyeLookIn_L | |
| let sortedValues = foo2.sorted(by: { $0.eyeLookIn_L > $1.eyeLookIn_L} ) | |
| for foos in foo2 { | |
| print("\(foos.eyeLookIn_L!)") | |
| } | |
| print("lookLeft \(minValue) \(maxValue)") | |
| } | |
| let angle = rad2deg(Double(faceAnchor!.transform.orientation.angle.binade)) | |
| let formatter = NumberFormatter() | |
| formatter.numberStyle = .decimal | |
| formatter.maximumFractionDigits = 2 | |
| // | |
| let number = NSNumber(value: angleY) | |
| let formattedValue = formatter.string(from: number)! | |
| looked.gazeY = formattedValue.description | |
| let number2 = NSNumber(value: angleX) | |
| let formattedValue2 = formatter.string(from: number2)! | |
| looked.gazeX = formattedValue2.description | |
| let number3 = NSNumber(value: angle) | |
| let formattedValue3 = formatter.string(from: number3)! | |
| looked.gazeA = formattedValue3.description | |
| var divisor = 0.3 | |
| var height:Float = 0.5 | |
| // if eyesLookLeft < 0.1 && eyesLookRight < 0.1 && eyesLookDown < 0.1 && eyesLookUp < 0.1 { | |
| // sphereNode.simdPosition = SIMD3(x: 0, y: height, z: -6) | |
| // sphereNode.geometry?.firstMaterial?.diffuse.contents = UIColor.green.withAlphaComponent(0.8) | |
| // //looked.gazeA = "" | |
| // return | |
| // } | |
| if angle < 8 && !looked.paused { | |
| looked.outOfRange = false | |
| switch modelName { | |
| case "eyesLeft": | |
| //talk.speaker(words: ["left"]) | |
| sphereNode.simdPosition = SIMD3(x: -1, y: height, z: -6) | |
| sphereNode.geometry?.firstMaterial?.diffuse.contents = UIColor.blue.withAlphaComponent(1) | |
| //looked.paused = true | |
| logEyes(look: "eyesLeft") | |
| case "eyesRight": | |
| //talk.speaker(words: ["right"]) | |
| sphereNode.simdPosition = SIMD3(x: 1, y: height, z: -6) | |
| sphereNode.geometry?.firstMaterial?.diffuse.contents = UIColor.blue.withAlphaComponent(1) | |
| //looked.paused = true | |
| logEyes(look: "eyesRight") | |
| case "eyesDown": | |
| //talk.speaker(words: ["down"]) | |
| sphereNode.simdPosition = SIMD3(x: 0, y: height - 1, z: -6) | |
| sphereNode.geometry?.firstMaterial?.diffuse.contents = UIColor.red.withAlphaComponent(1) | |
| //looked.paused = true | |
| logEyes(look: "eyesDown") | |
| case "eyesUp": | |
| //talk.speaker(words: ["up"]) | |
| sphereNode.simdPosition = SIMD3(x: 0, y: height + 1, z: -6) | |
| sphereNode.geometry?.firstMaterial?.diffuse.contents = UIColor.red.withAlphaComponent(1) | |
| //looked.paused = true | |
| logEyes(look: "eyesUp") | |
| case "eyesLeftUp": | |
| //talk.speaker(words: ["high left"]) | |
| sphereNode.simdPosition = SIMD3(x: -1, y: height + 1, z: -6) | |
| sphereNode.geometry?.firstMaterial?.diffuse.contents = UIColor.orange.withAlphaComponent(1) | |
| //looked.paused = true | |
| logEyes(look: "eyesLeftUp") | |
| case "eyesRightUp": | |
| //talk.speaker(words: ["high right"]) | |
| sphereNode.simdPosition = SIMD3(x: 1, y: height + 1, z: -6) | |
| sphereNode.geometry?.firstMaterial?.diffuse.contents = UIColor.orange.withAlphaComponent(1) | |
| //looked.paused = true | |
| logEyes(look: "eyesRightUp") | |
| case "eyesLeftDown": | |
| //talk.speaker(words: ["low left"]) | |
| sphereNode.simdPosition = SIMD3(x: -1, y: height - 1, z: -6) | |
| sphereNode.geometry?.firstMaterial?.diffuse.contents = UIColor.yellow.withAlphaComponent(1) | |
| //looked.paused = true | |
| logEyes(look: "eyesLeftDown") | |
| case "eyesRightDown": | |
| //talk.speaker(words: ["low right"]) | |
| sphereNode.simdPosition = SIMD3(x: 1, y: height - 1, z: -6) | |
| sphereNode.geometry?.firstMaterial?.diffuse.contents = UIColor.yellow.withAlphaComponent(1) | |
| //looked.paused = true | |
| logEyes(look: "eyesRightDown") | |
| default: | |
| sphereNode.simdPosition = SIMD3(x: 0, y: height, z: -6) | |
| sphereNode.geometry?.firstMaterial?.diffuse.contents = UIColor.green.withAlphaComponent(0.8) | |
| logEyes(look: "eyesCentre") | |
| } | |
| // if (eyesLookDown > 0.09) || (eyesLookUp > 0.09) { | |
| // if eyesLookDown > 0.2 && eyesLookLeft > 0.2 { | |
| // talk.speaker(words: ["southEast"]) | |
| // sphereNode.simdPosition = SIMD3(x: 1, y: height - 1, z: -6) | |
| // sphereNode.geometry?.firstMaterial?.diffuse.contents = UIColor.yellow.withAlphaComponent(1) | |
| // looked.paused = true | |
| // } | |
| // if eyesLookDown > 0.2 && eyesLookRight > 0.2 { | |
| // talk.speaker(words: ["southWest"]) | |
| // sphereNode.simdPosition = SIMD3(x: -1, y: height - 1, z: -6) | |
| // sphereNode.geometry?.firstMaterial?.diffuse.contents = UIColor.orange.withAlphaComponent(1) | |
| // looked.paused = true | |
| // } | |
| // if eyesLookDown > 0.2 && eyesLookSide < 0.3 && eyesLookUp < 0.1 { | |
| // talk.speaker(words: ["south"]) | |
| // sphereNode.simdPosition = SIMD3(x: 0, y: height - 1, z: -6) | |
| // sphereNode.geometry?.firstMaterial?.diffuse.contents = UIColor.red.withAlphaComponent(1) | |
| // looked.paused = true | |
| // } | |
| // if eyesLookUp > 0.2 && eyesLookRight > 0.2 { | |
| // talk.speaker(words: ["northEast"]) | |
| // sphereNode.simdPosition = SIMD3(x: 1, y: height + 1, z: -6) | |
| // sphereNode.geometry?.firstMaterial?.diffuse.contents = UIColor.yellow.withAlphaComponent(1) | |
| // looked.paused = true | |
| // } | |
| // if eyesLookUp > 0.2 && eyesLookLeft > 0.2 { | |
| // talk.speaker(words: ["northWest"]) | |
| // sphereNode.simdPosition = SIMD3(x: -1, y: height + 1, z: -6) | |
| // sphereNode.geometry?.firstMaterial?.diffuse.contents = UIColor.orange.withAlphaComponent(1) | |
| // looked.paused = true | |
| // } | |
| // if eyesLookUp > 0.2 && eyesLookSide < 0.3 && eyesLookDown < 0.1 { | |
| // talk.speaker(words: ["north"]) | |
| // sphereNode.simdPosition = SIMD3(x: 0, y: height + 1, z: -6) | |
| // sphereNode.geometry?.firstMaterial?.diffuse.contents = UIColor.red.withAlphaComponent(1) | |
| // looked.paused = true | |
| // } | |
| // } | |
| // if eyesLookLeft > 0.2 && eyesLookUp < 0.1 && eyesLookDown < 0.1 { | |
| // talk.speaker(words: ["east"]) | |
| // sphereNode.simdPosition = SIMD3(x: 1, y: height, z: -6) | |
| // sphereNode.geometry?.firstMaterial?.diffuse.contents = UIColor.blue.withAlphaComponent(1) | |
| // looked.paused = true | |
| // } | |
| // if eyesLookRight > 0.2 && eyesLookUp < 0.1 && eyesLookDown < 0.1 { | |
| // talk.speaker(words: ["west"]) | |
| // sphereNode.simdPosition = SIMD3(x: -1, y: height, z: -6) | |
| // sphereNode.geometry?.firstMaterial?.diffuse.contents = UIColor.blue.withAlphaComponent(1) | |
| // looked.paused = true | |
| // | |
| // } | |
| } else { | |
| looked.outOfRange = true | |
| } | |
| // let newX = (satelliteNode.worldPosition.x * 100).rounded() / 100 | |
| // let newY = (satelliteNode.worldPosition.y * 100).rounded() / 100 | |
| // | |
| // let formatter = NumberFormatter() | |
| // formatter.numberStyle = .decimal | |
| // formatter.maximumFractionDigits = 2 | |
| // | |
| // let number = NSNumber(value: newX) | |
| // let formattedValue = formatter.string(from: number)! | |
| // | |
| // let number2 = NSNumber(value: newY) | |
| // let formattedValue2 = formatter.string(from: number2)! | |
| // | |
| // looked.gazeX = formattedValue | |
| // looked.gazeY = formattedValue2 | |
| // if newX < 1.5 && newY < 1.5 { | |
| // sphereNode.simdPosition = SIMD3(x: 0, y: 0, z: -6) | |
| // sphereNode.geometry?.firstMaterial?.diffuse.contents = UIColor.red.withAlphaComponent(0.90) | |
| // } | |
| // if (newX > 0.7 || newX < -0.7) { | |
| // changeX = newX > 0 ? 2.0 : -2.0 | |
| // sphereNode.simdPosition = SIMD3(x: changeX, y: 0, z: -6) | |
| // sphereNode.geometry?.firstMaterial?.diffuse.contents = UIColor.blue.withAlphaComponent(0.90) | |
| // } | |
| // if (newY > 0.45 || newY < -0.9) { | |
| // changeY = newY > 0 ? 1.0 : -1.0 | |
| // sphereNode.simdPosition = SIMD3(x: 0, y: changeY, z: -6) | |
| // sphereNode.geometry?.firstMaterial?.diffuse.§ contents = UIColor.green.withAlphaComponent(0.90) | |
| // } | |
| } | |
| } | |
| spawnTime = time + TimeInterval(0.1) | |
| showFaces() | |
| } | |
| func logEyes(look:String) { | |
| let eyeLookIn_L = faceAnchor?.blendShapes[.eyeLookInLeft]?.doubleValue | |
| let eyeLookIn_R = faceAnchor?.blendShapes[.eyeLookInRight]?.doubleValue | |
| let eyeLookOut_R = faceAnchor?.blendShapes[.eyeLookOutRight]?.doubleValue | |
| let eyeLookOut_L = faceAnchor?.blendShapes[.eyeLookOutLeft]?.doubleValue | |
| let eyeLookDown_L = faceAnchor?.blendShapes[.eyeLookDownLeft]?.doubleValue | |
| let eyeLookUp_L = faceAnchor?.blendShapes[.eyeLookUpLeft]?.doubleValue | |
| let eyeLookDown_R = faceAnchor?.blendShapes[.eyeLookDownRight]?.doubleValue | |
| let eyeLookUp_R = faceAnchor?.blendShapes[.eyeLookUpRight]?.doubleValue | |
| let angleX = rad2deg(Double(self.cubeNode.eulerAngles.x)) | |
| let angleY = rad2deg(Double(self.cubeNode.eulerAngles.y)) | |
| let rec = Regard(direct: look, eyeLookIn_L: eyeLookIn_L, eyeLookOut_R: eyeLookOut_R, eyeLookIn_R: eyeLookIn_R, eyeLookOut_L: eyeLookOut_L, eyeLookDown_L: eyeLookDown_L, eyeLookDown_R: eyeLookDown_R, eyeLookUp_L: eyeLookUp_L, eyeLookUp_R: eyeLookUp_R, angleX: angleX, angleY: angleY) | |
| log.append(rec) | |
| } | |
| } | |
| func return2D(figure: Double) -> String { | |
| let formatter = NumberFormatter() | |
| formatter.numberStyle = .decimal | |
| formatter.maximumFractionDigits = 2 | |
| // | |
| let number = NSNumber(value: figure) | |
| let formattedValue = formatter.string(from: number)! | |
| return formattedValue | |
| } | |
| func showFaces() { | |
| for faces in [ARFaceAnchor.BlendShapeLocation.eyeLookOutRight, .eyeLookOutLeft, .eyeLookInRight, .eyeLookInLeft, .eyeLookUpLeft, .eyeLookUpRight, .eyeLookDownRight, .eyeLookDownLeft] { | |
| //print("looks \(faces) \(looks[faces])") | |
| } | |
| // if looks[.cheekPuff] != 0 { | |
| // DispatchQueue.main.async { | |
| // self.looked.paused = true | |
| // print("paused true") | |
| // } | |
| // } | |
| // if looks[.cheekPuff] != 0 { | |
| // DispatchQueue.main.async { | |
| // self.looked.paused = false | |
| // print("paused false") | |
| // } | |
| // } | |
| DispatchQueue.main.async { | |
| if !self.looked.outOfRange && !self.looked.paused { | |
| if self.looks[.eyeLookInLeft] != nil { | |
| self.looked.leftIn = self.return2D(figure: self.looks[.eyeLookInLeft] ?? 0) | |
| self.looked.leftOut = self.return2D(figure: self.looks[.eyeLookOutLeft] ?? 0) | |
| self.looked.rightIn = self.return2D(figure: self.looks[.eyeLookInRight] ?? 0) | |
| self.looked.rightOut = self.return2D(figure: self.looks[.eyeLookOutRight] ?? 0) | |
| self.looked.leftDown = self.return2D(figure: self.looks[.eyeLookDownLeft] ?? 0) | |
| self.looked.leftUp = self.return2D(figure: self.looks[.eyeLookUpLeft] ?? 0) | |
| self.looked.rightDown = self.return2D(figure: self.looks[.eyeLookDownRight] ?? 0) | |
| self.looked.rightUp = self.return2D(figure: self.looks[.eyeLookUpRight] ?? 0) | |
| } | |
| } | |
| } | |
| } | |
| func renderer(_ renderer: SCNSceneRenderer, didUpdate node: SCNNode, for anchor: ARAnchor) { | |
| guard let faceAnchor = anchor as? ARFaceAnchor else { return } | |
| self.faceAnchor = faceAnchor | |
| looks.removeAll() | |
| for faces in faceAnchor.blendShapes { | |
| if looks[faces.key] != nil { | |
| looks[faces.key]! = 0 | |
| } else { | |
| looks[faces.key] = faceAnchor.blendShapes[faces.key]!.doubleValue | |
| } | |
| } | |
| } | |
| func dummy(_ anchor: ARAnchor) { | |
| guard let faceAnchor = anchor as? ARFaceAnchor else { return } | |
| DispatchQueue.main.async { [self] in | |
| self.cubeNode.simdOrientation = faceAnchor.leftEyeTransform.orientation | |
| // self.cubeNode.simdPosition = faceAnchor.leftEyeTransform.translation | |
| // self.cubeNode.simdPosition.z = -10 | |
| // let foo = rad2deg(Double(quatToEulerAngles(self.cubeNode.simdOrientation).y)) | |
| //print("simdOrientation \(foo) \(satelliteNode.simdPosition.x) lnik \(link)") | |
| // if (foo > 2) && (foo < 6) { | |
| // SCNTransaction.begin() | |
| // SCNTransaction.animationDuration = 1.0 | |
| // sphereNode.simdPosition = SIMD3(x: -1, y: 1, z: -6) | |
| // SCNTransaction.commit() | |
| // } | |
| // if (foo > 6) && (foo < 12) { | |
| // SCNTransaction.begin() | |
| // SCNTransaction.animationDuration = 1.0 | |
| // sphereNode.simdPosition = SIMD3(x: -2, y: 1, z: -6) | |
| // SCNTransaction.commit() | |
| // } | |
| // if foo > 18 && foo < 24 { | |
| // SCNTransaction.begin() | |
| // SCNTransaction.animationDuration = 1.0 | |
| // sphereNode.simdPosition = SIMD3(x: -3, y: 1, z: -6) | |
| // SCNTransaction.commit() | |
| // } | |
| // if foo < -2 && foo > -6 { | |
| // SCNTransaction.begin() | |
| // SCNTransaction.animationDuration = 1.0 | |
| // sphereNode.simdPosition = SIMD3(x: 1, y: 1, z: -6) | |
| // SCNTransaction.commit() | |
| // } | |
| // if foo < -6 && foo > -12 { | |
| // SCNTransaction.begin() | |
| // SCNTransaction.animationDuration = 1.0 | |
| // sphereNode.simdPosition = SIMD3(x: 2, y: 1, z: -6) | |
| // SCNTransaction.commit() | |
| // } | |
| // if foo < -12 && foo > -24 { | |
| // SCNTransaction.begin() | |
| // SCNTransaction.animationDuration = 1.0 | |
| // sphereNode.simdPosition = SIMD3(x: 3, y: 1, z: -6) | |
| // SCNTransaction.commit() | |
| // } | |
| // if foo > -3 && foo < 3 { | |
| // SCNTransaction.begin() | |
| // SCNTransaction.animationDuration = 1.0 | |
| // sphereNode.simdPosition = SIMD3(x: 0, y: 1, z: -6) | |
| // SCNTransaction.commit() | |
| // } | |
| // print("BlendShapes ",faceAnchor.blendShapes.count) | |
| if faceAnchor.blendShapes[.eyeLookInLeft]!.doubleValue < 0.3 && | |
| faceAnchor.blendShapes[.eyeLookInRight]!.doubleValue < 0.3 && | |
| faceAnchor.blendShapes[.eyeLookUpLeft]!.doubleValue < 0.3 && | |
| faceAnchor.blendShapes[.eyeLookDownLeft]!.doubleValue < 0.3 { | |
| SCNTransaction.begin() | |
| SCNTransaction.animationDuration = 1.0 | |
| //sphereNode.simdPosition = SIMD3(x: 0, y: 0, z: -6) | |
| SCNTransaction.commit() | |
| } | |
| if faceAnchor.blendShapes[.eyeLookInLeft]!.doubleValue > 0.3 { | |
| //self.looked.lookRight = true | |
| SCNTransaction.begin() | |
| SCNTransaction.animationDuration = 1.0 | |
| //sphereNode.simdPosition = SIMD3(x: -3, y: 0, z: -6) | |
| SCNTransaction.commit() | |
| } | |
| if faceAnchor.blendShapes[.eyeLookInRight]!.doubleValue > 0.3 { | |
| //self.looked.lookLeft = true | |
| SCNTransaction.begin() | |
| SCNTransaction.animationDuration = 1.0 | |
| //sphereNode.simdPosition = SIMD3(x: 3, y: 0, z: -6) | |
| SCNTransaction.commit() | |
| } | |
| if faceAnchor.blendShapes[.eyeLookUpLeft]!.doubleValue > 0.3 { | |
| //self.looked.lookUp = true | |
| SCNTransaction.begin() | |
| SCNTransaction.animationDuration = 1.0 | |
| //sphereNode.simdPosition = SIMD3(x: 0, y: 1, z: -6) | |
| SCNTransaction.commit() | |
| } | |
| if faceAnchor.blendShapes[.eyeLookDownLeft]!.doubleValue > 0.3 { | |
| //self.looked.lookDown = true | |
| SCNTransaction.begin() | |
| SCNTransaction.animationDuration = 1.0 | |
| //sphereNode.simdPosition = SIMD3(x: 0, y: -1, z: -6) | |
| SCNTransaction.commit() | |
| } | |
| if faceAnchor.blendShapes[.eyeLookInLeft]!.doubleValue > 0.2 && | |
| faceAnchor.blendShapes[.eyeLookDownLeft]!.doubleValue > 0.2 { | |
| //self.looked.lookRight = true | |
| //sphereNode.simdPosition = SIMD3(x: -2, y: -1, z: -6) | |
| } | |
| if faceAnchor.blendShapes[.eyeLookInLeft]!.doubleValue > 0.2 && | |
| faceAnchor.blendShapes[.eyeLookUpLeft]!.doubleValue > 0.2 { | |
| //self.looked.lookLeft = true | |
| //sphereNode.simdPosition = SIMD3(x: -2, y: 1, z: -6) | |
| } | |
| if faceAnchor.blendShapes[.eyeLookInRight]!.doubleValue > 0.2 && | |
| faceAnchor.blendShapes[.eyeLookDownLeft]!.doubleValue > 0.2 { | |
| //self.looked.lookDown = true | |
| //sphereNode.simdPosition = SIMD3(x: 2, y: -1, z: -6) | |
| } | |
| if faceAnchor.blendShapes[.eyeLookInRight]!.doubleValue > 0.2 && | |
| faceAnchor.blendShapes[.eyeLookUpLeft]!.doubleValue > 0.2 { | |
| //self.looked.lookUp = true | |
| //sphereNode.simdPosition = SIMD3(x: 2, y: 1, z: -6) | |
| } | |
| //var newX = Float(faceAnchor.lookAtPoint.x) * 8 | |
| //let newY = Float(faceAnchor.lookAtPoint.y) * 4 | |
| //let newXR = (newX * 100).rounded() / 100 | |
| //let newYR = (newY * 100).rounded() / 100 | |
| if faceAnchor.blendShapes[.cheekPuff]!.doubleValue > 0.5 { | |
| blinked.toggle() | |
| //print("blink") | |
| } | |
| let newX = (satelliteNode.worldPosition.x * 100).rounded() / 100 | |
| let newY = (satelliteNodeY.worldPosition.y * 100).rounded() / 100 | |
| let newXR = (newX * 100).rounded() / 100 | |
| let newYR = (newY * 100).rounded() / 100 | |
| xCords.append(newX) | |
| yCords.append(newY) | |
| //print("satX \(blinked) \(xCords.min()) \(xCords.max()) satY \(yCords.min()) \(yCords.max()) X \(newX) Y \(newY)") | |
| SCNTransaction.begin() | |
| SCNTransaction.animationDuration = 1 | |
| //sphereNode.simdPosition = SIMD3(x: newX, y: newY, z: -6) | |
| if (newX > 1.0 && newX < -1.0) && blinked { | |
| sphereNode.simdPosition = SIMD3(x: newX, y: newY, z: -6) | |
| } | |
| if (newY > 1.0 || newY < -1.0) && blinked { | |
| sphereNode.simdPosition = SIMD3(x: newX, y: newY, z: -6) | |
| } | |
| if (newX > 1.0 || newX < -1.0) && !blinked { | |
| sphereNode.simdPosition = SIMD3(x: newX, y: 0, z: -6) | |
| } | |
| if (newY > 1.0 || newY < -1.0) && !blinked { | |
| sphereNode.simdPosition = SIMD3(x: 0, y: newY, z: -6) | |
| } | |
| SCNTransaction.commit() | |
| // self.looked.lookPoint.x += CGFloat(faceAnchor.lookAtPoint.x) * -1 | |
| // self.looked.lookPoint.y += CGFloat(faceAnchor.lookAtPoint.y) * 1 | |
| // | |
| // if self.sphereNode.simdPosition.x > -2 && self.sphereNode.simdPosition.x < 2 { | |
| // self.sphereNode.simdPosition.x += faceAnchor.lookAtPoint.x * 0.25 | |
| // } else { | |
| // if self.sphereNode.simdPosition.x > 0 { | |
| // self.sphereNode.simdPosition.x = 1.99 | |
| // } else { | |
| // self.sphereNode.simdPosition.x = -1.99 | |
| // } | |
| // } | |
| // if self.sphereNode.simdPosition.y > -2 { | |
| // self.sphereNode.simdPosition.y += faceAnchor.lookAtPoint.y * 0.25 | |
| // } else { | |
| // self.sphereNode.simdPosition.y = -1.99 | |
| // } | |
| } | |
| } | |
| func quatToEulerAngles(_ quat: simd_quatf) -> SIMD3<Float>{ | |
| let n = SCNNode() | |
| n.simdOrientation = quat | |
| return n.simdEulerAngles | |
| } | |
| func rad2deg(_ number: Double) -> Double { | |
| return number * 180 / .pi | |
| } | |
| } | |
| extension float4x4 { | |
| var translation: SIMD3<Float> { | |
| let translation = columns.3 | |
| return SIMD3<Float>(translation.x, translation.y, translation.z) | |
| } | |
| /** | |
| Factors out the orientation component of the transform. | |
| */ | |
| var orientation: simd_quatf { | |
| return simd_quaternion(self) | |
| } | |
| } | |
| struct ContentView_Previews: PreviewProvider { | |
| static var previews: some View { | |
| ContentView() | |
| } | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment