Interagir sur des Géométries Dynamo avec les mains grâce à un contrôleur UltraLeap
#DynamoBIM #Python #LeapMotion #UltraLeap #AutodeskExpertElite #AutodeskCommunity 
if this article is not in your language, use the Google Translate
widget ⬈ (bottom of page for Mobile version ⬇)
Contexte
Possédant un contrôleur Leap Motion, que j'avais autrefois utilisé avec l’un des premiers casques de réalité virtuelle (Oculus DK2), j’ai toujours rêvé d’interagir avec DynamoBIM à l’aide de gestes.
J’avais déjà tenté l’expérience par le passé, mais sans succès – principalement par manque de connaissances techniques à l’époque. Récemment, j’ai décidé de réessayer… et cette fois, je suis allé beaucoup plus loin !
Prérequis
Pour reproduire cette expérience et connecter le Leap Motion à DynamoBIM, voici ce dont vous aurez besoin :- DynamoBIM
-
Python3.xx installé en local
(même version que celle utilisée par le moteur Python de Dynamo)
Dans cet exemple, nous utiliserons le moteur PythonNet3 qui est actuellement sous Python3.11
- Un IDE tel que PyCharm (ou autre) avec Git installé
- Un contrôleur Leap Motion (ou UltraLeap)
🖐️ Mon setup personnel :
- Contrôleur Leap Motion
- Logiciel de tracking associé : Leap Gemini v5.20.0
Résultat
Procédure
1. Installer le SDK UltraLeap
"C:\Program Files\Ultraleap\ControlPanel\Ultraleap Tray.exe"
pour vérifier le tracking
2. Lancer PyCharm et cloner le repo Github suivant
-
-
Fonctionnalités rajoutées :
-
"C:\Program Files\Ultraleap\ControlPanel\Ultraleap Tray.exe"
pour vérifier le tracking
2. Lancer PyCharm et cloner le repo Github suivant
https://github.com/ultraleap/leapc-python-bindings
-
3. Définir la version de Python
La version doit être identique au moteur Python que vous allez utiliser sous Dynamo
-
4. Construire la solution
suivre les instructions du README pour construire un nouveau
module (package whl)
5. Copie des librairies
Important : le fichier .whl est à dézipper dans le dossier
"C:\Users\<Utilisateur>\AppData\Local\python-3.11.0-embed-amd64\Lib\"
6. Présentation du script Dynamo
Il se compose de 2 nœuds Python :
5. Copie des librairies
Une fois la solution compilée, on copie les librairies dans le
dossier Lib Python de Dynamo
Exemple du chemin du dossier Lib
"C:\Users\<Utilisateur>\AppData\Local\python-3.11.0-embed-amd64\Lib\"
-
Important : le fichier .whl est à dézipper dans le dossier
"C:\Users\<Utilisateur>\AppData\Local\python-3.11.0-embed-amd64\Lib\"
6. Présentation du script Dynamo
Il se compose de 2 nœuds Python :
- Un pour créer une instance de connexion
-
Un pour tracker les mains/doigts et les transformer en géométries
Dynamo
Note : Les coordonnées sont celles du LeapMotion, il faudra prochainement que je regarde pour corriger le système de coordonnées
- création de solide de type Cuboid par pincement entre les 2 mains
- sauvegarde des solides en mémoires
Nous rajoutons un nœud
-
DateTime.Now pour pouvoir utiliser le mode periodique
-
code 1
.
# Load the Python Standard and DesignScript Libraries
import sys
import clr
import System
clr.AddReference('ProtoGeometry')
from Autodesk.DesignScript.Geometry import *
import Autodesk.DesignScript.Geometry as DS
import sysconfig
# standard library path
sys.path.append(sysconfig.get_path("platstdlib"))
# site-package library path
sys.path.append(sysconfig.get_path("platlib"))
import leap
import time
connection = leap.Connection()
OUT = connection
code 2
.
# Load the Python Standard and DesignScript Libraries
import sys
import clr
import System
clr.AddReference('ProtoGeometry')
import Autodesk.DesignScript.Geometry as DS
import sysconfig
# standard library path
sys.path.append(sysconfig.get_path("platstdlib"))
# site-package library path
sys.path.append(sysconfig.get_path("platlib"))
import leap
from leap import datatypes as ldt
import time
# print variables in memory
from io import StringIO
sys.stdout = StringIO()
class MyListener(leap.Listener):
def __init__(self):
self.hands_format = "Skeleton"
self.lst_points = []
self.radius = 2
def on_connection_event(self, event):
pass
#print("Connected")
def on_device_event(self, event):
try:
with event.device.open():
info = event.device.get_info()
except leap.LeapCannotOpenDeviceError:
info = event.device.get_info()
#print(f"Found device {info.serial}")
def location_end_of_finger(self, hand: ldt.Hand, digit_idx: int) -> ldt.Vector:
digit = hand.digits[digit_idx]
return digit.distal.next_joint
def sub_vectors(self, v1: ldt.Vector, v2: ldt.Vector) -> list:
return map(float.__sub__, v1, v2)
def fingers_pinching(self, thumb: ldt.Vector, index: ldt.Vector):
diff = list(map(abs, self.sub_vectors(thumb, index)))
if diff[0] < 20 and diff[1] < 20 and diff[2] < 20:
return True, diff
else:
return False, diff
def get_joint_position(self, bone):
if bone:
return bone.x , bone.y , bone.z
else:
return None
def on_tracking_event(self, event):
geometries = []
pinch_pts = []
pinch_hand_type = set()
#print(f"Frame {event.tracking_frame_id} with {len(event.hands)} hands.")
for i in range(0, len(event.hands)):
hand = event.hands[i]
hand_type = "left" if str(hand.type) == "HandType.Left" else "right"
for index_digit in range(0, 5):
digit = hand.digits[index_digit]
for index_bone in range(0, 4):
bone = digit.bones[index_bone]
if self.hands_format == "Skeleton":
#print("Skeleton")
wrist = self.get_joint_position(hand.arm.next_joint)
elbow = self.get_joint_position(hand.arm.prev_joint)
if wrist:
#print("wrist")
pta = DS.Point.ByCoordinates(*wrist)
geometries.append(DS.Sphere.ByCenterPointRadius(pta, self.radius ))
if elbow:
#print("elbow")
ptb = DS.Point.ByCoordinates(*elbow)
geometries.append(DS.Sphere.ByCenterPointRadius(ptb, self.radius ))
if wrist and elbow:
#print("wrist and elbow")
try:
geometries.append(DS.Line.ByStartPointEndPoint(pta, ptb))
except Exception as ex:
pass
bone_start = self.get_joint_position(bone.prev_joint)
bone_end = self.get_joint_position(bone.next_joint)
#print(f"{bone_start=}")
#print(f"{bone_end=}")
ptc = DS.Point.ByCoordinates(*bone_start)
geometries.append(DS.Sphere.ByCenterPointRadius(ptc, self.radius ))
ptd = DS.Point.ByCoordinates(*bone_end)
geometries.append(DS.Sphere.ByCenterPointRadius(ptd, self.radius ))
try:
geometries.append(DS.Line.ByStartPointEndPoint(ptc, ptd))
except Exception as ex:
pass
#
# test if pinching
thumb = self.location_end_of_finger(hand, 0)
index = self.location_end_of_finger(hand, 1)
pinching, array = self.fingers_pinching(thumb, index)
#print(f"{pinching=}", array)
if pinching:
pinch_pts.append(DS.Point.ByCoordinates(*array))
pinch_hand_type.add(hand_type)
#
if len(pinch_hand_type) == 2 and len(geometries) >= 141:
bbx = DS.BoundingBox.ByGeometry([geometries[21], geometries[141]])
bbx_solid = DS.BoundingBox.ToCuboid(bbx)
geometries.insert(0, bbx_solid)
#
if len(geometries) > 0:
self.lst_points.append(geometries)
def main():
global connection
my_listener = MyListener()
#connection = leap.Connection()
connection.add_listener(my_listener)
with connection.open():
connection.set_tracking_mode(leap.TrackingMode.Desktop)
time.sleep(0.1)
connection.remove_listener(my_listener)
if len(my_listener.lst_points) > 0:
return my_listener.lst_points[-1]
else:
return []
connection = IN[0]
geoms = main()
sys.stdout.seek(0)
printResult = sys.stdout.read()
OUT = geoms, printResult
# reset the sys.stdout
sys.stdout = sys.__stdout__
sys.stderr = sys.__stderr__
# Load the Python Standard and DesignScript Libraries
import sys
import clr
import System
clr.AddReference('ProtoGeometry')
import Autodesk.DesignScript.Geometry as DS
import sysconfig
# standard library path
sys.path.append(sysconfig.get_path("platstdlib"))
# site-package library path
sys.path.append(sysconfig.get_path("platlib"))
import leap
from leap import datatypes as ldt
import time
from io import StringIO
sys.stdout = StringIO()
class MyListener(leap.Listener):
def __init__(self):
self.hands_format = "Skeleton"
self.lst_points = []
self.lst_cuboid = []
self.radius = 2
def on_connection_event(self, event):
pass
#print("Connected")
def on_device_event(self, event):
try:
with event.device.open():
info = event.device.get_info()
except leap.LeapCannotOpenDeviceError:
info = event.device.get_info()
#print(f"Found device {info.serial}")
def location_end_of_finger(self, hand: ldt.Hand, digit_idx: int) -> ldt.Vector:
digit = hand.digits[digit_idx]
return digit.distal.next_joint
def sub_vectors(self, v1: ldt.Vector, v2: ldt.Vector) -> list:
return map(float.__sub__, v1, v2)
def fingers_pinching(self, thumb: ldt.Vector, index: ldt.Vector):
diff = list(map(abs, self.sub_vectors(thumb, index)))
if diff[0] < 20 and diff[1] < 20 and diff[2] < 20:
return True, diff
else:
return False, diff
def get_joint_position(self, bone):
if bone:
#return int(bone.x + (self.screen_size[1] / 2)), int(bone.z + (self.screen_size[0] / 2))
return bone.x , bone.y , bone.z
else:
return None
def on_tracking_event(self, event):
geometries = []
pinch_pts = []
pinch_hand_type = set()
#print(f"Frame {event.tracking_frame_id} with {len(event.hands)} hands.")
for i in range(0, len(event.hands)):
hand = event.hands[i]
hand_type = "left" if str(hand.type) == "HandType.Left" else "right"
for index_digit in range(0, 5):
digit = hand.digits[index_digit]
for index_bone in range(0, 4):
bone = digit.bones[index_bone]
if self.hands_format == "Skeleton":
#print("Skeleton")
wrist = self.get_joint_position(hand.arm.next_joint)
elbow = self.get_joint_position(hand.arm.prev_joint)
if wrist:
#print("wrist")
pta = DS.Point.ByCoordinates(*wrist)
geometries.append(DS.Sphere.ByCenterPointRadius(pta, self.radius ))
if elbow:
#print("elbow")
ptb = DS.Point.ByCoordinates(*elbow)
geometries.append(DS.Sphere.ByCenterPointRadius(ptb, self.radius ))
if wrist and elbow:
#print("wrist and elbow")
try:
geometries.append(DS.Line.ByStartPointEndPoint(pta, ptb))
except Exception as ex:
pass
bone_start = self.get_joint_position(bone.prev_joint)
bone_end = self.get_joint_position(bone.next_joint)
#print(f"{bone_start=}")
#print(f"{bone_end=}")
ptc = DS.Point.ByCoordinates(*bone_start)
geometries.append(DS.Sphere.ByCenterPointRadius(ptc, self.radius ))
ptd = DS.Point.ByCoordinates(*bone_end)
geometries.append(DS.Sphere.ByCenterPointRadius(ptd, self.radius ))
try:
geometries.append(DS.Line.ByStartPointEndPoint(ptc, ptd))
except Exception as ex:
pass
#
# test if pinching
thumb = self.location_end_of_finger(hand, 0)
index = self.location_end_of_finger(hand, 1)
pinching, array = self.fingers_pinching(thumb, index)
#print(f"{pinching=}", array)
if pinching:
pinch_pts.append(DS.Point.ByCoordinates(*array))
pinch_hand_type.add(hand_type)
#
if len(pinch_hand_type) == 2 and len(geometries) >= 141:
bbx = DS.BoundingBox.ByGeometry([geometries[21], geometries[141]])
bbx_solid = DS.BoundingBox.ToCuboid(bbx)
self.lst_cuboid.append(bbx_solid)
#
if len(geometries) > 0:
self.lst_points.append(geometries)
def main():
global connection
my_listener = MyListener()
#connection = leap.Connection()
connection.add_listener(my_listener)
with connection.open():
connection.set_tracking_mode(leap.TrackingMode.Desktop)
time.sleep(0.1)
connection.remove_listener(my_listener)
if len(my_listener.lst_points) > 0 and len(my_listener.lst_cuboid) > 0:
return my_listener.lst_points[-1], my_listener.lst_cuboid[-1]
elif len(my_listener.lst_points) > 0 and len(my_listener.lst_cuboid) == 0:
return my_listener.lst_points[-1], None
else:
return [], None
connection = IN[0]
geoms, new_cuboid = main()
datam = []
dataKey = "LstGeoms"
if len(geoms) == 0:
System.AppDomain.CurrentDomain.SetData("_Dyn_Wireless_{}".format(dataKey), [])
if isinstance(new_cuboid, DS.Cuboid):
# set Data to memory
data = new_cuboid
# get Data in memory
datam = System.AppDomain.CurrentDomain.GetData("_Dyn_Wireless_{}".format(dataKey))
if datam is None and isinstance(data, DS.Cuboid):
System.AppDomain.CurrentDomain.SetData("_Dyn_Wireless_{}".format(dataKey), [data])
elif isinstance(data, DS.Cuboid):
try:
datam.append(data)
# clean solids
temp = []
for solid in datam[::-1]:
if all(not solid.DoesIntersect(s) for s in temp):
temp.append(solid)
System.AppDomain.CurrentDomain.SetData("_Dyn_Wireless_{}".format(dataKey), temp)
except Exception as ex:
print(ex)
else:
pass
#
datam = System.AppDomain.CurrentDomain.GetData("_Dyn_Wireless_{}".format(dataKey))
sys.stdout.seek(0)
printResult = sys.stdout.read()
OUT = geoms, printResult, datam
# reset the sys.stdout
sys.stdout = sys.__stdout__
sys.stderr = sys.__stderr__
« Qui ne peut attaquer le raisonnement, attaque le raisonneur. »
Paul Valéry
0 commentaires:
Enregistrer un commentaire