Skip to content

Commit 9e563cf

Browse files
author
shalstvedt
committed
Demo works fully (move, create, delete, color) and multimodally, on OS X
git-svn-id: http://nuicode.svnrepository.com/svn/ccf@56 49704ecc-8926-4d56-a4d4-72374b29732b
1 parent c526b5f commit 9e563cf

17 files changed

+545
-81
lines changed

‎SConstruct

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@ daemon_src = [
2323
'src/modules/ccxAudioOutputModule.cpp',
2424
'src/modules/ccxTemporalFusionModule.cpp',
2525
'src/modules/ccxDebugGestureOutputModule.cpp',
26+
'src/modules/ccxDebugGestureModule.cpp',
27+
'src/modules/ccxJSONInteractionModule.cpp',
2628
'src/models/ballWorldGrammar.cpp',
2729
'contrib/libresample/lib/resample.c',
2830
'contrib/libresample/lib/filterkit.c',

‎ccx/ccxFactory.cpp

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,9 +33,11 @@ static ccxFactory *instance = NULL;
3333
void ccxFactory::init() {
3434
// Keep the alphabetical order when adding new modules!
3535
REGISTER_MODULE(AudioOutput);
36+
REGISTER_MODULE(DebugGesture);
3637
REGISTER_MODULE(DebugGestureOutput);
3738
REGISTER_MODULE(DebugInput);
3839
REGISTER_MODULE(DebugOutput);
40+
REGISTER_MODULE(JSONInteraction);
3941
REGISTER_MODULE(SphinxASR);
4042
REGISTER_MODULE(SpiritParser);
4143
REGISTER_MODULE(TemporalFusion);

‎configs/presets/fusion.txt

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
1-
pipeline create DebugInput Debug
1+
pipeline create DebugGesture Gesture
22
pipeline create SpiritParser Parser
33
pipeline create SphinxASR ASR
44
pipeline create AudioOutput Audio
55
pipeline create TemporalFusion Fusion
6-
pipeline create DebugGestureOutput Gesture
6+
pipeline create JSONInteraction Interaction
77

88
pipeline set Audio x 73
99
pipeline set Audio y 33
@@ -14,8 +14,8 @@ pipeline set ASR y 362
1414
pipeline set Parser x 383
1515
pipeline set Parser y 72
1616

17-
pipeline set Debug x 655
18-
pipeline set Debug y 193
17+
pipeline set Interaction x 655
18+
pipeline set Interaction y 193
1919

2020
pipeline set Fusion x 619
2121
pipeline set Fusion y 385
@@ -31,5 +31,5 @@ pipeline set ASR noise_dictionary configs/ballworld/sphinxmodel/noisedict
3131
pipeline connect Audio 0 ASR 0
3232
pipeline connect ASR 0 Parser 0
3333
pipeline connect Parser 0 Fusion 0
34-
pipeline connect Fusion 0 Debug 0
34+
pipeline connect Fusion 0 Interaction 0
3535
pipeline connect Gesture 0 Fusion 1

‎demo/ballworld.kv

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
#:kivy 1.0
22
#:import kivy kivy
3+
#:import win kivy.core.window
34

4-
<BallWorld>:
5+
FloatLayout:
56
canvas:
67
Color:
78
rgb: 1, 1, 1
@@ -12,16 +13,22 @@
1213
BoxLayout:
1314
padding: 10
1415
spacing: 10
15-
size_hint: 1, None
16-
pos_hint: {'top': 1}
16+
size_hint: 1, 1
17+
pos_hint: {'bottom': 1}
1718
height: 24
18-
Image:
19-
size_hint: None, None
20-
size: 24, 24
21-
source: 'data/logo/kivy-icon-32.png'
2219
Label:
2320
height: 24
2421
text_size: self.size
2522
color: (1, 1, 1, .8)
26-
text: 'Kivy %s - Ballworld' % kivy.__version__
23+
text: 'BallWorld - Community Core Fusion 0.1a'
2724

25+
<Ball>:
26+
size: self.size
27+
size_hint: 0,0
28+
pos: (self.x, self.y)
29+
canvas:
30+
Color:
31+
rgb: self.color
32+
Ellipse:
33+
size: self.size
34+
pos: self.pos

‎demo/ballworld.py

Lines changed: 151 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@
99

1010
import time
1111

12+
import json
13+
1214
import kivy
1315
kivy.require('1.0.7')
1416

@@ -17,39 +19,164 @@
1719
from kivy.core.window import Window
1820
from kivy.config import Config
1921
from os.path import join, dirname
20-
from kivy.properties import StringProperty
22+
from kivy.properties import StringProperty, NumericProperty, ListProperty
2123
from glob import glob
2224
from kivy.logger import Logger
2325
from kivy.uix.widget import Widget
2426
from kivy.graphics import Color, Ellipse, Line
2527

28+
utteranceStarted = False
29+
ballCounter = 0
30+
gestureBuffer = []
31+
32+
class Ball(Widget):
33+
id = NumericProperty(0)
34+
x = NumericProperty(25)
35+
y = NumericProperty(25)
36+
color = ListProperty([1, 1, 1])
37+
width = NumericProperty(50)
38+
height = NumericProperty(50)
39+
40+
def on_touch_down(self, touch):
41+
global gestureBuffer
42+
if self.collide_point(*touch.pos) and utteranceStarted:
43+
print "Ball: %d" % self.id
44+
gestureBuffer.append({"type" : "ball", "val" : self.id})
45+
return True
46+
2647
class BallWorld(App):
2748

28-
def build(self):
29-
30-
def utterance_begin(*l):
31-
print "start"
32-
r = requests.get('http://127.0.0.1:7500/utterance/begin')
33-
print r.content
34-
r = requests.get('http://127.0.0.1:7500/utterance/speech/begin')
49+
def utterance_get(self, *args):
50+
global ballCounter
51+
print "get"
52+
r = requests.get('http://127.0.0.1:7500/utterance/get')
53+
if(r):
3554
print r.content
55+
tree = json.loads(r.content)
56+
colorIndex = {"red" : [1, 0, 0], "green" : [0, 1, 0], "blue" : [0, 0, 1], "white" : [1, 1, 1], "yellow" : [1, 1, 0], "purple" : [1, 0, 1], "orange" : [1, 0.5, 0]}
57+
if tree.has_key("val"):
58+
if tree["val"] == "create":
59+
coords = tree["children"][1]["val"].split(',')
60+
print coords
61+
self.root.add_widget(Ball(center_x = int(coords[0]), center_y = int(coords[1]), color=colorIndex[tree["children"][0]["val"]], id=ballCounter))
62+
ballCounter += 1
63+
elif tree["val"] == "destroy":
64+
for attrs in tree["children"]:
65+
if attrs.has_key("val") and attrs.has_key("type"):
66+
if attrs["type"] == "ball":
67+
for child in self.root.children:
68+
if hasattr(child, "id"):
69+
if child.id == int(tree["children"][0]["val"]):
70+
print "found ball by id"
71+
print child.id
72+
self.root.remove_widget(child)
73+
elif attrs["type"] == "color":
74+
for child in self.root.children:
75+
if hasattr(child, "color"):
76+
if child.color == colorIndex[tree["children"][0]["val"]]:
77+
print "found ball by color"
78+
print child.id
79+
print child.color
80+
self.root.remove_widget(child)
81+
elif tree["val"] == "color":
82+
if tree["children"][0]["type"] == "ball":
83+
for child in self.root.children:
84+
if hasattr(child, "id"):
85+
if child.id == int(tree["children"][0]["val"]):
86+
print "found ball by id"
87+
print child.id
88+
print child.color
89+
child.color = colorIndex[tree["children"][1]["val"]]
90+
elif tree["children"][0]["type"] == "color":
91+
for child in self.root.children:
92+
if hasattr(child, "color"):
93+
if child.color == colorIndex[tree["children"][0]["val"]]:
94+
print "found ball by color"
95+
print child.id
96+
child.color = colorIndex[tree["children"][1]["val"]]
97+
elif tree["val"] == "move":
98+
s = tree["children"][1]["val"]
99+
destination = list()
100+
for item in s.split(','):
101+
destination.append(int(item))
102+
if tree["children"][0]["type"] == "ball":
103+
for child in self.root.children:
104+
if hasattr(child, "id"):
105+
if child.id == int(tree["children"][0]["val"]):
106+
print "moving by id: %d" % child.id
107+
child.center = destination
108+
elif tree["children"][0]["type"] == "color":
109+
for child in self.root.children:
110+
if hasattr(child, "color"):
111+
if child.color == colorIndex[tree["children"][0]["val"]]:
112+
print "moving by color"
113+
child.center = destination
114+
36115

37-
def utterance_end(*l):
38-
time.sleep(0.25)
39-
print "end"
40-
r = requests.get('http://127.0.0.1:7500/utterance/speech/end')
41-
#print "gesture"
42-
#r = requests.get('http://127.0.0.1:7500/utterance/gesture/data')
43-
print r.content
44-
r = requests.get('http://127.0.0.1:7500/utterance/end')
45-
print r.content
46-
btn = Button(text='utterance')
47-
btn.bind(on_press=utterance_begin)
48-
btn.bind(on_release=utterance_end)
49-
return btn
50-
116+
def utterance_begin(self, *args):
117+
print "start"
118+
r = requests.get('http://127.0.0.1:7500/utterance/begin')
119+
#print r.content
120+
r = requests.get('http://127.0.0.1:7500/utterance/speech/begin')
121+
#print r.content
122+
123+
def utterance_end(self, *args):
124+
global gestureBuffer
125+
print "gesture"
126+
if gestureBuffer != []:
127+
jsonData = json.dumps(gestureBuffer)
128+
print jsonData
129+
r = requests.post('http://127.0.0.1:7500/utterance/gesture/data', jsonData)
130+
gestureBuffer = []
131+
time.sleep(0.25)
132+
print "end"
133+
r = requests.get('http://127.0.0.1:7500/utterance/speech/end')
134+
#print r.content
135+
r = requests.get('http://127.0.0.1:7500/utterance/end')
136+
#print r.content
137+
time.sleep(0.5)
138+
self.utterance_get()
139+
140+
141+
def on_key_down(self, instance, scancode, *largs):
142+
global utteranceStarted
143+
if scancode == 32: # spacebar
144+
if utteranceStarted == False:
145+
utteranceStarted = True
146+
self.utterance_begin()
147+
elif scancode == 50:
148+
self.utterance_get()
149+
else:
150+
print scancode
151+
152+
def on_key_up(self, scancode, *largs):
153+
global utteranceStarted
154+
if largs[0] == 32: # spacebar
155+
if utteranceStarted == True:
156+
utteranceStarted = False
157+
self.utterance_end()
158+
159+
def on_touch_down(instance, self, touch):
160+
global gestureBuffer
161+
for child in self.children[:]:
162+
if child.dispatch('on_touch_down', touch):
163+
return True
164+
if utteranceStarted:
165+
print "Point: %d %d" % touch.pos
166+
gestureBuffer.append({"type" : "point", "val" : "%d,%d" % touch.pos})
167+
168+
def build(self):
169+
root = self.root
170+
requests.get('http://127.0.0.1:7500/pipeline/start')
171+
Window.bind(on_key_down=self.on_key_down)
172+
Window.bind(on_key_up=self.on_key_up)
173+
root.bind(on_touch_down=self.on_touch_down)
174+
175+
51176
if __name__ == '__main__':
52-
Config.set('input', 'default', 'mouse')
53-
Config.write()
177+
#Config.set('input', 'default', 'mouse')
178+
#Config.write()
179+
# do the build_config sometime
54180
BallWorld().run()
181+
requests.get('http://127.0.0.1:7500/pipeline/stop')
55182

‎fusion/fusion.xcodeproj/project.pbxproj

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88

99
/* Begin PBXBuildFile section */
1010
7403310113F4DEEA0038EB32 /* ccxDebugGestureOutputModule.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 740330FF13F4DEEA0038EB32 /* ccxDebugGestureOutputModule.cpp */; };
11+
740C0EC213FDE29A00D9F8F8 /* ccxJSONInteractionModule.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 740C0EC013FDE29A00D9F8F8 /* ccxJSONInteractionModule.cpp */; };
1112
741C664013F0F940005219A6 /* ccxAudioOutputModule.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 741C663F13F0F940005219A6 /* ccxAudioOutputModule.cpp */; };
1213
745A598713EE501500736762 /* ccxSphinxASRModule.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 74708A7213EE362500112449 /* ccxSphinxASRModule.cpp */; };
1314
7461F2BB13F2156A005654C5 /* ccxTemporalFusionModule.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 74708A7613EE362500112449 /* ccxTemporalFusionModule.cpp */; };
@@ -25,6 +26,7 @@
2526
74708CA613EE3F9200112449 /* gui in CopyFiles */ = {isa = PBXBuildFile; fileRef = 74708CA413EE3F9200112449 /* gui */; };
2627
749B0B5413EF38CF004A673E /* libs3decoder.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = 749B0B5213EF38CF004A673E /* libs3decoder.dylib */; };
2728
749B0B5513EF38CF004A673E /* libsphinxbase.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = 749B0B5313EF38CF004A673E /* libsphinxbase.dylib */; };
29+
74ECAB0013FDD78600424329 /* ccxDebugGestureModule.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 74ECAAFC13FDD78600424329 /* ccxDebugGestureModule.cpp */; };
2830
74FEC02D13EEA14D009C1AB3 /* librtaudio.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 74FEC02C13EEA14D009C1AB3 /* librtaudio.a */; };
2931
74FEC02F13EEA408009C1AB3 /* CoreAudio.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 74FEC02E13EEA408009C1AB3 /* CoreAudio.framework */; };
3032
74FEC03113EEB2A0009C1AB3 /* resample.c in Sources */ = {isa = PBXBuildFile; fileRef = 74FEC03013EEB2A0009C1AB3 /* resample.c */; };
@@ -66,6 +68,8 @@
6668
/* Begin PBXFileReference section */
6769
740330FF13F4DEEA0038EB32 /* ccxDebugGestureOutputModule.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = ccxDebugGestureOutputModule.cpp; sourceTree = "<group>"; };
6870
7403310013F4DEEA0038EB32 /* ccxDebugGestureOutputModule.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = ccxDebugGestureOutputModule.h; sourceTree = "<group>"; };
71+
740C0EC013FDE29A00D9F8F8 /* ccxJSONInteractionModule.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = ccxJSONInteractionModule.cpp; sourceTree = "<group>"; };
72+
740C0EC113FDE29A00D9F8F8 /* ccxJSONInteractionModule.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = ccxJSONInteractionModule.h; sourceTree = "<group>"; };
6973
741C663E13F0F92F005219A6 /* ccxAudioOutputModule.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = ccxAudioOutputModule.h; sourceTree = "<group>"; };
7074
741C663F13F0F940005219A6 /* ccxAudioOutputModule.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = ccxAudioOutputModule.cpp; sourceTree = "<group>"; };
7175
74708A0B13EE358F00112449 /* fusion */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = fusion; sourceTree = BUILT_PRODUCTS_DIR; };
@@ -100,6 +104,10 @@
100104
74708CA413EE3F9200112449 /* gui */ = {isa = PBXFileReference; lastKnownFileType = folder; name = gui; path = ../gui; sourceTree = "<group>"; };
101105
749B0B5213EF38CF004A673E /* libs3decoder.dylib */ = {isa = PBXFileReference; lastKnownFileType = "compiled.mach-o.dylib"; name = libs3decoder.dylib; path = ../contrib/sphinx/lib/osx/libs3decoder.dylib; sourceTree = "<group>"; };
102106
749B0B5313EF38CF004A673E /* libsphinxbase.dylib */ = {isa = PBXFileReference; lastKnownFileType = "compiled.mach-o.dylib"; name = libsphinxbase.dylib; path = ../contrib/sphinx/lib/osx/libsphinxbase.dylib; sourceTree = "<group>"; };
107+
74ECAAFC13FDD78600424329 /* ccxDebugGestureModule.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = ccxDebugGestureModule.cpp; sourceTree = "<group>"; };
108+
74ECAAFD13FDD78600424329 /* ccxDebugGestureModule.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = ccxDebugGestureModule.h; sourceTree = "<group>"; };
109+
74ECAAFE13FDD78600424329 /* ccxJSONGestureModule.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = ccxJSONGestureModule.cpp; sourceTree = "<group>"; };
110+
74ECAAFF13FDD78600424329 /* ccxJSONGestureModule.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = ccxJSONGestureModule.h; sourceTree = "<group>"; };
103111
74FEC02C13EEA14D009C1AB3 /* librtaudio.a */ = {isa = PBXFileReference; lastKnownFileType = archive.ar; name = librtaudio.a; path = ../contrib/rtaudio/librtaudio.a; sourceTree = "<group>"; };
104112
74FEC02E13EEA408009C1AB3 /* CoreAudio.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = CoreAudio.framework; path = System/Library/Frameworks/CoreAudio.framework; sourceTree = SDKROOT; };
105113
74FEC03013EEB2A0009C1AB3 /* resample.c */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.c; name = resample.c; path = ../contrib/libresample/lib/resample.c; sourceTree = "<group>"; };
@@ -186,6 +194,12 @@
186194
74708A6513EE362500112449 /* modules */ = {
187195
isa = PBXGroup;
188196
children = (
197+
740C0EC013FDE29A00D9F8F8 /* ccxJSONInteractionModule.cpp */,
198+
740C0EC113FDE29A00D9F8F8 /* ccxJSONInteractionModule.h */,
199+
74ECAAFC13FDD78600424329 /* ccxDebugGestureModule.cpp */,
200+
74ECAAFD13FDD78600424329 /* ccxDebugGestureModule.h */,
201+
74ECAAFE13FDD78600424329 /* ccxJSONGestureModule.cpp */,
202+
74ECAAFF13FDD78600424329 /* ccxJSONGestureModule.h */,
189203
74708A6613EE362500112449 /* .dummy.cpp */,
190204
74708A6713EE362500112449 /* .dummy.h */,
191205
74708A6813EE362500112449 /* ccxASRModule.cpp */,
@@ -310,6 +324,8 @@
310324
74708A8313EE363200112449 /* ccxDebugOutputModule.cpp in Sources */,
311325
74708A8B13EE363200112449 /* ccxSpiritParserModule.cpp in Sources */,
312326
741C664013F0F940005219A6 /* ccxAudioOutputModule.cpp in Sources */,
327+
74ECAB0013FDD78600424329 /* ccxDebugGestureModule.cpp in Sources */,
328+
740C0EC213FDE29A00D9F8F8 /* ccxJSONInteractionModule.cpp in Sources */,
313329
);
314330
runOnlyForDeploymentPostprocessing = 0;
315331
};

0 commit comments

Comments
 (0)