Huxels
Camera controlled minigames. Made for Nordic Game Jam 2024.
One of the themes for the jam was “reflection”, and I ended up in a group where we decided to use Google’s MediaPipe, along with some wrapper code one of the other programmers had previously written, to make a game that uses body tracking through your webcam. We ended up deciding to make a collection of WarioWare-style minigames, both to be able to explore several of the ideas we had, and to avoid code conflicts by letting each programmer work on their own little thing.
I started with making a game where your face is a flower and your hands control bees that have to collect nectar and deliver it back to their hive. This turned out to be pretty straightforward as I could just get the center position for the hands and face (aka your nose), and we used p5.js for all the drawing, which I had previously used for some small things. There is a limit to how much nectar each bee can carry and how much it can get from each flower. The latter limit was introduced to encourage people to not just gather from their own faces but also from their potential co-players. While the flowers and bees are only drawn when faces and hands are detected, the latest captured image of the first face is always drawn in the sun, with a yellow tint.
MODES.bee = {
bees: [],
flowers: [],
hives: [110, 1550],
hiveRect: {y: 400, width: 270, height: 320},
countdown: 0,
start({p, tracker, huxels, time, particles, debugOptions}) {
tracker.scale = 2
let beeMax = []
this.bees = []
tracker.hands.forEach(hand => {
this.bees.push(0)
beeMax.push(500)
})
this.flowers = []
tracker.faces.forEach(face => {
this.flowers.push(Array.from(beeMax))
})
SOUND.beeAmbience.play()
SOUND.beeAmbience.setVolume(0.5)
this.countdown = 30000
},
stop({p, tracker, huxels, time, particles, debugOptions}) {
SOUND.beeAmbience.stop()
SOUND.beeLoop.stop()
SOUND.beePollinateLoop.stop()
SOUND.beePollinateEnd.stop()
SOUND.beeDropOff0.stop()
SOUND.beeDropOff1.stop()
SOUND.beeDropOff2.stop()
},
update({p, tracker, huxels, time, particles, debugOptions}) {
let beeActive = false
let isPollinating = false
tracker.hands.forEach((hand, index) => {
if (hand.isActive) {
beeActive = true
let position = this.getScreenPosition(hand.center)
let foundFlower = false
for (let index2 = 0; index2 < tracker.faces.length; index2++) {
let face = tracker.faces[index2]
if (face.isActive && Vector2D.distance(position, this.getScreenPosition(face.nose)) < 160) {
let nectar = Math.min(time.dt, 500 - this.bees[index], this.flowers[index2][index])
this.bees[index] += nectar
this.flowers[index2][index] -= nectar
foundFlower = true
if (nectar > 0) {
isPollinating = true
if (Math.min(500 - this.bees[index], this.flowers[index2][index]) <= 0) {
SOUND.beePollinateEnd.play()
SOUND.beePollinateEnd.setVolume(0.5)
}
}
break
}
}
if (!foundFlower && this.bees[index] > 0) {
this.hives.forEach(hive => {
if (position.x >= hive && position.x <= hive + this.hiveRect.width && position.y >= this.hiveRect.y && position.y <= this.hiveRect.y + this.hiveRect.height) {
if (this.bees[index] >= time.dt) {
app.score.value += time.dt
this.bees[index] -= time.dt
} else {
app.score.value += this.bees[index]
this.bees[index] = 0
}
if (this.bees[index] <= 0) {
switch (Math.floor(Math.random() * 3)) {
case 0:
SOUND.beeDropOff0.play()
SOUND.beeDropOff0.setVolume(0.5)
break
case 1:
SOUND.beeDropOff1.play()
SOUND.beeDropOff1.setVolume(0.5)
break
case 2:
SOUND.beeDropOff2.play()
SOUND.beeDropOff2.setVolume(0.5)
break
default:
break
}
}
}
})
}
}
})
if (beeActive && !SOUND.beeLoop.isLooping()) {
SOUND.beeLoop.loop()
} else if (!beeActive) {
SOUND.beeLoop.stop()
}
if (isPollinating && !SOUND.beePollinateLoop.isLooping()) {
SOUND.beePollinateLoop.loop()
SOUND.beePollinateLoop.setVolume(0.5)
} else if (!isPollinating) {
SOUND.beePollinateLoop.stop()
}
this.countdown -= time.dt
if (this.countdown <= 0) {
debugOptions.mode = "tetris"
}
},
drawBackground({p, tracker, huxels, time, particles, debugOptions}) {
p.image(IMAGE.beeClouds, 0, 0, p.width, p.height)
},
draw({p, tracker, huxels, time, particles, debugOptions}) {
p.imageMode(p.CENTER)
p.image(IMAGE.beeSun, 480, 240)
p.tint(48, 100, 50)
p.image(tracker.faces[0].thumbnail, 480, 230, 240, 240)
p.noTint()
p.imageMode(p.CORNER)
p.image(IMAGE.beeFront, 0, 0, p.width, p.height)
p.imageMode(p.CENTER)
tracker.faces.forEach((face, index) => {
if (face.isActive) {
let position = this.getScreenPosition(face.nose)
p.fill(83.48, 42.59, 57.65)
p.noStroke()
p.triangle(...position, position.x - 32, p.height, position.x + 32, p.height)
p.image(IMAGE.leaf, position.x - 50, position.y + (p.height - position.y) * 0.4, 100, 42)
p.image(IMAGE.leaf, position.x + 50, position.y + (p.height - position.y) * 0.5, 100, 42)
switch (index % 3) {
case 0:
p.image(IMAGE.beeFlowerBlue, ...position)
break
case 1:
p.image(IMAGE.beeFlowerOrange, ...position)
break
case 2:
p.image(IMAGE.beeFlowerRed, ...position)
break
default:
break
}
p.image(face.thumbnail, ...position, 96, 96)
}
})
tracker.hands.forEach((hand, index) => {
if (hand.isActive) {
let position = this.getScreenPosition(hand.center)
p.image(IMAGE.bee, ...position)
}
})
p.imageMode(p.CORNER)
},
getScreenPosition(position) {
return new Vector2D(position.x * 3, position.y * 2.25)
},
}
When I had mostly finished the bee game, I found myself wanting to realize another of our ideas, which was a cleaning game, so I quickly threw together some code that randomly placed bits of dirt across the screen and removed this dirt when your hands got close enough. After the ability to draw currently captured faces had been added by another programmer, I replaced the dirt with that just to test it out, and ended up sticking with it since I found it pretty funny.
MODES.clean = {
dirt: [],
stars: [],
countdown: 0,
start({p, tracker, huxels, time, particles, debugOptions}) {
tracker.scale = 6
this.dirt = []
this.stars = []
for (let index = 0; index < 256; index++) {
this.dirt.push(new Vector2D(Math.random() * (p.width - 256) + 128, Math.random() * (p.height - 256) + 128))
}
this.countdown = 30000
},
stop({p, tracker, huxels, time, particles, debugOptions}) {
SOUND.windowWipe.stop()
},
update({p, tracker, huxels, time, particles, debugOptions}) {
tracker.hands.forEach(hand => {
if (hand.isActive) {
let position = hand.center
for (let index = 0; index < this.dirt.length; index++) {
if (Vector2D.distance(position, this.dirt[index]) < 64) {
if (Math.random() < 0.1) {
this.stars.push(this.dirt[index])
app.score.value += 50
} else {
app.score.value += 10
}
this.dirt.splice(index, 1)
index--
if (!SOUND.windowWipe.isPlaying()) {
SOUND.windowWipe.play()
}
}
}
}
})
this.countdown -= time.dt
if (this.countdown <= 0) {
debugOptions.mode = "tetris"
}
},
drawBackground({p, tracker, huxels, time, particles, debugOptions}) {
p.image(IMAGE.cleanMirror, 0, 0, p.width, p.height)
},
draw({p, tracker, huxels, time, particles, debugOptions}) {
p.imageMode(p.CENTER)
this.stars.forEach(star => {
p.image(IMAGE.cleanStar, ...star)
})
this.dirt.forEach(spot => {
p.image(tracker.faces[0].thumbnail, ...spot)
})
tracker.hands.forEach(hand => {
if (hand.isActive) {
p.image(IMAGE.cleanSponge, ...hand.center, 128, 64)
}
})
p.imageMode(p.CORNER)
},
}
Trying out this (to me) new way of interacting with a computer was really fun, and it was definitely not close to something I had imagined I would do during the jam. I look forward to playing around with this more, and I certainly have a bunch of ideas for it already…