One of the themes for the jam was "reflection", and I ended up in a group with, among others, Kate Compton, who had the idea of using Google's MediaPipe along with some wrapper code she had previously written to make a game that uses body tracking through your webcam. We ended up deciding to make a collection of WarioWare-style minigames, both to be able to explore several of the ideas we had, and to avoid code conflicts by letting each programmer work on their own little thing.
I started with making a game where your face is a flower and your hands control bees that have to collect nectar and deliver it back to their hive. This turned out to be pretty straightforward as I could just get the center position for the hands and face (aka your nose), and we used p5.js for all the drawing, which I had previously used for some small things. There is a limit to how much nectar each bee can carry and how much it can get from each flower. The latter limit was introduced to encourage people to not just gather from their own faces but also from their potential co-players. While the flowers and bees are only drawn when faces and hands are detected, the latest captured image of the first face is always drawn in the sun, with a yellow tint.
MODES.bee = { bees: [], flowers: [], hives: [110, 1550], hiveRect: {y: 400, width: 270, height: 320}, countdown: 0, start({p, tracker, huxels, time, particles, debugOptions}) { tracker.scale = 2 let beeMax = [] this.bees = [] tracker.hands.forEach(hand => { this.bees.push(0) beeMax.push(500) }) this.flowers = [] tracker.faces.forEach(face => { this.flowers.push(Array.from(beeMax)) }) SOUND.beeAmbience.play() SOUND.beeAmbience.setVolume(0.5) this.countdown = 30000 }, stop({p, tracker, huxels, time, particles, debugOptions}) { SOUND.beeAmbience.stop() SOUND.beeLoop.stop() SOUND.beePollinateLoop.stop() SOUND.beePollinateEnd.stop() SOUND.beeDropOff0.stop() SOUND.beeDropOff1.stop() SOUND.beeDropOff2.stop() }, update({p, tracker, huxels, time, particles, debugOptions}) { let beeActive = false let isPollinating = false tracker.hands.forEach((hand, index) => { if (hand.isActive) { beeActive = true let position = this.getScreenPosition(hand.center) let foundFlower = false for (let index2 = 0; index2 < tracker.faces.length; index2++) { let face = tracker.faces[index2] if (face.isActive && Vector2D.distance(position, this.getScreenPosition(face.nose)) < 160) { let nectar = Math.min(time.dt, 500 - this.bees[index], this.flowers[index2][index]) this.bees[index] += nectar this.flowers[index2][index] -= nectar foundFlower = true if (nectar > 0) { isPollinating = true if (Math.min(500 - this.bees[index], this.flowers[index2][index]) <= 0) { SOUND.beePollinateEnd.play() SOUND.beePollinateEnd.setVolume(0.5) } } break } } if (!foundFlower && this.bees[index] > 0) { this.hives.forEach(hive => { if (position.x >= hive && position.x <= hive + this.hiveRect.width && position.y >= this.hiveRect.y && position.y <= this.hiveRect.y + this.hiveRect.height) { if (this.bees[index] >= time.dt) { app.score.value += time.dt this.bees[index] -= time.dt } else { app.score.value += this.bees[index] this.bees[index] = 0 } if (this.bees[index] <= 0) { switch (Math.floor(Math.random() * 3)) { case 0: SOUND.beeDropOff0.play() SOUND.beeDropOff0.setVolume(0.5) break case 1: SOUND.beeDropOff1.play() SOUND.beeDropOff1.setVolume(0.5) break case 2: SOUND.beeDropOff2.play() SOUND.beeDropOff2.setVolume(0.5) break default: break } } } }) } } }) if (beeActive && !SOUND.beeLoop.isLooping()) { SOUND.beeLoop.loop() } else if (!beeActive) { SOUND.beeLoop.stop() } if (isPollinating && !SOUND.beePollinateLoop.isLooping()) { SOUND.beePollinateLoop.loop() SOUND.beePollinateLoop.setVolume(0.5) } else if (!isPollinating) { SOUND.beePollinateLoop.stop() } this.countdown -= time.dt if (this.countdown <= 0) { debugOptions.mode = "tetris" } }, drawBackground({p, tracker, huxels, time, particles, debugOptions}) { p.image(IMAGE.beeClouds, 0, 0, p.width, p.height) }, draw({p, tracker, huxels, time, particles, debugOptions}) { p.imageMode(p.CENTER) p.image(IMAGE.beeSun, 480, 240) p.tint(48, 100, 50) p.image(tracker.faces[0].thumbnail, 480, 230, 240, 240) p.noTint() p.imageMode(p.CORNER) p.image(IMAGE.beeFront, 0, 0, p.width, p.height) p.imageMode(p.CENTER) tracker.faces.forEach((face, index) => { if (face.isActive) { let position = this.getScreenPosition(face.nose) p.fill(83.48, 42.59, 57.65) p.noStroke() p.triangle(...position, position.x - 32, p.height, position.x + 32, p.height) p.image(IMAGE.leaf, position.x - 50, position.y + (p.height - position.y) * 0.4, 100, 42) p.image(IMAGE.leaf, position.x + 50, position.y + (p.height - position.y) * 0.5, 100, 42) switch (index % 3) { case 0: p.image(IMAGE.beeFlowerBlue, ...position) break case 1: p.image(IMAGE.beeFlowerOrange, ...position) break case 2: p.image(IMAGE.beeFlowerRed, ...position) break default: break } p.image(face.thumbnail, ...position, 96, 96) } }) tracker.hands.forEach((hand, index) => { if (hand.isActive) { let position = this.getScreenPosition(hand.center) p.image(IMAGE.bee, ...position) } }) p.imageMode(p.CORNER) }, getScreenPosition(position) { return new Vector2D(position.x * 3, position.y * 2.25) }, }
When I had mostly finished the bee game, I found myself wanting to realize another of our ideas, which was a cleaning game, so I quickly threw together some code that randomly placed bits of dirt across the screen and removed this dirt when your hands got close enough. After Kate added the ability to draw currently captured faces, I replaced the dirt with that just to test it out, and ended up sticking with it since I found it pretty funny.
MODES.clean = { dirt: [], stars: [], countdown: 0, start({p, tracker, huxels, time, particles, debugOptions}) { tracker.scale = 6 this.dirt = [] this.stars = [] for (let index = 0; index < 256; index++) { this.dirt.push(new Vector2D(Math.random() * (p.width - 256) + 128, Math.random() * (p.height - 256) + 128)) } this.countdown = 30000 }, stop({p, tracker, huxels, time, particles, debugOptions}) { SOUND.windowWipe.stop() }, update({p, tracker, huxels, time, particles, debugOptions}) { tracker.hands.forEach(hand => { if (hand.isActive) { let position = hand.center for (let index = 0; index < this.dirt.length; index++) { if (Vector2D.distance(position, this.dirt[index]) < 64) { if (Math.random() < 0.1) { this.stars.push(this.dirt[index]) app.score.value += 50 } else { app.score.value += 10 } this.dirt.splice(index, 1) index-- if (!SOUND.windowWipe.isPlaying()) { SOUND.windowWipe.play() } } } } }) this.countdown -= time.dt if (this.countdown <= 0) { debugOptions.mode = "tetris" } }, drawBackground({p, tracker, huxels, time, particles, debugOptions}) { p.image(IMAGE.cleanMirror, 0, 0, p.width, p.height) }, draw({p, tracker, huxels, time, particles, debugOptions}) { p.imageMode(p.CENTER) this.stars.forEach(star => { p.image(IMAGE.cleanStar, ...star) }) this.dirt.forEach(spot => { p.image(tracker.faces[0].thumbnail, ...spot) }) tracker.hands.forEach(hand => { if (hand.isActive) { p.image(IMAGE.cleanSponge, ...hand.center, 128, 64) } }) p.imageMode(p.CORNER) }, }
Trying out this (to me) new way of interacting with a computer was really fun, and it was definitely not close to something I had imagined I would do during the jam. I look forward to playing around with this more, and I certainly have a bunch of ideas for it already...