Expression
a note under the finger: each held note bent, pressed, and slid on its own, drawn as a ribbon at its pitch, as tall as its slide and as wide as its pressure, on a surface the mouse plays through a virtual MIDI source (so a bend crosses Core MIDI the way a controller's does) and a polyphonic-expression controller plays for real; left alone, a hand of the sketch's own plays a phrase, and bowed makes pressure the bow (PlayingNote, bend/press/slide, MIDIInput.heldNotes, mpeZones)
- Run
swift run --package-path Examples Example-Audio-Expression- Imports
OllinAudioOllinMIDI- Source
- Examples/Audio/Expression/Sketch.swift
import Foundation
import Ollin
import OllinAudio
import OllinMIDI
/// A note under the finger. Each note a polyphonic-expression controller
/// holds is bent, pressed, and slid on its own, and the synth follows each
/// one: across the surface is pitch, up it is slide (the filter opening), and
/// how long a note has been held stands in for pressure when the mouse plays.
/// The mouse plays through a virtual MIDI source, so what the sketch draws
/// and plays has crossed Core MIDI the way a real controller's notes do; plug
/// one in (a polyphonic-expression controller, or a phone app that speaks
/// MPE) and its notes join the same picture. Left alone, a hand of the
/// sketch's own plays a phrase, straight into the instrument.
///
/// swift run Example-Audio-Expression
///
/// Press and drag on the surface to play: moving across bends the note from
/// where it was struck, moving up and down slides it, and holding presses it
/// harder. Switch `bowed` on for the cello, where pressure is the bow and a
/// note pressed to nothing goes quiet. See `Docs/Helpers/Synthesis.md` and
/// `Docs/Integration/MIDI.md`.
@main
final class Expression: Sketch {
/// Everything a note on the surface is, whoever is playing it.
struct Ribbon: Equatable {
let id: Int
let note: Int
let pitch: Double
let velocity: Double
let pressure: Double
let slide: Double
init(_ held: HeldNote) {
id = held.id
note = held.note
pitch = held.pitch
velocity = held.velocity
pressure = held.pressure
slide = held.slide
}
init(id: Int, note: Int, pitch: Double, velocity: Double, pressure: Double, slide: Double) {
self.id = id
self.note = note
self.pitch = pitch
self.velocity = velocity
self.pressure = pressure
self.slide = slide
}
}
/// A lead whose filter opens with the slide and whose level rises with
/// pressure: the two things a wave can do with a finger, beside the bend.
static let lead = Voice(
waveform: .sawtooth,
envelope: Envelope(attack: 0.02, decay: 0.3, sustain: 0.7, release: 0.5),
filter: Voice.Filter(mode: .lowpass, cutoff: 500, resonance: 0.35, keyTracking: 0.6, slideAmount: 2),
detune: 0.06
)
let midi = MIDIInput()
let out = MIDIOutput(name: "Ollin Expression")
let synth = Synth(Expression.lead, polyphony: 12)
@Param var bowed = false
var wasBowed = false
/// The notes the synth is playing, by the surface's own identity for them.
var playing: [Int: PlayingNote] = [:]
/// The mouse's note while it is down: the channel it went out on, and
/// where and when it was struck, so a drag bends from there.
struct Finger {
let channel: Int
let note: Int
let struckX: Double
let since: Double
}
var finger: Finger?
var nextChannel = 2
var lastTouch = -10.0
var wireNotesSeen = 0
let lowest = 48.0 // C3
let octaves = 2.0
var surface: Rectangle {
Rectangle(x: 60 * scale, y: 160 * scale, width: width - 120 * scale, height: height - 300 * scale)
}
override func setup() {
do {
try out.openVirtual(named: "Ollin Expression")
try midi.start()
} catch {
print("could not open the MIDI loopback: \(error)")
}
noStroke()
}
override func draw() {
background(Color(white: 0.07))
announceZone()
applyVoice()
playMouse()
// Whoever is playing: the surface and any controller through the
// wire, and the sketch's own hand when nobody has touched it lately.
let wired = midi.heldNotes
if !wired.isEmpty { wireNotesSeen = frameCount; lastTouch = time }
var ribbons = wired.map(Ribbon.init)
let idle = time - lastTouch > 2.5
if idle { ribbons += hand(at: time) }
follow(ribbons)
drawSurface()
for ribbon in ribbons { drawRibbon(ribbon) }
drawLabels(ribbons: ribbons, idle: idle)
}
// MARK: The wire
/// The zone is announced until the input has it: the input connects to
/// the new virtual source a moment after it opens, and anything sent
/// before that is lost.
func announceZone() {
guard midi.mpeZones.isEmpty, frameCount % 30 == 0 else { return }
out.send(mpeZone: .lower())
}
func applyVoice() {
guard bowed != wasBowed || frameCount == 1 else { return }
wasBowed = bowed
synth.voice = bowed ? .cello : Expression.lead
synth.pressure = 0.7
}
/// The mouse as a polyphonic-expression controller: each press takes the
/// next member channel, says where the finger is, then puts it down.
func playMouse() {
if mouseIsPressed, finger == nil, surface.contains(Vector2(mouseX, mouseY)) {
let note = Int(pitch(atX: mouseX).rounded())
let channel = nextChannel
nextChannel = nextChannel == 16 ? 2 : nextChannel + 1
out.pitchBend(8192, channel: channel)
out.controlChange(74, value: Int(slide(atY: mouseY) * 127), channel: channel)
out.channelPressure(Int(0.25 * 127), channel: channel)
out.noteOn(note, velocity: 100, channel: channel)
finger = Finger(channel: channel, note: note, struckX: mouseX, since: time)
lastTouch = time
}
guard let finger else { return }
lastTouch = time
if !mouseIsPressed {
out.noteOff(finger.note, channel: finger.channel)
self.finger = nil
return
}
// Where the finger is now: bent from where it struck, slid by height,
// and pressed harder the longer it stays.
let semitones = (mouseX - finger.struckX) / (surface.width / (12 * octaves))
let bend = Int((8192 + semitones / 48 * 8192).rounded())
out.pitchBend(min(max(0, bend), 16383), channel: finger.channel)
out.controlChange(74, value: Int(slide(atY: mouseY) * 127), channel: finger.channel)
let pressure = min(1, 0.25 + 0.75 * (time - finger.since) / 1.5)
out.channelPressure(Int(pressure * 127), channel: finger.channel)
}
// MARK: The instrument
/// Starts each note as it appears, lets it go as it leaves, and every
/// frame hands each held note what the surface says about it.
func follow(_ ribbons: [Ribbon]) {
for ribbon in ribbons where playing[ribbon.id] == nil {
playing[ribbon.id] = synth.noteOn(Pitch(Double(ribbon.note)), velocity: ribbon.velocity)
}
for (id, note) in playing where !ribbons.contains(where: { $0.id == id }) {
synth.noteOff(note)
playing[id] = nil
}
for ribbon in ribbons {
guard let note = playing[ribbon.id] else { continue }
synth.bend(note, semitones: ribbon.pitch - Double(ribbon.note))
synth.press(note, ribbon.pressure)
synth.slide(note, ribbon.slide)
}
}
/// A phrase the sketch plays itself: four notes, each sliding a little
/// from where it was struck, pressed hardest in its middle, the finger
/// drifting up the key as it goes.
func hand(at t: Double) -> [Ribbon] {
let cycle = 9.6
let phrase: [(start: Double, length: Double, note: Int, rise: Double)] = [
(0.0, 3.2, 55, 5), (1.6, 3.2, 62, 3), (4.8, 3.6, 59, 7), (6.4, 2.6, 67, -2),
]
let local = t.truncatingRemainder(dividingBy: cycle)
var ribbons: [Ribbon] = []
for (index, step) in phrase.enumerated() where local >= step.start && local < step.start + step.length {
let u = (local - step.start) / step.length
let eased = u * u * (3 - 2 * u)
ribbons.append(Ribbon(id: 1000 + index, note: step.note,
pitch: Double(step.note) + step.rise * eased,
velocity: 0.8, pressure: sin(u * .pi),
slide: 0.25 + 0.6 * u))
}
return ribbons
}
// MARK: The picture
func pitch(atX x: Double) -> Double {
lowest + (x - surface.x) / surface.width * 12 * octaves
}
func x(ofPitch pitch: Double) -> Double {
surface.x + (pitch - lowest) / (12 * octaves) * surface.width
}
func slide(atY y: Double) -> Double {
min(max(0, 1 - (y - surface.y) / surface.height), 1)
}
func drawSurface() {
fill(Color(white: 0.11))
drawRect(surface.x, surface.y, surface.width, surface.height)
// A line per semitone, the C of each octave brighter.
for semitone in 0...Int(12 * octaves) {
let x = x(ofPitch: lowest + Double(semitone))
fill(Color(white: semitone % 12 == 0 ? 0.32 : 0.17))
drawRect(x - scale, surface.y, 2 * scale, surface.height)
}
for octave in 0...Int(octaves) {
fill(Color(white: 0.45))
textSize(16 * scale)
textAlign(.center, .top)
drawText("C\(3 + octave)", x(ofPitch: lowest + Double(12 * octave)), surface.bottomRight.y + 10 * scale)
}
}
/// A held note as a ribbon standing on the surface: at its pitch, as tall
/// as its slide, as wide and as bright as its pressure.
func drawRibbon(_ ribbon: Ribbon) {
let x = x(ofPitch: ribbon.pitch)
let top = surface.y + (1 - ribbon.slide) * surface.height
let widthNow = (14 + 44 * ribbon.pressure) * scale
let hue = (ribbon.pitch - lowest) / (12 * octaves)
let color = Color(hue: 0.55 + 0.35 * hue, saturation: 0.7, brightness: 0.55 + 0.45 * ribbon.pressure)
fill(color.withAlpha(0.28))
drawRect(x - widthNow, top, 2 * widthNow, surface.bottomRight.y - top)
fill(color)
drawRect(x - widthNow / 2, top, widthNow, surface.bottomRight.y - top)
drawCircle(x, top, widthNow * 0.7)
fill(Color(white: 0.92))
textSize(15 * scale)
textAlign(.center, .bottom)
let bend = ribbon.pitch - Double(ribbon.note)
let name = Pitch(Double(ribbon.note)).description
drawText(abs(bend) < 0.05 ? name : String(format: "%@ %+.1f", name, bend), x, top - widthNow * 0.7 - 6 * scale)
}
func drawLabels(ribbons: [Ribbon], idle: Bool) {
fill(Color(white: 0.95))
textAlign(.left, .top)
textSize(30 * scale)
drawText("Expression: each note bent, pressed, and slid on its own", 40 * scale, 44 * scale)
fill(Color(white: 0.55))
textSize(19 * scale)
let who = idle ? "the sketch's own hand" : finger != nil ? "the mouse, through the virtual source" : "a controller, through the wire"
let zone = midi.mpeZones.first.map { "zone: channel \($0.masterChannel) master, \($0.memberChannels.lowerBound) to \($0.memberChannels.upperBound) for notes" } ?? "waiting for the zone to cross the wire"
drawText("\(ribbons.count) note\(ribbons.count == 1 ? "" : "s") held, played by \(who) \(zone)", 40 * scale, 92 * scale)
drawText("across is pitch, up is slide, holding presses; `bowed` makes pressure the bow", 40 * scale, 122 * scale)
}
}