/* * Realtime beat tracker. * * 1. Onsets: every 16 ms, Hann-windowed 512-point FFT; novelty is the * half-wave rectified rise in log magnitude (spectral flux) plus the rise * in bass power. Log flux catches every instrument but rates a faint hat * like a loud kick; the uncompressed bass term keeps the kick on the beat. * 2. Tempo: every ~0.5 s, autocorrelate the last ~8 s of log flux alone * (bass would pull rock with kicks on 1 and 3 to half time), weight * lags with a log-normal prior around 120 BPM, and add the 2x lag so the * bar-level pulse backs up the beat. A new tempo has to clearly beat the * current one, several times in a row, before it is taken. * 3. Phase: every frame, find the offset whose comb of past beat positions * collects the most novelty. When that offset wraps back to zero a beat is * due, so beats keep coming through quiet bars instead of only on onsets. * 4. Lock: beats only go out while autocorrelation confidence times phase * salience (on-grid vs. average onset energy) is high. Speech and noise * can score on one of them but not both. Tuned on real music clips. * 5. Bar: each grid beat gets two downbeat cues, the bass hit on it and how * much new spectral energy its beat brings over the one before (chords * change on the one). Cues are averaged per bar position * for 3/4 and 4/4; the meter whose positions differ most wins, and its * strongest position is the downbeat. */ #include #include #include #include "beat_dsp.h" #define NV_MASK (BEAT_NV_LEN - 1) #define FLUX_MAX_BIN 192 // ~6 kHz; above that is mostly hats and hiss #define BASS_MAX_BIN 6 // <~190 Hz, kick drums #define BASS_WEIGHT 1.0f // bass power rise vs. full-band log flux #define LOG_GAMMA 100.0f // log(1 + g*|X|) compression #define TEMPO_EVERY 32 // frames between tempo estimates #define TEMPO_MIN_EXTRA 128 // history beyond the longest lag before estimating #define PRIOR_BPM 120.0f #define PRIOR_OCTAVES 1.0f // std dev of the tempo prior #define TEMPO_TOL 0.06f // relative change treated as the same tempo #define TEMPO_MEMORY_S 3.0f // time constant of the history fade #define TEMPO_MARGIN 1.5f // a new tempo must score this much better #define TEMPO_SWITCH 3 // estimates in a row needed to jump tempo #define COMB_BEATS 6 // past beats the phase comb looks at #define COMB_DECAY 0.88f #define PHASE_STICK 1.3f // bonus for the phase that follows on #define LOCK_STRENGTH 0.9f // confidence x salience to start emitting beats #define UNLOCK_STRENGTH 0.5f // ...and to keep going #define ONSET_K 1.5f // onset threshold, x running novelty mean #define ONSET_GAP_S 0.1f #define SALIENCE_S 2.0f // smoothing of the on-beat energy ratio #define ACTIVE_HOLD_S 2.0f // keep beating this long into silence #define KICK_WEIGHT 1.0f // downbeat cue: bass hit on the beat #define CHANGE_WEIGHT 1.0f // downbeat cue: spectral change into the beat #define SPAN_FRAC 1.0f // part of each beat compared for harmonic change #define BAR_RATE 0.08f // per-bar update rate of the position scores #define BAR_MARGIN 1.1f // a new downbeat position must score this much more #define METER_MARGIN 1.5f // ...and a new meter must stand out this much more #define METER3_BIAS 0.5f // weight on the 3/4 evidence #define BAR_SURE_ON 0.3f // contrast to call the bar position sure #define BAR_SURE_OFF 0.2f // ...and to stop #define BAR_WARMUP 8 // grid beats before guessing a bar position #define TAP_BOOST 2.0f // head start for a tapped beat 1, x the best score #define TAP_HOLD_BARS 16 // bars a tap holds the bar position static float s_window[BEAT_FFT_SIZE]; static float s_cos[BEAT_FFT_SIZE / 2], s_sin[BEAT_FFT_SIZE / 2]; static uint16_t s_bitrev[BEAT_FFT_SIZE]; static int8_t s_band[FLUX_MAX_BIN + 1]; // bin -> band, -1 if unused static float s_band_w[BEAT_BANDS]; // 1 / bins in band static const int s_meters[BEAT_METERS] = {3, 4}; static bool s_tables_ready; static void init_tables(void) { if (s_tables_ready) { return; } const int n = BEAT_FFT_SIZE; int bits = 0; while ((1 << bits) < n) { bits++; } for (int i = 0; i < n; i++) { s_window[i] = 0.5f - 0.5f * cosf(2.0f * (float)M_PI * i / n); int r = 0; for (int b = 0; b < bits; b++) { r |= ((i >> b) & 1) << (bits - 1 - b); } s_bitrev[i] = r; } for (int i = 0; i < n / 2; i++) { s_cos[i] = cosf(2.0f * (float)M_PI * i / n); s_sin[i] = -sinf(2.0f * (float)M_PI * i / n); } // Bands spaced evenly in log frequency from bin 2 (~60 Hz) up int count[BEAT_BANDS] = {0}; for (int k = 0; k <= FLUX_MAX_BIN; k++) { int b = k < 2 ? -1 : (int)(BEAT_BANDS * logf(k / 2.0f) / logf(FLUX_MAX_BIN / 2.0f)); s_band[k] = b < 0 ? -1 : b >= BEAT_BANDS ? BEAT_BANDS - 1 : b; if (s_band[k] >= 0) { count[s_band[k]]++; } } for (int b = 0; b < BEAT_BANDS; b++) { s_band_w[b] = count[b] ? 1.0f / count[b] : 0.0f; } s_tables_ready = true; } // In-place iterative radix-2 complex FFT static void fft(float *re, float *im) { const int n = BEAT_FFT_SIZE; for (int i = 0; i < n; i++) { int j = s_bitrev[i]; if (j > i) { float t = re[i]; re[i] = re[j]; re[j] = t; t = im[i]; im[i] = im[j]; im[j] = t; } } for (int len = 2; len <= n; len <<= 1) { int half = len >> 1, step = n / len; for (int start = 0; start < n; start += len) { for (int k = 0; k < half; k++) { float wr = s_cos[k * step], wi = s_sin[k * step]; int a = start + k, b = a + half; float tr = re[b] * wr - im[b] * wi; float ti = re[b] * wi + im[b] * wr; re[b] = re[a] - tr; im[b] = im[a] - ti; re[a] += tr; im[a] += ti; } } } } static inline float nv_at(const beat_dsp_t *d, uint32_t i) { return d->nv[i & NV_MASK]; } // Onsets can land a frame either side of the beat grid static inline float nv_max3(const beat_dsp_t *d, uint32_t i, uint32_t cur) { float v = fmaxf(nv_at(d, i), nv_at(d, i - 1)); return i < cur ? fmaxf(v, nv_at(d, i + 1)) : v; } void beat_dsp_init(beat_dsp_t *d, float sample_rate, float gate_rms, float lead_s) { init_tables(); memset(d, 0, sizeof(*d)); d->frame_rate = sample_rate / BEAT_HOP; d->gate_rms = gate_rms; d->lead_frames = (int)lrintf(lead_s * d->frame_rate); d->min_lag = (int)floorf(d->frame_rate * 60.0f / BEAT_MAX_BPM); d->max_lag = (int)ceilf(d->frame_rate * 60.0f / BEAT_MIN_BPM); // Start half a window in so the first frame comes after one hop d->in_fill = BEAT_FFT_SIZE - BEAT_HOP; d->meter = 4; } static float onset_strength(beat_dsp_t *d, bool *active, float *tempo_nv, float *bass_nv) { float sum_sq = 0.0f; for (int i = 0; i < BEAT_FFT_SIZE; i++) { float x = d->in[i]; sum_sq += x * x; d->re[i] = x * s_window[i]; d->im[i] = 0.0f; } *active = sqrtf(sum_sq / BEAT_FFT_SIZE) >= d->gate_rms; fft(d->re, d->im); float flux = 0.0f, bass = 0.0f; memset(d->band_now, 0, sizeof(d->band_now)); for (int k = 1; k <= FLUX_MAX_BIN; k++) { float mag2 = d->re[k] * d->re[k] + d->im[k] * d->im[k]; float l = log1pf(LOG_GAMMA * sqrtf(mag2)); flux += fmaxf(l - d->prev_log[k], 0.0f); d->prev_log[k] = l; if (s_band[k] >= 0) { d->band_now[s_band[k]] += l * s_band_w[s_band[k]]; } if (k <= BASS_MAX_BIN) { bass += mag2; } } float bass_rise = fmaxf(bass - d->prev_bass, 0.0f); d->prev_bass = bass; if (!*active) { flux = bass_rise = 0.0f; } // Each channel minus its ~0.5 s running mean, so sustained texture // doesn't count, and over it, so neither channel's units dominate const float a = 1.0f / (0.5f * d->frame_rate); d->flux_mean += a * (flux - d->flux_mean); d->bass_mean += a * (bass_rise - d->bass_mean); float full = fmaxf(flux - d->flux_mean, 0.0f) / (d->flux_mean + 1e-6f); *tempo_nv = full; *bass_nv = fmaxf(bass_rise - d->bass_mean, 0.0f) / (d->bass_mean + 1e-12f); return full + BASS_WEIGHT * *bass_nv; } static void estimate_tempo(beat_dsp_t *d) { int n = d->t < BEAT_NV_LEN ? (int)d->t : BEAT_NV_LEN; int top = 2 * (d->max_lag + 1); if (n < top + TEMPO_MIN_EXTRA || top >= BEAT_AC_LEN) { return; } // Oldest-first, mean-removed copy of the history (re[] is free here) float *x = d->re, mean = 0.0f; for (int i = 0; i < n; i++) { x[i] = d->tempo_nv[(d->t - n + i) & NV_MASK]; mean += x[i]; } mean /= n; // Fade older frames so a tempo change wins within a few seconds const float fade = expf(-1.0f / (TEMPO_MEMORY_S * d->frame_rate)); float w = 1.0f; for (int i = n - 1; i >= 0; i--) { x[i] = (x[i] - mean) * w; w *= fade; } for (int l = 0; l <= top; l++) { float s = 0.0f; for (int i = l; i < n; i++) { s += x[i] * x[i - l]; } d->ac[l] = s / (n - l); } if (d->ac[0] <= 1e-9f) { d->confidence *= 0.5f; return; } // score[l] for l in [min_lag-1, max_lag+1], kept in im[] for interpolation float *score = d->im; const float l0 = d->frame_rate * 60.0f / PRIOR_BPM; int best = d->min_lag; for (int l = d->min_lag - 1; l <= d->max_lag + 1; l++) { float o = log2f((float)l / l0) / PRIOR_OCTAVES; // The 2x lag backs up a beat but can't carry it: in 3/4, 2/3 of the // beat is barely periodic yet its double is a whole bar score[l] = expf(-0.5f * o * o) * (d->ac[l] + 0.5f * fminf(d->ac[2 * l], fmaxf(d->ac[l], 0.0f))); if (l >= d->min_lag && l <= d->max_lag && score[l] > score[best]) { best = l; } } // Hold the current tempo unless another clearly beats it; real music // often has a 3:2 or 2:1 alternative that is nearly as periodic if (d->period > 0.0f) { int lo = (int)fmaxf(floorf(d->period * (1.0f - TEMPO_TOL)), d->min_lag); int hi = (int)fminf(ceilf(d->period * (1.0f + TEMPO_TOL)), d->max_lag); int near = lo; for (int l = lo + 1; l <= hi; l++) { if (score[l] > score[near]) { near = l; } } if (score[near] > 0.0f && score[best] < TEMPO_MARGIN * score[near]) { best = near; } } float a = score[best - 1], b = score[best], c = score[best + 1]; float denom = a - 2.0f * b + c; float period = best + (denom < 0.0f ? 0.5f * (a - c) / denom : 0.0f); float conf = fminf(fmaxf(d->ac[best] / d->ac[0], 0.0f), 1.0f); d->confidence += 0.3f * (conf - d->confidence); if (d->period == 0.0f) { d->period = period; // tentative; locking decides whether beats go out } else if (fabsf(period / d->period - 1.0f) < TEMPO_TOL) { d->period += 0.2f * (period - d->period); d->pending_count = 0; } else if (d->pending_count > 0 && fabsf(period / d->pending_period - 1.0f) < TEMPO_TOL) { if (++d->pending_count >= TEMPO_SWITCH) { d->period = period; d->pending_count = 0; } } else { d->pending_period = period; d->pending_count = 1; } } // Frames since the most recent beat according to the comb over past beats static int best_phase(beat_dsp_t *d, uint32_t cur) { const float p = d->period; const int np = (int)p; const int expect = (d->prev_phase - d->lead_frames + 1 + 2 * np) % np; int best = 0; float best_score = -1.0f, total = 0.0f; for (int phi = 0; phi < np; phi++) { float s = 0.0f, w = 1.0f; for (int k = 0; k < COMB_BEATS; k++) { float back = phi + k * p; if (back >= BEAT_NV_LEN - 2 || back > cur) { break; } s += w * nv_max3(d, cur - (uint32_t)lrintf(back), cur); w *= COMB_DECAY; } total += s; if (abs(phi - expect) <= 1) { s *= PHASE_STICK; } if (s > best_score) { best_score = s; best = phi; } } // How much more onset energy sits on the beat grid than off it float salience = total > 0.0f ? best_score * np / total : 0.0f; // Plain running mean until the window fills, so locking isn't held back float span = fminf(SALIENCE_S * d->frame_rate, (float)++d->salience_n); d->salience += (salience - d->salience) / span; return best; } // Add one cue value to the bar position of grid beat n, for every meter static void bar_cue(beat_dsp_t *d, uint32_t n, float v) { for (int m = 0; m < BEAT_METERS; m++) { float *s = &d->bar_score[m][n % s_meters[m]]; *s += BAR_RATE * (v - *s); } } // Pick the downbeat position per meter, then the meter, with hysteresis static void bar_decide(beat_dsp_t *d) { float contrast[BEAT_METERS]; for (int m = 0; m < BEAT_METERS; m++) { const int len = s_meters[m]; const float *s = d->bar_score[m]; int best = 0; float mean = 0.0f; for (int i = 0; i < len; i++) { mean += s[i] / len; if (s[i] > s[best]) { best = i; } } // A tap holds the bar for a while; the audio only takes over after if (d->tap_hold == 0 && s[best] > BAR_MARGIN * s[d->down_pos[m]]) { d->down_pos[m] = best; } contrast[m] = mean > 0.0f ? s[d->down_pos[m]] / mean - 1.0f : 0.0f; } // Most music is in 4/4, so 3/4 has to make a stronger case contrast[0] *= METER3_BIAS; int cur = d->meter == 3 ? 0 : 1, other = 1 - cur; if (d->tap_hold == 0 && contrast[other] > METER_MARGIN * contrast[cur]) { cur = other; } d->meter = s_meters[cur]; d->bar_contrast = contrast[cur]; } // Runs a couple of frames after grid beat pending_n landed static void analyse_beat(beat_dsp_t *d) { uint32_t beat = d->pending_frame - 2; float kick = 0.0f; for (uint32_t i = beat - 1; i <= beat + 2; i++) { kick = fmaxf(kick, d->bass_nv[i & NV_MASK]); } d->kick_mean += 0.1f * (kick - d->kick_mean); bar_cue(d, d->pending_n, KICK_WEIGHT * kick / (d->kick_mean + 1e-6f)); bar_decide(d); d->pending = false; } // A beat just landed: close the spectrum of the previous beat and score how // much new energy it brought over the one before as a bar start static void close_span(beat_dsp_t *d) { if (d->band_frames > 0) { float change = 0.0f; for (int b = 0; b < BEAT_BANDS; b++) { float v = d->band_acc[b] / d->band_frames; // Rises only: a chord fading out late in the bar isn't a change change += fmaxf(v - d->prev_span[b], 0.0f); d->prev_span[b] = v; d->band_acc[b] = 0.0f; } d->band_frames = 0; d->change_mean += 0.1f * (change - d->change_mean); if (d->span_n > 0) { // the first span has nothing to compare with bar_cue(d, d->span_n, CHANGE_WEIGHT * change / (d->change_mean + 1e-6f)); } } d->span_n = d->pending_n; } static int process_frame(beat_dsp_t *d) { int events = 0; bool active; float tempo_nv, bass_nv; float nv = onset_strength(d, &active, &tempo_nv, &bass_nv); uint32_t cur = d->t++; d->nv[cur & NV_MASK] = nv; d->tempo_nv[cur & NV_MASK] = tempo_nv; d->bass_nv[cur & NV_MASK] = bass_nv; if (d->pending && cur == d->pending_frame - 2) { close_span(d); } if (d->period > 0.0f && cur - (d->pending_frame - 2) < SPAN_FRAC * d->period) { for (int b = 0; b < BEAT_BANDS; b++) { d->band_acc[b] += d->band_now[b]; } d->band_frames++; } if (d->pending && cur >= d->pending_frame) { analyse_beat(d); } if (active) { d->last_active = cur; } // Onset: previous frame is a local peak above the adaptive threshold if (cur >= 2) { uint32_t c = cur - 1; float v = nv_at(d, c); if (v > nv_at(d, c - 1) && v >= nv && v > ONSET_K * d->nv_mean + 1e-3f && (d->last_onset == 0 || c - d->last_onset >= ONSET_GAP_S * d->frame_rate)) { d->last_onset = c; events |= BEAT_EV_ONSET; } } d->nv_mean += (nv - d->nv_mean) / d->frame_rate; if (d->t % TEMPO_EVERY == 0) { estimate_tempo(d); } if (d->period == 0.0f) { return events; } bool audible = cur - d->last_active < ACTIVE_HOLD_S * d->frame_rate; // Speech and noise can score on one measure, music with a beat on both float strength = d->confidence * d->salience; d->locked = audible && strength >= (d->locked ? UNLOCK_STRENGTH : LOCK_STRENGTH); // Shifting the phase by the lead makes it wrap that many frames early int np = (int)d->period; int phase = (best_phase(d, cur) + d->lead_frames) % np; // Phase dropping by more than half a beat means it wrapped: beat due now if (phase + d->period / 2 < d->prev_phase && cur - d->last_beat >= d->period / 2) { if (d->pending) { if (cur < d->pending_frame - 2) { close_span(d); } analyse_beat(d); } uint32_t n = d->grid_n++; int pos = (int)((n + 4 * 3 - (uint32_t)d->down_pos[d->meter == 3 ? 0 : 1]) % d->meter); // Always offer a best guess, but only call downbeats when sure if (d->tap_hold > 0) { d->tap_hold--; d->bar_beat = pos + 1; d->bar_sure = true; } else { d->bar_beat = n >= BAR_WARMUP ? pos + 1 : 0; d->bar_sure = d->bar_contrast >= (d->bar_sure ? BAR_SURE_OFF : BAR_SURE_ON); } if (d->locked) { events |= BEAT_EV_BEAT; if (d->bar_beat == 1 && d->bar_sure) { events |= BEAT_EV_DOWNBEAT; } } d->last_beat = cur; // The beat lands lead_frames from now; look at it 2 frames after d->pending_frame = cur + d->lead_frames + 2; d->pending_n = n; d->pending = true; } d->prev_phase = phase; return events; } void beat_dsp_tap(beat_dsp_t *d, float frames_ago) { if (d->period <= 0.0f || d->grid_n == 0) { return; } // Grid beat grid_n - 1 lands lead_frames after it was announced; snap the // tap to that beat or the one either side, whichever is nearest float tap = (float)d->t - frames_ago; float last = (float)(d->last_beat + d->lead_frames); int k = (int)lrintf((tap - last) / d->period); k = k < -1 ? -1 : k > 1 ? 1 : k; int32_t n = (int32_t)d->grid_n - 1 + k; n = n < 0 ? 0 : n; for (int m = 0; m < BEAT_METERS; m++) { float top = 1e-3f; for (int i = 0; i < s_meters[m]; i++) { top = fmaxf(top, d->bar_score[m][i]); } int pos = n % s_meters[m]; d->bar_score[m][pos] = TAP_BOOST * top; d->down_pos[m] = pos; } d->tap_hold = TAP_HOLD_BARS * d->meter; d->bar_sure = true; } void beat_dsp_feed(beat_dsp_t *d, const float *x, size_t n, beat_dsp_cb_t cb, void *ctx) { while (n > 0) { size_t take = BEAT_FFT_SIZE - d->in_fill; take = take < n ? take : n; memcpy(d->in + d->in_fill, x, take * sizeof(float)); d->in_fill += take; x += take; n -= take; if (d->in_fill < BEAT_FFT_SIZE) { break; } int events = process_frame(d); memmove(d->in, d->in + BEAT_HOP, (BEAT_FFT_SIZE - BEAT_HOP) * sizeof(float)); d->in_fill = BEAT_FFT_SIZE - BEAT_HOP; if (events && cb) { cb(ctx, events, d); } } }