From 5c792eb86970f86a5039475790e39b45e87f703c Mon Sep 17 00:00:00 2001 From: Frank Date: Sun, 28 Aug 2022 13:13:25 +0200 Subject: [PATCH 1/3] some cleanup and re-grouping of variables - put variables with same context next to each other. - removed a few vars that are not needed any more. - replaced "16" by a more descriptive constant --- usermods/audioreactive/audio_reactive.h | 108 ++++++++++++------------ 1 file changed, 53 insertions(+), 55 deletions(-) diff --git a/usermods/audioreactive/audio_reactive.h b/usermods/audioreactive/audio_reactive.h index 826ad7f4..8bc10553 100644 --- a/usermods/audioreactive/audio_reactive.h +++ b/usermods/audioreactive/audio_reactive.h @@ -85,14 +85,11 @@ const float agcSampleSmooth[AGC_NUM_PRESETS] = { 1/12.f, 1/6.f, 1/16.f}; // static AudioSource *audioSource = nullptr; static volatile bool disableSoundProcessing = false; // if true, sound processing (FFT, filters, AGC) will be suspended. "volatile" as its shared between tasks. +// audioreactive variables shared with FFT task static float micDataReal = 0.0f; // MicIn data with full 24bit resolution - lowest 8bit after decimal point -static float sampleReal = 0.0f; // "sampleRaw" as float, to provide bits that are lost otherwise (before amplification by sampleGain or inputLevel). Needed for AGC. static float multAgc = 1.0f; // sample * multAgc = sampleAgc. Our AGC multiplier +static float sampleAvg = 0.0f; // Smoothed Average sample - sampleAvg < 1 means "quiet" (simple noise gate) -static int16_t sampleRaw = 0; // Current sample. Must only be updated ONCE!!! (amplified mic value by sampleGain and inputLevel) -static int16_t rawSampleAgc = 0; // not smoothed AGC sample -static float sampleAvg = 0.0f; // Smoothed Average sampleRaw -static float sampleAgc = 0.0f; // Smoothed AGC sample //////////////////// // Begin FFT Code // @@ -105,21 +102,23 @@ static float sampleAgc = 0.0f; // Smoothed AGC sample #endif #include "arduinoFFT.h" -// FFT Variables +// FFT Output variables shared with animations +#define NUM_GEQ_CHANNELS 16 // number of frequency channels. Don't change !! +static float FFT_MajorPeak = 1.0f; // FFT: strongest (peak) frequency +static float FFT_Magnitude = 0.0f; // FFT: magintude peak frequency +static uint8_t fftResult[NUM_GEQ_CHANNELS]= {0};// Our calculated freq. channel result table to be used by effects + +// FFT Constants constexpr uint16_t samplesFFT = 512; // Samples in an FFT batch - This value MUST ALWAYS be a power of 2 constexpr uint16_t samplesFFT_2 = 256; // meaningfull part of FFT results - only the "lower half" contains useful information. -static float FFT_MajorPeak = 1.0f; -static float FFT_Magnitude = 0.0f; - // These are the input and output vectors. Input vectors receive computed results from FFT. -static float vReal[samplesFFT] = {0.0f}; -static float vImag[samplesFFT] = {0.0f}; +static float vReal[samplesFFT] = {0.0f}; // FFT sample inputs / freq output - these are our raw result bins +static float vImag[samplesFFT] = {0.0f}; // imaginary parts static float fftBin[samplesFFT_2] = {0.0f}; // the following are observed values, supported by a bit of "educated guessing" //#define FFT_DOWNSCALE 0.65f // 20kHz - downscaling factor for FFT results - "Flat-Top" window @20Khz, old freq channels - #define FFT_DOWNSCALE 0.46f // downscaling factor for FFT results - for "Flat-Top" window @22Khz, new freq channels #define LOG_256 5.54517744 @@ -129,12 +128,11 @@ static float windowWeighingFactors[samplesFFT] = {0.0f}; // Try and normalize fftBin values to a max of 4096, so that 4096/16 = 256. // Oh, and bins 0,1,2 are no good, so we'll zero them out. -static float fftCalc[16] = {0.0f}; -static uint8_t fftResult[16] = {0}; // Our calculated result table, which we feed to the animations. +static float fftCalc[NUM_GEQ_CHANNELS] = {0.0f}; +static float fftAvg[NUM_GEQ_CHANNELS] = {0.0f}; // Calculated frequency channel results, with smoothing (used if dynamics limiter is ON) #ifdef SR_DEBUG -static float fftResultMax[16] = {0.0f}; // A table used for testing to determine how our post-processing is working. +static float fftResultMax[NUM_GEQ_CHANNELS] = {0.0f}; // A table used for testing to determine how our post-processing is working. #endif -static float fftAvg[16] = {0.0f}; #ifdef WLED_DEBUG static unsigned long fftTime = 0; @@ -142,7 +140,7 @@ static unsigned long sampleTime = 0; #endif // Table of multiplication factors so that we can even out the frequency response. -static float fftResultPink[16] = { 1.70f, 1.71f, 1.73f, 1.78f, 1.68f, 1.56f, 1.55f, 1.63f, 1.79f, 1.62f, 1.80f, 2.06f, 2.47f, 3.35f, 6.83f, 9.55f }; +static float fftResultPink[NUM_GEQ_CHANNELS] = { 1.70f, 1.71f, 1.73f, 1.78f, 1.68f, 1.56f, 1.55f, 1.63f, 1.79f, 1.62f, 1.80f, 2.06f, 2.47f, 3.35f, 6.83f, 9.55f }; // Create FFT object #ifdef UM_AUDIOREACTIVE_USE_NEW_FFT @@ -292,14 +290,14 @@ void FFTcode(void * parameter) // don't use the last bins from 216 to 255. They are usually contaminated by aliasing (aka noise) #endif } else { // noise gate closed - just decay old values - for (int i=0; i < 16; i++) { + for (int i=0; i < NUM_GEQ_CHANNELS; i++) { fftCalc[i] *= 0.85f; // decay to zero if (fftCalc[i] < 4.0f) fftCalc[i] = 0.0f; } } // post-processing of frequency channels (pink noise adjustment, AGC, smooting, scaling) - for (int i=0; i < 16; i++) { + for (int i=0; i < NUM_GEQ_CHANNELS; i++) { if (sampleAvg > 1) { // noise gate open // Adjustment for frequency curves. @@ -377,10 +375,9 @@ void FFTcode(void * parameter) unsigned long fftTimeInMillis = ((esp_timer_get_time() - start) +500ULL) / 1000ULL; // "+500" to ensure proper rounding fftTime = (fftTimeInMillis*3 + fftTime*7)/10; // smooth } -#endif - - } // for(;;) -} // FFTcode() +#endif + } // for(;;)ever +} // FFTcode() task end //class name. Use something descriptive and leave the ": public Usermod" part :) @@ -453,39 +450,41 @@ class AudioReactive : public Usermod { double FFT_MajorPeak; // 08 Bytes }; - WiFiUDP fftUdp; - // set your config variables to their boot default value (this can also be done in readFromConfig() or a constructor if you prefer) bool enabled = false; bool initDone = false; - const uint16_t delayMs = 10; // I don't want to sample too often and overload WLED + // variables for UDP sound sync + WiFiUDP fftUdp; // UDP object for sound sync (from WiFi UDP, not Async UDP!) + bool udpSyncConnected = false;// UDP connection status -> true if connected to multicast group + unsigned long lastTime = 0; // last time of running UDP Microphone Sync + const uint16_t delayMs = 10; // I don't want to sample too often and overload WLED + uint16_t audioSyncPort= 11988;// default port for UDP sound sync + + // used for AGC + int last_soundAgc = -1; // used to detect AGC mode change (for resetting AGC internal error buffers) + double control_integrated = 0.0; // persistent across calls to agcAvg(); "integrator control" = accumulated error + + // variables used by getSample() and agcAvg() + int16_t micIn = 0; // Current sample starts with negative values and large values, which is why it's 16 bit signed + double sampleMax = 0.0; // Max sample over a few seconds. Needed for AGC controler. + float micLev = 0.0f; // Used to convert returned value to have '0' as minimum. A leveller + float expAdjF = 0.0f; // Used for exponential filter. + float sampleReal = 0.0f; // "sampleRaw" as float, to provide bits that are lost otherwise (before amplification by sampleGain or inputLevel). Needed for AGC. + int16_t sampleRaw = 0; // Current sample. Must only be updated ONCE!!! (amplified mic value by sampleGain and inputLevel) + int16_t rawSampleAgc = 0; // not smoothed AGC sample + float sampleAgc = 0.0f; // Smoothed AGC sample + // variables used in effects - uint8_t maxVol = 10; // Reasonable value for constant volume for 'peak detector', as it won't always trigger (deprecated) - uint8_t binNum = 8; // Used to select the bin for FFT based beat detection (deprecated) - bool samplePeak = 0; // Boolean flag for peak. Responding routine must reset this flag float volumeSmth = 0.0f; // either sampleAvg or sampleAgc depending on soundAgc; smoothed sample int16_t volumeRaw = 0; // either sampleRaw or rawSampleAgc depending on soundAgc float my_magnitude =0.0f; // FFT_Magnitude, scaled by multAgc - - bool udpSamplePeak = 0; // Boolean flag for peak. Set at the same tiem as samplePeak, but reset by transmitAudioData - int16_t micIn = 0; // Current sample starts with negative values and large values, which is why it's 16 bit signed - double sampleMax = 0.0; // Max sample over a few seconds. Needed for AGC controler. - uint32_t timeOfPeak = 0; - unsigned long lastTime = 0; // last time of running UDP Microphone Sync - float micLev = 0.0f; // Used to convert returned value to have '0' as minimum. A leveller - float expAdjF = 0.0f; // Used for exponential filter. - - bool udpSyncConnected = false; - uint16_t audioSyncPort = 11988; - - // used for AGC - uint8_t lastMode = 0; // last known effect mode - int last_soundAgc = -1; - double control_integrated = 0.0; // persistent across calls to agcAvg(); "integrator control" = accumulated error - unsigned long last_update_time = 0; - unsigned long last_kick_time = 0; - uint8_t last_user_inputLevel = 0; + // peak detection + uint8_t maxVol = 10; // Reasonable value for constant volume for 'peak detector', as it won't always trigger (deprecated) + uint8_t binNum = 8; // Used to select the bin for FFT based beat detection (deprecated) + bool samplePeak = false; // Boolean flag for peak. Responding routine may reset this flag. Auto-reset after strip.getMinShowDelay() + bool udpSamplePeak = false; // Boolean flag for peak. Set at the same tiem as samplePeak, but reset by transmitAudioData + unsigned long timeOfPeak = 0; // time of last sample peak detection // used to feed "Info" Page unsigned long last_UDPTime = 0; // time of last valid UDP sound sync datapacket @@ -525,7 +524,7 @@ class AudioReactive : public Usermod { #ifdef FFT_SAMPLING_LOG #if 0 - for(int i=0; i<16; i++) { + for(int i=0; i maxVal) maxVal = fftResult[i]; if(fftResult[i] < minVal) minVal = fftResult[i]; } - for(int i = 0; i < 16; i++) { + for(int i = 0; i < NUM_GEQ_CHANNELS; i++) { Serial.print(i); Serial.print(":"); Serial.printf("%04ld ", map(fftResult[i], 0, (scaleValuesFromCurrentMaxVal ? maxVal : defaultScalingFromHighValue), (mapValuesToPlotterSpace*i*scalingToHighValue)+0, (mapValuesToPlotterSpace*i*scalingToHighValue)+scalingToHighValue-1)); } @@ -795,7 +794,7 @@ class AudioReactive : public Usermod { udpSamplePeak = false; // Reset udpSamplePeak after we've transmitted it transmitData.reserved1 = 0; - for (int i = 0; i < 16; i++) { + for (int i = 0; i < NUM_GEQ_CHANNELS; i++) { transmitData.fftResult[i] = (uint8_t)constrain(fftResult[i], 0, 254); } @@ -855,7 +854,7 @@ class AudioReactive : public Usermod { } //These values are only available on the ESP32 - for (int i = 0; i < 16; i++) fftResult[i] = receivedPacket->fftResult[i]; + for (int i = 0; i < NUM_GEQ_CHANNELS; i++) fftResult[i] = receivedPacket->fftResult[i]; my_magnitude = fmaxf(receivedPacket->FFT_Magnitude, 0.0f); FFT_Magnitude = my_magnitude; @@ -1036,7 +1035,6 @@ class AudioReactive : public Usermod { // Only run the sampling code IF we're not in Receive mode or realtime mode if (!(audioSyncEnabled & 0x02) && !disableSoundProcessing) { - bool agcEffect = false; if (soundAgc > AGC_NUM_PRESETS) soundAgc = 0; // make sure that AGC preset is valid (to avoid array bounds violation) unsigned long t_now = millis(); // remember current time @@ -1138,7 +1136,7 @@ class AudioReactive : public Usermod { memset(fftCalc, 0, sizeof(fftCalc)); memset(fftAvg, 0, sizeof(fftAvg)); memset(fftResult, 0, sizeof(fftResult)); - for(int i=(init?0:1); i<16; i+=2) fftResult[i] = 16; // make a tiny pattern + for(int i=(init?0:1); i Date: Sun, 28 Aug 2022 13:33:48 +0200 Subject: [PATCH 2/3] GEQ: use full frequency range also when user wants less than 16 bands. Previously when users selected fewer bands (like 4 instead of 16), only the lowest freq channels were displayed. --- wled00/FX.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/wled00/FX.cpp b/wled00/FX.cpp index a57aaafb..3a1f7dd3 100644 --- a/wled00/FX.cpp +++ b/wled00/FX.cpp @@ -7109,6 +7109,7 @@ uint16_t mode_2DGEQ(void) { // By Will Tatam. Code reduction by Ewoud Wijma. for (int x=0; x < cols; x++) { uint8_t band = map(x, 0, cols-1, 0, NUM_BANDS - 1); + if (NUM_BANDS < 16) band = map(band, 0, NUM_BANDS - 1, 0, 15); // always use full range. comment out this line to get the previous behaviour. band = constrain(band, 0, 15); uint16_t colorIndex = band * 17; uint16_t barHeight = map(fftResult[band], 0, 255, 0, rows); // do not subtract -1 from rows here From 64970772c7de733eb838e411c582042a9cfc93f7 Mon Sep 17 00:00:00 2001 From: Frank Date: Sun, 28 Aug 2022 16:26:34 +0200 Subject: [PATCH 3/3] optimization, and moving peak detection into own function - save 1K of RAM by optimizing out fftBin[]. - moved several copies of the peak reset code into a single function - moved peak detection out of getSample(). - call peak detection function as last step of FFTcode. More optimal, and we can be sure that fresh FFT result are available. Peak detection/reset are now called from both tasks, so I had to move some peak-related vars out of AudioReactive class and make them global (static). --- usermods/audioreactive/audio_reactive.h | 91 ++++++++++++++----------- 1 file changed, 51 insertions(+), 40 deletions(-) diff --git a/usermods/audioreactive/audio_reactive.h b/usermods/audioreactive/audio_reactive.h index 8bc10553..bd3b6ec1 100644 --- a/usermods/audioreactive/audio_reactive.h +++ b/usermods/audioreactive/audio_reactive.h @@ -90,6 +90,15 @@ static float micDataReal = 0.0f; // MicIn data with full 24bit re static float multAgc = 1.0f; // sample * multAgc = sampleAgc. Our AGC multiplier static float sampleAvg = 0.0f; // Smoothed Average sample - sampleAvg < 1 means "quiet" (simple noise gate) +// peak detection +static bool samplePeak = false; // Boolean flag for peak - used in effects. Responding routine may reset this flag. Auto-reset after strip.getMinShowDelay() +static uint8_t maxVol = 10; // Reasonable value for constant volume for 'peak detector', as it won't always trigger (deprecated) +static uint8_t binNum = 8; // Used to select the bin for FFT based beat detection (deprecated) +static bool udpSamplePeak = false; // Boolean flag for peak. Set at the same tiem as samplePeak, but reset by transmitAudioData +static unsigned long timeOfPeak = 0; // time of last sample peak detection. +static void detectSamplePeak(void); // peak detection function (needs scaled FFT reasults in vReal[]) +static void autoResetPeak(void); // peak auto-reset function + //////////////////// // Begin FFT Code // @@ -105,7 +114,7 @@ static float sampleAvg = 0.0f; // Smoothed Average sample - sam // FFT Output variables shared with animations #define NUM_GEQ_CHANNELS 16 // number of frequency channels. Don't change !! static float FFT_MajorPeak = 1.0f; // FFT: strongest (peak) frequency -static float FFT_Magnitude = 0.0f; // FFT: magintude peak frequency +static float FFT_Magnitude = 0.0f; // FFT: volume (magnitude) of peak frequency static uint8_t fftResult[NUM_GEQ_CHANNELS]= {0};// Our calculated freq. channel result table to be used by effects // FFT Constants @@ -115,7 +124,6 @@ constexpr uint16_t samplesFFT_2 = 256; // meaningfull part of FFT resul // These are the input and output vectors. Input vectors receive computed results from FFT. static float vReal[samplesFFT] = {0.0f}; // FFT sample inputs / freq output - these are our raw result bins static float vImag[samplesFFT] = {0.0f}; // imaginary parts -static float fftBin[samplesFFT_2] = {0.0f}; // the following are observed values, supported by a bit of "educated guessing" //#define FFT_DOWNSCALE 0.65f // 20kHz - downscaling factor for FFT results - "Flat-Top" window @20Khz, old freq channels @@ -159,7 +167,7 @@ static float mapf(float x, float in_min, float in_max, float out_min, float out_ static float fftAddAvg(int from, int to) { float result = 0.0f; for (int i = from; i <= to; i++) { - result += fftBin[i]; + result += vReal[i]; } return result / float(to - from + 1); } @@ -235,9 +243,9 @@ void FFTcode(void * parameter) #endif FFT_MajorPeak = constrain(FFT_MajorPeak, 1.0f, 11025.0f); // restrict value to range expected by effects - for (int i = 0; i < samplesFFT_2; i++) { // Values for bins 0 and 1 are WAY too large. Might as well start at 3. + for (int i = 0; i < samplesFFT; i++) { float t = fabsf(vReal[i]); // just to be sure - values in fft bins should be positive any way - fftBin[i] = t / 16.0f; // Reduce magnitude. Want end result to be linear and ~4096 max. + vReal[i] = t / 16.0f; // Reduce magnitude. Want end result to be scaled linear and ~4096 max. } // for() // mapping of FFT result bins to frequency channels @@ -375,11 +383,40 @@ void FFTcode(void * parameter) unsigned long fftTimeInMillis = ((esp_timer_get_time() - start) +500ULL) / 1000ULL; // "+500" to ensure proper rounding fftTime = (fftTimeInMillis*3 + fftTime*7)/10; // smooth } -#endif +#endif + // run peak detection + autoResetPeak(); + detectSamplePeak(); + } // for(;;)ever } // FFTcode() task end +//////////////////// +// Peak detection // +//////////////////// + +// peak detection is called from FFT task when vReal[] contains valid FFT results +static void detectSamplePeak(void) { + // Poor man's beat detection by seeing if sample > Average + some value. + if ((sampleAvg > 1) && (maxVol > 0) && (binNum > 1) && (vReal[binNum] > maxVol) && ((millis() - timeOfPeak) > 100)) { + // This goes through ALL of the 255 bins - but ignores stupid settings + // Then we got a peak, else we don't. The peak has to time out on its own in order to support UDP sound sync. + samplePeak = true; + timeOfPeak = millis(); + udpSamplePeak = true; + } +} + +static void autoResetPeak(void) { + uint16_t MinShowDelay = MAX(50, strip.getMinShowDelay()); // Fixes private class variable compiler error. Unsure if this is the correct way of fixing the root problem. -THATDONFC + if (millis() - timeOfPeak > MinShowDelay) { // Auto-reset of samplePeak after a complete frame has passed. + samplePeak = false; + if (audioSyncEnabled == 0) udpSamplePeak = false; // this is normally reset by transmitAudioData + } +} + + //class name. Use something descriptive and leave the ": public Usermod" part :) class AudioReactive : public Usermod { @@ -479,12 +516,6 @@ class AudioReactive : public Usermod { float volumeSmth = 0.0f; // either sampleAvg or sampleAgc depending on soundAgc; smoothed sample int16_t volumeRaw = 0; // either sampleRaw or rawSampleAgc depending on soundAgc float my_magnitude =0.0f; // FFT_Magnitude, scaled by multAgc - // peak detection - uint8_t maxVol = 10; // Reasonable value for constant volume for 'peak detector', as it won't always trigger (deprecated) - uint8_t binNum = 8; // Used to select the bin for FFT based beat detection (deprecated) - bool samplePeak = false; // Boolean flag for peak. Responding routine may reset this flag. Auto-reset after strip.getMinShowDelay() - bool udpSamplePeak = false; // Boolean flag for peak. Set at the same tiem as samplePeak, but reset by transmitAudioData - unsigned long timeOfPeak = 0; // time of last sample peak detection // used to feed "Info" Page unsigned long last_UDPTime = 0; // time of last valid UDP sound sync datapacket @@ -728,24 +759,6 @@ class AudioReactive : public Usermod { if (sampleMax < 0.5f) sampleMax = 0.0f; sampleAvg = ((sampleAvg * 15.0f) + sampleAdj) / 16.0f; // Smooth it out over the last 16 samples. - - // Fixes private class variable compiler error. Unsure if this is the correct way of fixing the root problem. -THATDONFC - uint16_t MinShowDelay = strip.getMinShowDelay(); - - if (millis() - timeOfPeak > MinShowDelay) { // Auto-reset of samplePeak after a complete frame has passed. - samplePeak = false; - udpSamplePeak = false; - } - //if (userVar1 == 0) samplePeak = 0; - - // Poor man's beat detection by seeing if sample > Average + some value. - if ((maxVol > 0) && (binNum > 1) && (fftBin[binNum] > maxVol) && (millis() > (timeOfPeak + 100))) { - // This goes through ALL of the 255 bins - but ignores stupid settings - // Then we got a peak, else we don't. The peak has to time out on its own in order to support UDP sound sync. - samplePeak = true; - timeOfPeak = millis(); - udpSamplePeak = true; - } } // getSample() @@ -838,13 +851,7 @@ class AudioReactive : public Usermod { sampleAgc = volumeSmth; multAgc = 1.0f; - // auto-reset sample peak. Need to do it here, because getSample() is not running - uint16_t MinShowDelay = strip.getMinShowDelay(); - if (millis() - timeOfPeak > MinShowDelay) { // Auto-reset of samplePeak after a complete frame has passed. - samplePeak = false; - udpSamplePeak = false; - } - //if (userVar1 == 0) samplePeak = 0; + autoResetPeak(); // Only change samplePeak IF it's currently false. // If it's true already, then the animation still needs to respond. if (!samplePeak) { @@ -1066,9 +1073,11 @@ class AudioReactive : public Usermod { if (soundAgc) my_magnitude *= multAgc; if (volumeSmth < 1 ) my_magnitude = 0.001f; // noise gate closed - mute - limitSampleDynamics(); // optional - makes volumeSmth very smooth and fluent - } + limitSampleDynamics(); + } // if (!disableSoundProcessing) + autoResetPeak(); // auto-reset sample peak after strip minShowDelay + if (!udpSyncConnected) udpSamplePeak = false; // reset UDP samplePeak while UDP is unconnected // UDP Microphone Sync - receive mode if ((audioSyncEnabled & 0x02) && udpSyncConnected) { @@ -1091,7 +1100,7 @@ class AudioReactive : public Usermod { } #endif - // peak sample from last 5 seconds + // Info Page: keep max sample from last 5 seconds if ((millis() - sampleMaxTimer) > CYCLE_SAMPLEMAX) { sampleMaxTimer = millis(); maxSample5sec = (0.15 * maxSample5sec) + 0.85 *((soundAgc) ? sampleAgc : sampleAvg); // reset, and start with some smoothing @@ -1099,6 +1108,7 @@ class AudioReactive : public Usermod { } else { if ((sampleAvg >= 1)) maxSample5sec = fmaxf(maxSample5sec, (soundAgc) ? rawSampleAgc : sampleRaw); // follow maximum volume } + //UDP Microphone Sync - transmit mode if ((audioSyncEnabled & 0x01) && (millis() - lastTime > 20)) { // Only run the transmit code IF we're in Transmit mode @@ -1138,6 +1148,7 @@ class AudioReactive : public Usermod { memset(fftResult, 0, sizeof(fftResult)); for(int i=(init?0:1); i