3D Audio GSoC:
authorJoerg Mueller <nexyon@gmail.com>
Tue, 9 Aug 2011 14:10:32 +0000 (14:10 +0000)
committerJoerg Mueller <nexyon@gmail.com>
Tue, 9 Aug 2011 14:10:32 +0000 (14:10 +0000)
Improved waveform drawing in the sequencer.

* Drawing the waveform of a sequencer strip is now independent from whether the sound is cached or not.
* Improved drawing of the waveform in the sequencer (especially speed!).
* Making it possible to vertically zoom more in the sequencer to better see the waveform for lipsync.
* Fixed a bug which crashed blender on loading a sound file via ffmpeg.

12 files changed:
intern/audaspace/ffmpeg/AUD_FFMPEGReader.cpp
intern/audaspace/intern/AUD_C-API.cpp
intern/audaspace/intern/AUD_C-API.h
release/scripts/startup/bl_ui/space_sequencer.py
source/blender/blenkernel/BKE_sound.h
source/blender/blenkernel/intern/sound.c
source/blender/blenloader/intern/readfile.c
source/blender/editors/space_sequencer/sequencer_draw.c
source/blender/editors/space_sequencer/space_sequencer.c
source/blender/makesdna/DNA_sequence_types.h
source/blender/makesdna/DNA_sound_types.h
source/blender/makesrna/intern/rna_sequencer.c

index b980e1d98e057d8e2b0ed3ad846f3ad70ad44a8a..1683a9a61c0395025c7c09ce8e4035366c58a70e 100644 (file)
@@ -177,6 +177,7 @@ static const char* fileopen_error = "AUD_FFMPEGReader: File couldn't be "
 
 AUD_FFMPEGReader::AUD_FFMPEGReader(std::string filename) :
        m_pkgbuf(AVCODEC_MAX_AUDIO_FRAME_SIZE<<1),
+       m_formatCtx(NULL),
        m_aviocontext(NULL),
        m_membuf(NULL)
 {
index e5c966fdcaeb5597d9a1606bb71120959fe683aa..85a053238d02fd00ed9fe14af840d676e1b7d4cc 100644 (file)
@@ -816,7 +816,7 @@ float* AUD_readSoundBuffer(const char* filename, float low, float high,
        if(high < rate)
                sound = new AUD_LowpassFactory(sound, high);
        if(low > 0)
-               sound = new AUD_HighpassFactory(sound, low);;
+               sound = new AUD_HighpassFactory(sound, low);
 
        sound = new AUD_EnvelopeFactory(sound, attack, release, threshold, 0.1f);
        sound = new AUD_LinearResampleFactory(sound, specs);
@@ -1055,7 +1055,7 @@ int AUD_doesPlayback()
        return -1;
 }
 
-int AUD_readSound(AUD_Sound* sound, sample_t* buffer, int length)
+int AUD_readSound(AUD_Sound* sound, sample_t* buffer, int length, int samples_per_second)
 {
        AUD_DeviceSpecs specs;
        sample_t* buf;
@@ -1067,38 +1067,40 @@ int AUD_readSound(AUD_Sound* sound, sample_t* buffer, int length)
 
        AUD_Reference<AUD_IReader> reader = AUD_ChannelMapperFactory(*sound, specs).createReader();
 
-       int len = reader->getLength();
-       float samplejump = (float)len / (float)length;
-       float min, max;
+       specs.specs = reader->getSpecs();
+       int len;
+       float samplejump = specs.rate / samples_per_second;
+       float min, max, power;
        bool eos;
 
        for(int i = 0; i < length; i++)
        {
                len = floor(samplejump * (i+1)) - floor(samplejump * i);
 
-               if(aBuffer.getSize() < len * AUD_SAMPLE_SIZE(reader->getSpecs()))
-               {
-                       aBuffer.resize(len * AUD_SAMPLE_SIZE(reader->getSpecs()));
-                       buf = aBuffer.getBuffer();
-               }
+               aBuffer.assureSize(len * AUD_SAMPLE_SIZE(specs));
+               buf = aBuffer.getBuffer();
 
                reader->read(len, eos, buf);
 
-               if(eos)
-               {
-                       length = i;
-                       break;
-               }
-
                max = min = *buf;
+               power = *buf * *buf;
                for(int j = 1; j < len; j++)
                {
                        if(buf[j] < min)
                                min = buf[j];
                        if(buf[j] > max)
                                max = buf[j];
-                       buffer[i * 2] = min;
-                       buffer[i * 2 + 1] = max;
+                       power += buf[j] * buf[j];
+               }
+
+               buffer[i * 3] = min;
+               buffer[i * 3 + 1] = max;
+               buffer[i * 3 + 2] = sqrt(power) / len;
+
+               if(eos)
+               {
+                       length = i;
+                       break;
                }
        }
 
index 7689d1b06b5b3d28f4deebe921a698a284bbcb85..2cd24551dd90f749e3db005e79d56088ce20610e 100644 (file)
@@ -502,7 +502,7 @@ extern void AUD_setSyncCallback(AUD_syncFunction function, void* data);
 
 extern int AUD_doesPlayback(void);
 
-extern int AUD_readSound(AUD_Sound* sound, sample_t* buffer, int length);
+extern int AUD_readSound(AUD_Sound* sound, sample_t* buffer, int length, int samples_per_second);
 
 extern AUD_Sound* AUD_copy(AUD_Sound* sound);
 
index 5320297dce253c145e26777c879196a769f6f73d..19202088faf8a660f0c629a897f1dedd04984cce 100644 (file)
@@ -638,6 +638,7 @@ class SEQUENCER_PT_sound(SequencerButtonsPanel, bpy.types.Panel):
 
         row.prop(strip.sound, "use_memory_cache")
 
+        layout.prop(strip, "waveform")
         layout.prop(strip, "volume")
         layout.prop(strip, "attenuation")
         layout.prop(strip, "pitch")
index c36532ee4cc4d1abe09ccf50a8a22fc6c1843889..ecf0d7e459a3ded444aa8944dc515b01fabe3455 100644 (file)
@@ -35,6 +35,8 @@
  *  \author nzc
  */
 
+#define SOUND_WAVE_SAMPLES_PER_SECOND 250
+
 struct PackedFile;
 struct bSound;
 struct bContext;
@@ -42,6 +44,12 @@ struct ListBase;
 struct Main;
 struct Sequence;
 
+typedef struct SoundWaveform
+{
+       int length;
+       float *data;
+} SoundWaveform;
+
 void sound_init_once(void);
 
 void sound_init(struct Main *main);
@@ -122,7 +130,9 @@ float sound_sync_scene(struct Scene *scene);
 
 int sound_scene_playing(struct Scene *scene);
 
-int sound_read_sound_buffer(struct bSound* sound, float* buffer, int length, float start, float end);
+void sound_free_waveform(struct bSound* sound);
+
+void sound_read_waveform(struct bSound* sound);
 
 int sound_get_channels(struct bSound* sound);
 
index 17df6ba23cbecb6559122279baad29952f67377d..888c719a304ca59859872c49b2ec3c6513dc993d 100644 (file)
@@ -24,6 +24,7 @@
 #include "DNA_sound_types.h"
 #include "DNA_speaker_types.h"
 
+#define WITH_AUDASPACE
 #ifdef WITH_AUDASPACE
 #  include "AUD_C-API.h"
 #endif
@@ -96,6 +97,8 @@ void sound_free(struct bSound* sound)
                AUD_unload(sound->cache);
                sound->cache = NULL;
        }
+
+       sound_free_waveform(sound);
 #endif // WITH_AUDASPACE
 }
 
@@ -608,12 +611,34 @@ int sound_scene_playing(struct Scene *scene)
                return -1;
 }
 
-int sound_read_sound_buffer(struct bSound* sound, float* buffer, int length, float start, float end)
+void sound_free_waveform(struct bSound* sound)
+{
+       if(sound->waveform)
+       {
+               MEM_freeN(((SoundWaveform*)sound->waveform)->data);
+               MEM_freeN(sound->waveform);
+       }
+
+       sound->waveform = NULL;
+}
+
+void sound_read_waveform(struct bSound* sound)
 {
-       AUD_Sound* limiter = AUD_limitSound(sound->cache, start, end);
-       int ret= AUD_readSound(limiter, buffer, length);
-       AUD_unload(limiter);
-       return ret;
+       AUD_SoundInfo info;
+
+       info = AUD_getInfo(sound->playback_handle);
+
+       if(info.length > 0)
+       {
+               SoundWaveform* waveform = MEM_mallocN(sizeof(SoundWaveform), "SoundWaveform");;
+               int length = info.length * SOUND_WAVE_SAMPLES_PER_SECOND;
+
+               waveform->data = MEM_mallocN(length * sizeof(float) * 3, "SoundWaveform.samples");
+               waveform->length = AUD_readSound(sound->playback_handle, waveform->data, length, SOUND_WAVE_SAMPLES_PER_SECOND);
+
+               sound_free_waveform(sound);
+               sound->waveform = waveform;
+       }
 }
 
 int sound_get_channels(struct bSound* sound)
index ef71df31df49dd87a4108e84e520814c52f3dc9c..ae5bafa2d083df1fe80d1268397516e20575e33e 100644 (file)
@@ -5609,6 +5609,7 @@ static void direct_link_sound(FileData *fd, bSound *sound)
 {
        sound->handle = NULL;
        sound->playback_handle = NULL;
+       sound->waveform = NULL;
 
        // versioning stuff, if there was a cache, then we enable caching:
        if(sound->cache)
@@ -11766,6 +11767,36 @@ static void do_versions(FileData *fd, Library *lib, Main *main)
                                SEQ_END
                        }
                }
+               {
+                       bScreen *screen;
+                       for(screen= main->screen.first; screen; screen= screen->id.next) {
+                               ScrArea *sa;
+                               /* add regions */
+                               for(sa= screen->areabase.first; sa; sa= sa->next) {
+                                       SpaceLink *sl= sa->spacedata.first;
+                                       if(sl->spacetype==SPACE_SEQ) {
+                                               ARegion *ar;
+                                               for (ar=sa->regionbase.first; ar; ar= ar->next) {
+                                                       if(ar->regiontype == RGN_TYPE_WINDOW) {
+                                                               if(ar->v2d.min[1] == 4.0f)
+                                                                       ar->v2d.min[1]= 0.5f;
+                                                       }
+                                               }
+                                       }
+                                       for (sl= sa->spacedata.first; sl; sl= sl->next) {
+                                               if(sl->spacetype==SPACE_SEQ) {
+                                                       ARegion *ar;
+                                                       for (ar=sl->regionbase.first; ar; ar= ar->next) {
+                                                               if(ar->regiontype == RGN_TYPE_WINDOW) {
+                                                                       if(ar->v2d.min[1] == 4.0f)
+                                                                               ar->v2d.min[1]= 0.5f;
+                                                               }
+                                                       }
+                                               }
+                                       }
+                               }
+                       }
+               }
                {
                        /* Make "auto-clamped" handles a per-keyframe setting instead of per-FCurve 
                         *
index 7e4207c75feca2685c5f020bc7f5ba9120442be3..bc97ff341db76182c328452e8343aac4c39d1e91 100644 (file)
@@ -173,30 +173,63 @@ static void drawseqwave(Scene *scene, Sequence *seq, float x1, float y1, float x
        x2 the end x value, same for y1 and y2
        stepsize is width of a pixel.
        */
-       if(seq->sound->cache)
+       if(seq->flag & SEQ_AUDIO_DRAW_WAVEFORM)
        {
-               int i;
+               int i, j, pos;
                int length = floor((x2-x1)/stepsize)+1;
                float ymid = (y1+y2)/2;
                float yscale = (y2-y1)/2;
-               float* samples = MEM_mallocN(length * sizeof(float) * 2, "seqwave_samples");
-               if(!samples)
-                       return;
-               if(sound_read_sound_buffer(seq->sound, samples, length,
-                                                                  (seq->startofs + seq->anim_startofs)/FPS,
-                                                                  (seq->startofs + seq->anim_startofs + seq->enddisp - seq->startdisp)/FPS) != length)
+               float samplestep;
+               float startsample, endsample;
+               float value;
+
+               SoundWaveform* waveform;
+
+               if(!seq->sound->waveform)
+                       sound_read_waveform(seq->sound);
+
+               waveform = seq->sound->waveform;
+
+               startsample = floor((seq->startofs + seq->anim_startofs)/FPS * SOUND_WAVE_SAMPLES_PER_SECOND);
+               endsample = ceil((seq->startofs + seq->anim_startofs + seq->enddisp - seq->startdisp)/FPS * SOUND_WAVE_SAMPLES_PER_SECOND);
+               samplestep = (endsample-startsample) * stepsize / (x2-x1);
+
+               if(length > floor((waveform->length - startsample) / samplestep))
+                       length = floor((waveform->length - startsample) / samplestep);
+
+               glBegin(GL_LINE_STRIP);
+               for(i = 0; i < length; i++)
                {
-                       MEM_freeN(samples);
-                       return;
+                       pos = startsample + i * samplestep;
+
+                       value = waveform->data[pos * 3];
+
+                       for(j = pos+1; (j < waveform->length) && (j < pos + samplestep); j++)
+                       {
+                               if(value > waveform->data[j * 3])
+                                       value = waveform->data[j * 3];
+                       }
+
+                       glVertex2f(x1+i*stepsize, ymid + value * yscale);
                }
-               glBegin(GL_LINES);
+               glEnd();
+
+               glBegin(GL_LINE_STRIP);
                for(i = 0; i < length; i++)
                {
-                       glVertex2f(x1+i*stepsize, ymid + samples[i * 2] * yscale);
-                       glVertex2f(x1+i*stepsize, ymid + samples[i * 2 + 1] * yscale);
+                       pos = startsample + i * samplestep;
+
+                       value = waveform->data[pos * 3 + 1];
+
+                       for(j = pos+1; (j < waveform->length) && (j < pos + samplestep); j++)
+                       {
+                               if(value < waveform->data[j * 3 + 1])
+                                       value = waveform->data[j * 3 + 1];
+                       }
+
+                       glVertex2f(x1+i*stepsize, ymid + value * yscale);
                }
                glEnd();
-               MEM_freeN(samples);
        }
 }
 
index d1df9699fa3b9159a1a7314ba7b4d2b7cab90a15..36471c7ffcfd598b718f47206f697340baecb523 100644 (file)
@@ -223,7 +223,7 @@ static SpaceLink *sequencer_new(const bContext *C)
        ar->v2d.cur= ar->v2d.tot;
        
        ar->v2d.min[0]= 10.0f;
-       ar->v2d.min[1]= 4.0f;
+       ar->v2d.min[1]= 0.5f;
        
        ar->v2d.max[0]= MAXFRAMEF;
        ar->v2d.max[1]= MAXSEQ;
index aa04dc9ac130b9cd1bef578521c6e558fa103e48..359ef8449e9b177e5031fb293bc4f75e18428b40 100644 (file)
@@ -288,6 +288,7 @@ typedef struct SpeedControlVars {
 #define SEQ_AUDIO_VOLUME_ANIMATED   (1<<24)
 #define SEQ_AUDIO_PITCH_ANIMATED    (1<<25)
 #define SEQ_AUDIO_PAN_ANIMATED      (1<<26)
+#define SEQ_AUDIO_DRAW_WAVEFORM     (1<<27)
 
 #define SEQ_INVALID_EFFECT          (1<<31)
 
index 56ad0e1ff8475b5c6b249ae608b4566b447c6db6..d7546e84bbfe76cf899d4aa25996331861d3746b 100644 (file)
@@ -84,6 +84,11 @@ typedef struct bSound {
         */
        void *cache;
 
+       /**
+        * Waveform display data.
+        */
+       void *waveform;
+
        /**
         * The audaspace handle that should actually be played back.
         * Should be cache if cache != NULL; otherwise it's handle
index 6e18f955bf5d61761c373b6ecb0196cf8f31d42e..38575242fd6b7ad77a8f1b1e1e228e32505281bf 100644 (file)
@@ -976,6 +976,11 @@ static void rna_def_sequence(BlenderRNA *brna)
        RNA_def_property_ui_text(prop, "Lock", "Lock strip so that it can't be transformed");
        RNA_def_property_update(prop, NC_SCENE|ND_SEQUENCER, NULL);
 
+       prop= RNA_def_property(srna, "waveform", PROP_BOOLEAN, PROP_NONE);
+       RNA_def_property_boolean_sdna(prop, NULL, "flag", SEQ_AUDIO_DRAW_WAVEFORM);
+       RNA_def_property_ui_text(prop, "Draw Waveform", "Whether to draw the sound's waveform.");
+       RNA_def_property_update(prop, NC_SCENE|ND_SEQUENCER, NULL);
+
        /* strip positioning */
 
        prop= RNA_def_property(srna, "frame_final_duration", PROP_INT, PROP_TIME);