3D Audio GSoC:

Improved waveform drawing in the sequencer.

* Drawing the waveform of a sequencer strip is now independent from whether the sound is cached or not.
* Improved drawing of the waveform in the sequencer (especially speed!).
* Making it possible to vertically zoom more in the sequencer to better see the waveform for lipsync.
* Fixed a bug which crashed blender on loading a sound file via ffmpeg.

intern/audaspace/ffmpeg/AUD_FFMPEGReader.cpp

@@ -177,6 +177,7 @@ static const char* fileopen_error = "AUD_FFMPEGReader: File couldn't be "
 
 AUD_FFMPEGReader::AUD_FFMPEGReader(std::string filename) :
 	m_pkgbuf(AVCODEC_MAX_AUDIO_FRAME_SIZE<<1),
+	m_formatCtx(NULL),
 	m_aviocontext(NULL),
 	m_membuf(NULL)
 {

intern/audaspace/intern/AUD_C-API.cpp

@@ -816,7 +816,7 @@ float* AUD_readSoundBuffer(const char* filename, float low, float high,
 	if(high < rate)
 		sound = new AUD_LowpassFactory(sound, high);
 	if(low > 0)
-		sound = new AUD_HighpassFactory(sound, low);;
+		sound = new AUD_HighpassFactory(sound, low);
 
 	sound = new AUD_EnvelopeFactory(sound, attack, release, threshold, 0.1f);
 	sound = new AUD_LinearResampleFactory(sound, specs);
@@ -1055,7 +1055,7 @@ int AUD_doesPlayback()
 	return -1;
 }
 
-int AUD_readSound(AUD_Sound* sound, sample_t* buffer, int length)
+int AUD_readSound(AUD_Sound* sound, sample_t* buffer, int length, int samples_per_second)
 {
 	AUD_DeviceSpecs specs;
 	sample_t* buf;
@@ -1067,38 +1067,40 @@ int AUD_readSound(AUD_Sound* sound, sample_t* buffer, int length)
 
 	AUD_Reference<AUD_IReader> reader = AUD_ChannelMapperFactory(*sound, specs).createReader();
 
-	int len = reader->getLength();
-	float samplejump = (float)len / (float)length;
-	float min, max;
+	specs.specs = reader->getSpecs();
+	int len;
+	float samplejump = specs.rate / samples_per_second;
+	float min, max, power;
 	bool eos;
 
 	for(int i = 0; i < length; i++)
 	{
 		len = floor(samplejump * (i+1)) - floor(samplejump * i);
 
-		if(aBuffer.getSize() < len * AUD_SAMPLE_SIZE(reader->getSpecs()))
-		{
-			aBuffer.resize(len * AUD_SAMPLE_SIZE(reader->getSpecs()));
-			buf = aBuffer.getBuffer();
-		}
+		aBuffer.assureSize(len * AUD_SAMPLE_SIZE(specs));
+		buf = aBuffer.getBuffer();
 
 		reader->read(len, eos, buf);
 
-		if(eos)
-		{
-			length = i;
-			break;
-		}
-
 		max = min = *buf;
+		power = *buf * *buf;
 		for(int j = 1; j < len; j++)
 		{
 			if(buf[j] < min)
 				min = buf[j];
 			if(buf[j] > max)
 				max = buf[j];
-			buffer[i * 2] = min;
-			buffer[i * 2 + 1] = max;
+			power += buf[j] * buf[j];
+		}
+
+		buffer[i * 3] = min;
+		buffer[i * 3 + 1] = max;
+		buffer[i * 3 + 2] = sqrt(power) / len;
+
+		if(eos)
+		{
+			length = i;
+			break;
 		}
 	}
 

intern/audaspace/intern/AUD_C-API.h

@@ -502,7 +502,7 @@ extern void AUD_setSyncCallback(AUD_syncFunction function, void* data);
 
 extern int AUD_doesPlayback(void);
 
-extern int AUD_readSound(AUD_Sound* sound, sample_t* buffer, int length);
+extern int AUD_readSound(AUD_Sound* sound, sample_t* buffer, int length, int samples_per_second);
 
 extern AUD_Sound* AUD_copy(AUD_Sound* sound);
 

release/scripts/startup/bl_ui/space_sequencer.py

@@ -638,6 +638,7 @@ class SEQUENCER_PT_sound(SequencerButtonsPanel, bpy.types.Panel):
 
         row.prop(strip.sound, "use_memory_cache")
 
+        layout.prop(strip, "waveform")
         layout.prop(strip, "volume")
         layout.prop(strip, "attenuation")
         layout.prop(strip, "pitch")

source/blender/blenkernel/BKE_sound.h

@@ -35,6 +35,8 @@
  *  \author nzc
  */
 
+#define SOUND_WAVE_SAMPLES_PER_SECOND 250
+
 struct PackedFile;
 struct bSound;
 struct bContext;
@@ -42,6 +44,12 @@ struct ListBase;
 struct Main;
 struct Sequence;
 
+typedef struct SoundWaveform
+{
+	int length;
+	float *data;
+} SoundWaveform;
+
 void sound_init_once(void);
 
 void sound_init(struct Main *main);
@@ -122,7 +130,9 @@ float sound_sync_scene(struct Scene *scene);
 
 int sound_scene_playing(struct Scene *scene);
 
-int sound_read_sound_buffer(struct bSound* sound, float* buffer, int length, float start, float end);
+void sound_free_waveform(struct bSound* sound);
+
+void sound_read_waveform(struct bSound* sound);
 
 int sound_get_channels(struct bSound* sound);
 

source/blender/blenkernel/intern/sound.c

@@ -24,6 +24,7 @@
 #include "DNA_sound_types.h"
 #include "DNA_speaker_types.h"
 
+#define WITH_AUDASPACE
 #ifdef WITH_AUDASPACE
 #  include "AUD_C-API.h"
 #endif
@@ -96,6 +97,8 @@ void sound_free(struct bSound* sound)
 		AUD_unload(sound->cache);
 		sound->cache = NULL;
 	}
+
+	sound_free_waveform(sound);
 #endif // WITH_AUDASPACE
 }
 
@@ -608,12 +611,34 @@ int sound_scene_playing(struct Scene *scene)
 		return -1;
 }
 
-int sound_read_sound_buffer(struct bSound* sound, float* buffer, int length, float start, float end)
+void sound_free_waveform(struct bSound* sound)
 {
-	AUD_Sound* limiter = AUD_limitSound(sound->cache, start, end);
-	int ret= AUD_readSound(limiter, buffer, length);
-	AUD_unload(limiter);
-	return ret;
+	if(sound->waveform)
+	{
+		MEM_freeN(((SoundWaveform*)sound->waveform)->data);
+		MEM_freeN(sound->waveform);
+	}
+
+	sound->waveform = NULL;
+}
+
+void sound_read_waveform(struct bSound* sound)
+{
+	AUD_SoundInfo info;
+
+	info = AUD_getInfo(sound->playback_handle);
+
+	if(info.length > 0)
+	{
+		SoundWaveform* waveform = MEM_mallocN(sizeof(SoundWaveform), "SoundWaveform");;
+		int length = info.length * SOUND_WAVE_SAMPLES_PER_SECOND;
+
+		waveform->data = MEM_mallocN(length * sizeof(float) * 3, "SoundWaveform.samples");
+		waveform->length = AUD_readSound(sound->playback_handle, waveform->data, length, SOUND_WAVE_SAMPLES_PER_SECOND);
+
+		sound_free_waveform(sound);
+		sound->waveform = waveform;
+	}
 }
 
 int sound_get_channels(struct bSound* sound)

source/blender/blenloader/intern/readfile.c

@@ -5609,6 +5609,7 @@ static void direct_link_sound(FileData *fd, bSound *sound)
 {
 	sound->handle = NULL;
 	sound->playback_handle = NULL;
+	sound->waveform = NULL;
 
 	// versioning stuff, if there was a cache, then we enable caching:
 	if(sound->cache)
@@ -11766,6 +11767,36 @@ static void do_versions(FileData *fd, Library *lib, Main *main)
 				SEQ_END
 			}
 		}
+		{
+			bScreen *screen;
+			for(screen= main->screen.first; screen; screen= screen->id.next) {
+				ScrArea *sa;
+				/* add regions */
+				for(sa= screen->areabase.first; sa; sa= sa->next) {
+					SpaceLink *sl= sa->spacedata.first;
+					if(sl->spacetype==SPACE_SEQ) {
+						ARegion *ar;
+						for (ar=sa->regionbase.first; ar; ar= ar->next) {
+							if(ar->regiontype == RGN_TYPE_WINDOW) {
+								if(ar->v2d.min[1] == 4.0f)
+									ar->v2d.min[1]= 0.5f;
+							}
+						}
+					}
+					for (sl= sa->spacedata.first; sl; sl= sl->next) {
+						if(sl->spacetype==SPACE_SEQ) {
+							ARegion *ar;
+							for (ar=sl->regionbase.first; ar; ar= ar->next) {
+								if(ar->regiontype == RGN_TYPE_WINDOW) {
+									if(ar->v2d.min[1] == 4.0f)
+										ar->v2d.min[1]= 0.5f;
+								}
+							}
+						}
+					}
+				}
+			}
+		}
 		{
 			/* Make "auto-clamped" handles a per-keyframe setting instead of per-FCurve 
 			 *

source/blender/editors/space_sequencer/sequencer_draw.c

@@ -173,30 +173,63 @@ static void drawseqwave(Scene *scene, Sequence *seq, float x1, float y1, float x
 	x2 the end x value, same for y1 and y2
 	stepsize is width of a pixel.
 	*/
-	if(seq->sound->cache)
+	if(seq->flag & SEQ_AUDIO_DRAW_WAVEFORM)
 	{
-		int i;
+		int i, j, pos;
 		int length = floor((x2-x1)/stepsize)+1;
 		float ymid = (y1+y2)/2;
 		float yscale = (y2-y1)/2;
-		float* samples = MEM_mallocN(length * sizeof(float) * 2, "seqwave_samples");
-		if(!samples)
-			return;
-		if(sound_read_sound_buffer(seq->sound, samples, length,
-								   (seq->startofs + seq->anim_startofs)/FPS,
-								   (seq->startofs + seq->anim_startofs + seq->enddisp - seq->startdisp)/FPS) != length)
-		{
-			MEM_freeN(samples);
-			return;
-		}
-		glBegin(GL_LINES);
+		float samplestep;
+		float startsample, endsample;
+		float value;
+
+		SoundWaveform* waveform;
+
+		if(!seq->sound->waveform)
+			sound_read_waveform(seq->sound);
+
+		waveform = seq->sound->waveform;
+
+		startsample = floor((seq->startofs + seq->anim_startofs)/FPS * SOUND_WAVE_SAMPLES_PER_SECOND);
+		endsample = ceil((seq->startofs + seq->anim_startofs + seq->enddisp - seq->startdisp)/FPS * SOUND_WAVE_SAMPLES_PER_SECOND);
+		samplestep = (endsample-startsample) * stepsize / (x2-x1);
+
+		if(length > floor((waveform->length - startsample) / samplestep))
+			length = floor((waveform->length - startsample) / samplestep);
+
+		glBegin(GL_LINE_STRIP);
 		for(i = 0; i < length; i++)
 		{
-			glVertex2f(x1+i*stepsize, ymid + samples[i * 2] * yscale);
-			glVertex2f(x1+i*stepsize, ymid + samples[i * 2 + 1] * yscale);
+			pos = startsample + i * samplestep;
+
+			value = waveform->data[pos * 3];
+
+			for(j = pos+1; (j < waveform->length) && (j < pos + samplestep); j++)
+			{
+				if(value > waveform->data[j * 3])
+					value = waveform->data[j * 3];
+			}
+
+			glVertex2f(x1+i*stepsize, ymid + value * yscale);
+		}
+		glEnd();
+
+		glBegin(GL_LINE_STRIP);
+		for(i = 0; i < length; i++)
+		{
+			pos = startsample + i * samplestep;
+
+			value = waveform->data[pos * 3 + 1];
+
+			for(j = pos+1; (j < waveform->length) && (j < pos + samplestep); j++)
+			{
+				if(value < waveform->data[j * 3 + 1])
+					value = waveform->data[j * 3 + 1];
+			}
+
+			glVertex2f(x1+i*stepsize, ymid + value * yscale);
 		}
 		glEnd();
-		MEM_freeN(samples);
 	}
 }
 

source/blender/editors/space_sequencer/space_sequencer.c

@@ -223,7 +223,7 @@ static SpaceLink *sequencer_new(const bContext *C)
 	ar->v2d.cur= ar->v2d.tot;
 	
 	ar->v2d.min[0]= 10.0f;
-	ar->v2d.min[1]= 4.0f;
+	ar->v2d.min[1]= 0.5f;
 	
 	ar->v2d.max[0]= MAXFRAMEF;
 	ar->v2d.max[1]= MAXSEQ;

source/blender/makesdna/DNA_sequence_types.h

@@ -288,6 +288,7 @@ typedef struct SpeedControlVars {
 #define SEQ_AUDIO_VOLUME_ANIMATED   (1<<24)
 #define SEQ_AUDIO_PITCH_ANIMATED    (1<<25)
 #define SEQ_AUDIO_PAN_ANIMATED      (1<<26)
+#define SEQ_AUDIO_DRAW_WAVEFORM     (1<<27)
 
 #define SEQ_INVALID_EFFECT          (1<<31)
 

source/blender/makesdna/DNA_sound_types.h

@@ -84,6 +84,11 @@ typedef struct bSound {
 	 */
 	void *cache;
 
+	/**
+	 * Waveform display data.
+	 */
+	void *waveform;
+
 	/**
 	 * The audaspace handle that should actually be played back.
 	 * Should be cache if cache != NULL; otherwise it's handle

source/blender/makesrna/intern/rna_sequencer.c

@@ -976,6 +976,11 @@ static void rna_def_sequence(BlenderRNA *brna)
 	RNA_def_property_ui_text(prop, "Lock", "Lock strip so that it can't be transformed");
 	RNA_def_property_update(prop, NC_SCENE|ND_SEQUENCER, NULL);
 
+	prop= RNA_def_property(srna, "waveform", PROP_BOOLEAN, PROP_NONE);
+	RNA_def_property_boolean_sdna(prop, NULL, "flag", SEQ_AUDIO_DRAW_WAVEFORM);
+	RNA_def_property_ui_text(prop, "Draw Waveform", "Whether to draw the sound's waveform.");
+	RNA_def_property_update(prop, NC_SCENE|ND_SEQUENCER, NULL);
+
 	/* strip positioning */
 
 	prop= RNA_def_property(srna, "frame_final_duration", PROP_INT, PROP_TIME);