C#.Net 透過Kinect取聲音並繪製成音波圖

Kinect是有內建麥克風陣列在裡頭，所謂的陣列麥克風就是兩組以上，並以麥克風進行偵測，所得聲音用DSP做比對，來還原原始音效。

Kinect的麥克風陣列內含了四組麥克風，分為兩組，一組來取得聲音並進行消除雜音等演算法處理之後，可提供另一組聲音辨識的功能。


最近因為上課需要，所以要在寫Kinect的相關程式，當然這篇就是拿來練功的，不過這次是透過『Kinect體感程式探索-使用C#』去學習關於Kinect的聲音處理部份。而不再透過官方的API

這樣學起來也比較快，省時間XDDD

照慣例，還是要分析一下程式碼，這樣才能有所學習到

主要宣告的有以下幾種，最關鍵在於imageRect和imageDataArray以及soundSample
imageRect是一個空矩形的結構，imageDataArray是用來記錄波形位置，soundSample則是用來記錄聲音

private KinectSensor sensor;
        private KinectAudioSource audioSource;
        private WriteableBitmap writeableBitmap;
        private Int32Rect imageRect;
        private short[] imageDataArray;
        private byte[] soundSample;
        int witdh;
        int height;

來看一下程式載入點的程式碼！

public MainWindow()
        {
            InitializeComponent();

            witdh = (int)DisplaySound.Width;
            height = (int)DisplaySound.Height;
            imageDataArray = new short[witdh * height];

            for (int index = 0; index < imageDataArray.Length; index++)
            {
                imageDataArray[index] = 32767;
            }

            soundSample = new byte[witdh * 2];
            writeableBitmap = new WriteableBitmap(witdh, height, 96, 96,
                PixelFormats.Gray16, null);
            imageRect = new Int32Rect(0, 0, witdh, height);

            DisplaySound.Source = writeableBitmap;
            KinectSensor.KinectSensors.StatusChanged += KinectSensors_StatusChanged;
            writeableBitmap.WritePixels(imageRect,
                imageDataArray, 640 * 2, 0);
        }

imageDataArray是用來記錄波形位置，所以他的陣列大小為『高乘上寬』，並且將值設為32767，32767為short最高的數值

imageDataArray = new short[witdh * height];

            for (int index = 0; index < imageDataArray.Length; index++)
            {
                imageDataArray[index] = 32767;
            }

並將soundSample陣列大小設為『寬度乘上2』

soundSample = new byte[witdh * 2];

透過writeableBitmap將背景設定為灰色，dpi為96，DisplaySound來源指定為writeableBitmap

writeableBitmap = new WriteableBitmap(witdh, height, 96, 96,
                PixelFormats.Gray16, null);

DisplaySound.Source = writeableBitmap;

只要從Kinect取得的Sensor狀態有改變，則會觸發

KinectSensor.KinectSensors.StatusChanged += KinectSensors_StatusChanged;

該方法主要偵測狀態如果連結了，但sensor本身為空，則會將KinectSensor指定給sensor，並且啟動Kinect硬體。如果為中斷連結，則將Kinect啟動的硬體都關閉

private void KinectSensors_StatusChanged(object sender, StatusChangedEventArgs e)
        {
            switch (e.Status)
            {
                case KinectStatus.Connected:
                    if (this.sensor == null)
                    {
                        this.sensor = e.Sensor;
                        this.sensor.Start();
                    }
                    break;
                case KinectStatus.Disconnected:
                    this.sensor.Stop();
                    this.sensor = null;
                    break;
            }
        }

在程式載入的那一刻會觸發『Window_Loaded』，這個事件會判斷電腦接上Kinect了嗎？接上了KinectSensor.KinectSensors.Count會大於0，如果沒有的話則會顯示出『請將Kinect接上電腦』，如果接上就將第一個接上Kinect的機器控制權交由sensor去處理，並且啟動硬體

private void Window_Loaded(object sender, RoutedEventArgs e)
        {
            if (KinectSensor.KinectSensors.Count == 0)
            {
                MessageBox.Show("請將Kinect接上電腦");
            }
            else if (KinectSensor.KinectSensors[0].Status == KinectStatus.Connected)
            {
                this.sensor = KinectSensor.KinectSensors[0];
                this.sensor.Start();
            }
        }

在滑鼠按下左鍵時，會觸發『DisplaySound_MouseLeftButtonDown』事件，該事件會啟動執行緒，並且將執行緒的執行權限調為優先，並在背後執行

private void DisplaySound_MouseLeftButtonDown(object sender, MouseButtonEventArgs e)
        {
            Thread thread = new Thread(new ThreadStart(ShowWave));
            thread.Priority = ThreadPriority.Highest;
            thread.IsBackground = true;
            thread.Start();
        }

執行緒主要執行的方法是ShowWave，所以我們來看看該方法吧！

private void ShowWave()
        {
            audioSource = sensor.AudioSource;
            var audioStream = audioSource.Start();
            short soundLevel;
            int imageY;
            int heightBias = height / 2;
            int imagePosition;

            while (audioStream.Read(soundSample, 0, soundSample.Length) > 0)
            {
                for (int index = 0; index < imageDataArray.Length; index++)
                {
                    imageDataArray[index] = 32767;
                }

                int soundSampleIndex = 0;

                for (int imageX = 0; imageX < witdh; imageX++)
                {
                    soundLevel = (short)(soundSample[soundSampleIndex]
                        | (soundSample[soundSampleIndex + 1] << 8));
                    imageY = (soundLevel * height) /
                        65535 + heightBias;
                    imageY = height - imageY;
                    imageY = imageY * witdh;
                    imagePosition = imageX + imageY;
                    if (imagePosition > 307199)
                        imagePosition = 307199;
                    imageDataArray[imagePosition] = 0;
                    soundSampleIndex += 2;
                }

                Dispatcher.Invoke(new Action(() => UpdateDisplay()));
            }
        }

透過audioSource去取得目前Kinect的Audio來源，並啟動

audioSource = sensor.AudioSource;
            var audioStream = audioSource.Start();

透過一個While不斷地從Kinect取得音源

while (audioStream.Read(soundSample, 0, soundSample.Length) > 0)

在將得的值去換算，得到高低差

for (int imageX = 0; imageX < witdh; imageX++)
                {
                    soundLevel = (short)(soundSample[soundSampleIndex]
                        | (soundSample[soundSampleIndex + 1] << 8));
                    imageY = (soundLevel * height) /
                        65535 + heightBias;
                    imageY = height - imageY;
                    imageY = imageY * witdh;
                    imagePosition = imageX + imageY;
                    if (imagePosition > 307199)
                        imagePosition = 307199;
                    imageDataArray[imagePosition] = 0;
                    soundSampleIndex += 2;
                }

為什麼imagePosition大於307199的數值就設定為307199，你可以去算『640乘上480』數值為多少？

並且不斷地去呼叫UpdateDisplay，更新DisplaySound的圖形內容，就會看到整個音波圖了

Dispatcher.Invoke(new Action(() => UpdateDisplay()));

XAML：

<Window x:Class="WpfApplication1.MainWindow"
        xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
        xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
        Title="MainWindow" Height="480" Width="640" Loaded="Window_Loaded">
    <Grid>
        <Image x:Name="DisplaySound" Height="480" Width="640" 
               MouseLeftButtonDown="DisplaySound_MouseLeftButtonDown"/>
    </Grid>
</Window>

程式碼：

using Microsoft.Kinect;
using System;
using System.Threading;
using System.Windows;
using System.Windows.Input;
using System.Windows.Media;
using System.Windows.Media.Imaging;

namespace WpfApplication1
{
    /// <summary>
    /// MainWindow.xaml 的互動邏輯
    /// </summary>
    public partial class MainWindow : Window
    {
        private KinectSensor sensor;
        private KinectAudioSource audioSource;
        private WriteableBitmap writeableBitmap;
        private Int32Rect imageRect;
        private short[] imageDataArray;
        private byte[] soundSample;
        int witdh;
        int height;


        public MainWindow()
        {
            InitializeComponent();

            witdh = (int)DisplaySound.Width;
            height = (int)DisplaySound.Height;
            imageDataArray = new short[witdh * height];

            for (int index = 0; index < imageDataArray.Length; index++)
            {
                imageDataArray[index] = 32767;
            }

            soundSample = new byte[witdh * 2];
            writeableBitmap = new WriteableBitmap(witdh, height, 96, 96,
                PixelFormats.Gray16, null);
            imageRect = new Int32Rect(0, 0, witdh, height);

            DisplaySound.Source = writeableBitmap;
            KinectSensor.KinectSensors.StatusChanged += KinectSensors_StatusChanged;
            writeableBitmap.WritePixels(imageRect,
                imageDataArray, 640 * 2, 0);
        }

        private void Window_Loaded(object sender, RoutedEventArgs e)
        {
            if (KinectSensor.KinectSensors.Count == 0)
            {
                MessageBox.Show("請將Kinect接上電腦");
            }
            else if (KinectSensor.KinectSensors[0].Status == KinectStatus.Connected)
            {
                this.sensor = KinectSensor.KinectSensors[0];
                this.sensor.Start();
            }
        }

        private void KinectSensors_StatusChanged(object sender, StatusChangedEventArgs e)
        {
            switch (e.Status)
            {
                case KinectStatus.Connected:
                    if (this.sensor == null)
                    {
                        this.sensor = e.Sensor;
                        this.sensor.Start();
                    }
                    break;
                case KinectStatus.Disconnected:
                    this.sensor.Stop();
                    this.sensor = null;
                    break;
            }
        }

        private void DisplaySound_MouseLeftButtonDown(object sender, MouseButtonEventArgs e)
        {
            Thread thread = new Thread(new ThreadStart(ShowWave));
            thread.Priority = ThreadPriority.Highest;
            thread.IsBackground = true;
            thread.Start();
        }

        private void ShowWave()
        {
            audioSource = sensor.AudioSource;
            var audioStream = audioSource.Start();
            short soundLevel;
            int imageY;
            int heightBias = height / 2;
            int imagePosition;

            while (audioStream.Read(soundSample, 0, soundSample.Length) > 0)
            {
                for (int index = 0; index < imageDataArray.Length; index++)
                {
                    imageDataArray[index] = 32767;
                }

                int soundSampleIndex = 0;

                for (int imageX = 0; imageX < witdh; imageX++)
                {
                    soundLevel = (short)(soundSample[soundSampleIndex]
                        | (soundSample[soundSampleIndex + 1] << 8));
                    imageY = (soundLevel * height) /
                        65535 + heightBias;
                    imageY = height - imageY;
                    imageY = imageY * witdh;
                    imagePosition = imageX + imageY;
                    if (imagePosition > 307199)
                        imagePosition = 307199;
                    imageDataArray[imagePosition] = 0;
                    soundSampleIndex += 2;
                }

                Dispatcher.Invoke(new Action(() => UpdateDisplay()));
            }
        }

        private void UpdateDisplay()
        {
            writeableBitmap.WritePixels(imageRect,
                imageDataArray, witdh * 2, 0);
        }

    }
}

這本書是跟學弟借的，非常感激^^

參考資料：
Kinect體感程式探索-使用C#
http://ashonehuang.pixnet.net/blog/post/20028926
http://zh.wikipedia.org/wiki/%E9%BA%A5%E5%85%8B%E9%A2%A8%E9%99%A3%E5%88%97