Downmix surround 5.1 to stereo

sunshine/audio.cpp

@@ -94,16 +94,16 @@ void capture(safe::signal_t *shutdown_event, packet_queue_t packets, config_t co
   //FIXME: Pick correct opus_stream_config_t based on config.channels
   auto stream = &stereo;
 
-  auto mic = platf::microphone(stream->sampleRate);
+  auto frame_size = config.packetDuration * stream->sampleRate / 1000;
+  int samples_per_frame = frame_size * stream->channelCount;
+
+  auto mic = platf::microphone(stream->sampleRate, frame_size);
   if(!mic) {
     BOOST_LOG(error) << "Couldn't create audio input"sv ;
 
     return;
   }
 
-  auto frame_size = config.packetDuration * stream->sampleRate / 1000;
-  int samples_per_frame = frame_size * stream->channelCount;
-
   while(!shutdown_event->peek()) {
     std::vector<std::int16_t> sample_buffer;
     sample_buffer.resize(samples_per_frame);
@@ -116,7 +116,7 @@ void capture(safe::signal_t *shutdown_event, packet_queue_t packets, config_t co
         continue;
       case platf::capture_e::reinit:
         mic.reset();
-        mic = platf::microphone(stream->sampleRate);
+        mic = platf::microphone(stream->sampleRate, frame_size);
         if(!mic) {
           BOOST_LOG(error) << "Couldn't re-initialize audio input"sv ;
 

sunshine/platform/common.h

@@ -157,7 +157,7 @@ std::string get_mac_address(const std::string_view &address);
 std::string from_sockaddr(const sockaddr *const);
 std::pair<std::uint16_t, std::string> from_sockaddr_ex(const sockaddr *const);
 
-std::unique_ptr<mic_t> microphone(std::uint32_t sample_rate);
+std::unique_ptr<mic_t> microphone(std::uint32_t sample_rate, std::uint32_t frame_size);
 std::shared_ptr<display_t> display(dev_type_e hwdevice_type);
 
 input_t input();

sunshine/platform/linux/display.cpp

@@ -410,7 +410,7 @@ std::shared_ptr<display_t> display(platf::dev_type_e hwdevice_type) {
   return x11_disp;
 }
 
-std::unique_ptr<mic_t> microphone(std::uint32_t sample_rate) {
+std::unique_ptr<mic_t> microphone(std::uint32_t sample_rate, std::uint32_t) {
   auto mic = std::make_unique<mic_attr_t>(PA_SAMPLE_S16LE, sample_rate, 2);
 
   int status;

sunshine/platform/windows/audio.cpp

@@ -49,10 +49,154 @@ public:
   }
 };
 
+struct format_t {
+  std::string_view name;
+  int channels;
+  int channel_mask;
+} formats [] {
+  {
+    "Stereo"sv,
+    2,
+    SPEAKER_FRONT_LEFT | SPEAKER_FRONT_RIGHT
+  },
+  {
+    "Mono"sv,
+    1,
+    SPEAKER_FRONT_CENTER
+  },
+  {
+    "Surround 5.1"sv,
+    6,
+    SPEAKER_FRONT_LEFT    |
+    SPEAKER_FRONT_RIGHT   |
+    SPEAKER_FRONT_CENTER  |
+    SPEAKER_LOW_FREQUENCY |
+    SPEAKER_BACK_LEFT     |
+    SPEAKER_BACK_RIGHT
+  }
+};
+
+void set_wave_format(audio::wave_format_t &wave_format, const format_t &format) {
+  wave_format->nChannels = format.channels;
+  wave_format->nBlockAlign = wave_format->nChannels * wave_format->wBitsPerSample / 8;
+  wave_format->nAvgBytesPerSec = wave_format->nSamplesPerSec * wave_format->nBlockAlign;
+
+  if(wave_format->wFormatTag == WAVE_FORMAT_EXTENSIBLE) {
+    ((PWAVEFORMATEXTENSIBLE)wave_format.get())->dwChannelMask = format.channel_mask;
+  }
+}
+
+void surround51_to_stereo(std::vector<std::int16_t> &sample_in, const util::buffer_t<std::int16_t> &sample_out) {
+  enum surround51_e : int {
+    front_left,
+    front_right,
+    front_center,
+    low_frequency, // subwoofer
+    back_left,
+    back_right,
+    channels51 // number of channels in surround sound
+  };
+
+  auto sample_in_pos  = std::begin(sample_in);
+  auto sample_end = std::begin(sample_out) + sample_in.size() / 2 * channels51;
+
+  for(auto sample_out_p = std::begin(sample_out); sample_out_p != sample_end; sample_out_p += channels51) {
+    std::uint32_t left {}, right {};
+
+    left += sample_out_p[front_left];
+    left += sample_out_p[front_center] * 90 / 100;
+    left += sample_out_p[low_frequency] * 30 / 100;
+    left += sample_out_p[back_left] * 70 / 100;
+    left += sample_out_p[back_right] * 30 / 100;
+
+    right += sample_out_p[front_right];
+    right += sample_out_p[front_center] * 90 / 100;
+    right += sample_out_p[low_frequency] * 30 / 100;
+    right += sample_out_p[back_left] * 30 / 100;
+    right += sample_out_p[back_right] * 70 / 100;;
+
+    *sample_in_pos++ = (std::uint16_t)left;
+    *sample_in_pos++ = (std::uint16_t)right;
+  }
+}
+
+void mono_to_stereo(std::vector<std::int16_t> &sample_in, const util::buffer_t<std::int16_t> &sample_out) {
+  auto sample_in_pos  = std::begin(sample_in);
+  auto sample_end = std::begin(sample_out) + sample_in.size() / 2;
+
+  for(auto sample_out_p = std::begin(sample_out); sample_out_p != sample_end; ++sample_out_p) {
+    *sample_in_pos++ = *sample_out_p;
+    *sample_in_pos++ = *sample_out_p;
+  }
+}
+
+audio_client_t make_audio_client(device_t &device, const format_t &format, int sample_rate) {
+  audio_client_t audio_client;
+  auto status = device->Activate(
+    IID_IAudioClient,
+    CLSCTX_ALL,
+    nullptr,
+    (void **)&audio_client);
+
+  if(FAILED(status)) {
+    BOOST_LOG(error) << "Couldn't activate Device: [0x"sv << util::hex(status).to_string_view() << ']';
+
+    return nullptr;
+  }
+
+  wave_format_t wave_format;
+  status = audio_client->GetMixFormat(&wave_format);
+
+  if(FAILED(status)) {
+    BOOST_LOG(error) << "Couldn't acquire Wave Format [0x"sv << util::hex(status).to_string_view() << ']';
+
+    return nullptr;
+  }
+
+  wave_format->wBitsPerSample = 16;
+  wave_format->nSamplesPerSec = sample_rate;
+  switch(wave_format->wFormatTag) {
+    case WAVE_FORMAT_PCM:
+      break;
+    case WAVE_FORMAT_IEEE_FLOAT:
+      break;
+    case WAVE_FORMAT_EXTENSIBLE: {
+      auto wave_ex = (PWAVEFORMATEXTENSIBLE) wave_format.get();
+      if (IsEqualGUID(KSDATAFORMAT_SUBTYPE_IEEE_FLOAT, wave_ex->SubFormat)) {
+        wave_ex->SubFormat = KSDATAFORMAT_SUBTYPE_PCM;
+        wave_ex->Samples.wValidBitsPerSample = 16;
+        break;
+      }
+
+      BOOST_LOG(error) << "Unsupported Sub Format for WAVE_FORMAT_EXTENSIBLE: [0x"sv << util::hex(wave_ex->SubFormat).to_string_view() << ']';
+    }
+    default:
+      BOOST_LOG(error) << "Unsupported Wave Format: [0x"sv << util::hex(wave_format->wFormatTag).to_string_view() << ']';
+      return nullptr;
+  };
+
+  set_wave_format(wave_format, format);
+
+  status = audio_client->Initialize(
+    AUDCLNT_SHAREMODE_SHARED,
+    AUDCLNT_STREAMFLAGS_LOOPBACK | AUDCLNT_STREAMFLAGS_EVENTCALLBACK,
+    0, 0,
+    wave_format.get(),
+    nullptr);
+
+  if(status) {
+    BOOST_LOG(debug) << "Couldn't initialize audio client for ["sv << format.name << "]: [0x"sv << util::hex(status).to_string_view() << ']';
+    return nullptr;
+  }
+
+  return audio_client;
+}
+
 class mic_wasapi_t : public mic_t {
 public:
   capture_e sample(std::vector<std::int16_t> &sample_in) override {
-    while(sample_buf_pos - std::begin(sample_buf) < sample_in.size()) {
+    auto sample_size = sample_in.size() /2 * format->channels;
+    while(sample_buf_pos - std::begin(sample_buf) < sample_size) {
       //FIXME: Use IAudioClient3 instead of IAudioClient, that would allows for adjusting the latency of the audio samples
       auto capture_result = _fill_buffer();
 
@@ -61,17 +205,32 @@ public:
       }
     }
 
-    std::copy_n(std::begin(sample_buf), sample_in.size(), std::begin(sample_in));
+    switch(format->channels) {
+    case 1:
+      mono_to_stereo(sample_in, sample_buf);
+      break;
+    case 2:
+      std::copy_n(std::begin(sample_buf), sample_size, std::begin(sample_in));
+      break;
+    case 6:
+      if(format->name == "Surround 5.1"sv) {
+        surround51_to_stereo(sample_in, sample_buf);
+        break;
+      }
+
+      BOOST_LOG(error) << '[' << format->name << "] not yet supported"sv;
+      return capture_e::error;
+    }
 
     // The excess samples should be in front of the queue
-    std::move(&sample_buf[sample_in.size()], sample_buf_pos, std::begin(sample_buf));
+    std::move(&sample_buf[sample_size], sample_buf_pos, std::begin(sample_buf));
-    sample_buf_pos -= sample_in.size();
+    sample_buf_pos -= sample_size;
 
     return capture_e::ok;
   }
 
 
-  int init(std::uint32_t sample_rate) {
+  int init(std::uint32_t sample_rate, std::uint32_t frame_size) {
     audio_event.reset(CreateEventA(nullptr, FALSE, FALSE, nullptr));
     if(!audio_event) {
       BOOST_LOG(error) << "Couldn't create Event handle"sv;
@@ -113,70 +272,26 @@ public:
       return -1;
     }
 
-    status = device->Activate(
+    for(auto &format : formats) {
-      IID_IAudioClient,
+      BOOST_LOG(debug) << "Trying audio format ["sv << format.name << ']';
-      CLSCTX_ALL,
+      audio_client = make_audio_client(device, format, sample_rate);
-      nullptr,
-      (void **) &audio_client);
 
-    if (FAILED(status)) {
+      if(audio_client) {
-      BOOST_LOG(error) << "Couldn't activate audio Device [0x"sv << util::hex(status).to_string_view() << ']';
+        BOOST_LOG(debug) << "Found audio format ["sv << format.name << ']';
-
+        this->format = &format;
-      return -1;
-    }
-
-    status = audio_client->GetMixFormat(&wave_format);
-    if(FAILED(status)) {
-      BOOST_LOG(error) << "Couldn't acquire Wave Format [0x"sv << util::hex(status).to_string_view() << ']';
-
-      return -1;
-    }
-
-    wave_format->nChannels = 2;
-    wave_format->wBitsPerSample = 16;
-    wave_format->nSamplesPerSec = sample_rate;
-    wave_format->nBlockAlign = wave_format->nChannels * wave_format->wBitsPerSample / 8;
-    wave_format->nAvgBytesPerSec = wave_format->nSamplesPerSec * wave_format->nBlockAlign;
-
-    switch(wave_format->wFormatTag) {
-      case WAVE_FORMAT_PCM:
         break;
-      case WAVE_FORMAT_IEEE_FLOAT:
-        wave_format->wFormatTag = WAVE_FORMAT_PCM;
-        break;
-      case WAVE_FORMAT_EXTENSIBLE: {
-        auto wave_ex = (PWAVEFORMATEXTENSIBLE) wave_format.get();
-        if (IsEqualGUID(KSDATAFORMAT_SUBTYPE_IEEE_FLOAT, wave_ex->SubFormat)) {
-          wave_ex->SubFormat = KSDATAFORMAT_SUBTYPE_PCM;
-          wave_ex->Samples.wValidBitsPerSample = 16;
-          break;
-        }
-
-        BOOST_LOG(error) << "Unsupported Sub Format for WAVE_FORMAT_EXTENSIBLE: [0x"sv << util::hex(wave_ex->SubFormat).to_string_view() << ']';
-        return -1;
       }
-      default:
+    }
-        BOOST_LOG(error) << "Unsupported Wave Format: [0x"sv << util::hex(wave_format->wFormatTag).to_string_view() << ']';
+
-        return -1;
+    if(!audio_client) {
-    };
+      BOOST_LOG(error) << "Couldn't find supported format for audio"sv;
+      return -1;
+    }
 
     REFERENCE_TIME default_latency;
     audio_client->GetDevicePeriod(&default_latency, nullptr);
     default_latency_ms = default_latency / 1000;
 
-    status = audio_client->Initialize(
-      AUDCLNT_SHAREMODE_SHARED,
-      AUDCLNT_STREAMFLAGS_LOOPBACK | AUDCLNT_STREAMFLAGS_EVENTCALLBACK,
-      0, 0,
-      wave_format.get(),
-      nullptr);
-
-    if (FAILED(status)) {
-      BOOST_LOG(error) << "Couldn't initialize audio client [0x"sv << util::hex(status).to_string_view() << ']';
-
-      return -1;
-    }
-
     std::uint32_t frames;
     status = audio_client->GetBufferSize(&frames);
     if (FAILED(status)) {
@@ -185,7 +300,8 @@ public:
       return -1;
     }
 
-    sample_buf = util::buffer_t<std::int16_t> { frames };
+    // *2 --> needs to fit double
+    sample_buf = util::buffer_t<std::int16_t> { std::max(frames *2, frame_size * format->channels *2) };
     sample_buf_pos = std::begin(sample_buf);
 
     status = audio_client->GetService(IID_IAudioCaptureClient, (void**)&audio_capture);
@@ -267,7 +383,7 @@ private:
       }
 
       sample_aligned.uninitialized = std::end(sample_buf) - sample_buf_pos;
-      auto n = std::min(sample_aligned.uninitialized, block_aligned.audio_sample_size * wave_format->nChannels);
+      auto n = std::min(sample_aligned.uninitialized, block_aligned.audio_sample_size * format->channels);
 
       if (buffer_flags & AUDCLNT_BUFFERFLAGS_SILENT) {
         std::fill_n(sample_buf_pos, n, 0);
@@ -296,13 +412,14 @@ public:
   device_enum_t device_enum;
   device_t device;
   audio_client_t audio_client;
-  wave_format_t wave_format;
   audio_capture_t audio_capture;
 
   REFERENCE_TIME default_latency_ms;
 
   util::buffer_t<std::int16_t> sample_buf;
   std::int16_t *sample_buf_pos;
+
+  format_t *format;
 };
 }
 
@@ -313,10 +430,10 @@ namespace dxgi {
 int init();
 }
 
-std::unique_ptr<mic_t> microphone(std::uint32_t sample_rate) {
+std::unique_ptr<mic_t> microphone(std::uint32_t sample_rate, std::uint32_t frame_size) {
   auto mic = std::make_unique<audio::mic_wasapi_t>();
 
-  if(mic->init(sample_rate)) {
+  if(mic->init(sample_rate, frame_size)) {
     return nullptr;
   }
 

tools/audio.cpp

@@ -26,7 +26,9 @@ const IID IID_IMMDeviceEnumerator    = __uuidof(IMMDeviceEnumerator);
 const IID IID_IAudioClient           = __uuidof(IAudioClient);
 const IID IID_IAudioCaptureClient    = __uuidof(IAudioCaptureClient);
 
+constexpr auto SAMPLE_RATE = 48000;
 int device_state_filter = DEVICE_STATE_ACTIVE;
+
 namespace audio {
 template<class T>
 void Release(T *p) {
@@ -66,19 +68,114 @@ public:
 const wchar_t *no_null(const wchar_t *str) {
   return str ? str : L"Unknown";
 }
-void print_device(device_t &device) {
-  HRESULT status;
 
-  audio::wstring_t::pointer wstring_p {};
+struct format_t {
+  std::string_view name;
+  int channels;
+  int channel_mask;
+} formats [] {
+  {
+    "Mono"sv,
+    1,
+    SPEAKER_FRONT_CENTER
+  },
+  {
+    "Stereo"sv,
+    2,
+    SPEAKER_FRONT_LEFT | SPEAKER_FRONT_RIGHT
+  },
+  {
+    "Surround 5.1"sv,
+    6,
+    SPEAKER_FRONT_LEFT    |
+    SPEAKER_FRONT_RIGHT   |
+    SPEAKER_FRONT_CENTER  |
+    SPEAKER_LOW_FREQUENCY |
+    SPEAKER_BACK_LEFT     |
+    SPEAKER_BACK_RIGHT
+  }
+};
+
+void set_wave_format(audio::wave_format_t &wave_format, const format_t &format) {
+  wave_format->nChannels = format.channels;
+  wave_format->nBlockAlign = wave_format->nChannels * wave_format->wBitsPerSample / 8;
+  wave_format->nAvgBytesPerSec = wave_format->nSamplesPerSec * wave_format->nBlockAlign;
+
+  if(wave_format->wFormatTag == WAVE_FORMAT_EXTENSIBLE) {
+    ((PWAVEFORMATEXTENSIBLE)wave_format.get())->dwChannelMask = format.channel_mask;
+  }
+}
+
+audio_client_t make_audio_client(device_t &device, const format_t &format) {
+  audio_client_t audio_client;
+  auto status = device->Activate(
+    IID_IAudioClient,
+    CLSCTX_ALL,
+    nullptr,
+    (void **) &audio_client);
+
+  if(FAILED(status)) {
+    std::cout << "Couldn't activate Device: [0x"sv << util::hex(status).to_string_view() << ']' << std::endl;
+
+    return nullptr;
+  }
+
+  wave_format_t wave_format;
+  status = audio_client->GetMixFormat(&wave_format);
+
+  if (FAILED(status)) {
+    std::cout << "Couldn't acquire Wave Format [0x"sv << util::hex(status).to_string_view() << ']' << std::endl;
+
+    return nullptr;
+  }
+
+  wave_format->wBitsPerSample = 16;
+  wave_format->nSamplesPerSec = SAMPLE_RATE;
+  switch(wave_format->wFormatTag) {
+    case WAVE_FORMAT_PCM:
+      break;
+    case WAVE_FORMAT_IEEE_FLOAT:
+      break;
+    case WAVE_FORMAT_EXTENSIBLE: {
+      auto wave_ex = (PWAVEFORMATEXTENSIBLE) wave_format.get();
+      if (IsEqualGUID(KSDATAFORMAT_SUBTYPE_IEEE_FLOAT, wave_ex->SubFormat)) {
+        wave_ex->SubFormat = KSDATAFORMAT_SUBTYPE_PCM;
+        wave_ex->Samples.wValidBitsPerSample = 16;
+        break;
+      }
+
+      std::cout << "Unsupported Sub Format for WAVE_FORMAT_EXTENSIBLE: [0x"sv << util::hex(wave_ex->SubFormat).to_string_view() << ']' << std::endl;
+    }
+    default:
+      std::cout << "Unsupported Wave Format: [0x"sv << util::hex(wave_format->wFormatTag).to_string_view() << ']' << std::endl;
+      return nullptr;
+  };
+
+  set_wave_format(wave_format, format);
+
+  status = audio_client->Initialize(
+    AUDCLNT_SHAREMODE_SHARED,
+    AUDCLNT_STREAMFLAGS_LOOPBACK | AUDCLNT_STREAMFLAGS_EVENTCALLBACK,
+    0, 0,
+    wave_format.get(),
+    nullptr);
+
+  if(status) {
+    return nullptr;
+  }
+
+  return audio_client;
+}
+
+void print_device(device_t &device) {
+  audio::wstring_t wstring;
   DWORD device_state;
 
   device->GetState(&device_state);
-  device->GetId(&wstring_p);
+  device->GetId(&wstring);
-  audio::wstring_t wstring { wstring_p };
 
-  audio::prop_t::pointer prop_p {};
+  audio::prop_t prop;
-  device->OpenPropertyStore(STGM_READ, &prop_p);
+  device->OpenPropertyStore(STGM_READ, &prop);
-  audio::prop_t prop { prop_p };
 
   prop_var_t adapter_friendly_name;
   prop_var_t device_friendly_name;
@@ -120,47 +217,12 @@ void print_device(device_t &device) {
     return;
   }
 
-  // Ensure WaveFromat is compatible
+  for(const auto &format : formats) {
-  audio_client_t::pointer audio_client_p{};
+    // Ensure WaveFromat is compatible
-  status = device->Activate(
+    auto audio_client = make_audio_client(device, format);
-    IID_IAudioClient,
-    CLSCTX_ALL,
-    nullptr,
-    (void **) &audio_client_p);
-  audio_client_t audio_client { audio_client_p };
 
-  if (FAILED(status)) {
+    std::cout << format.name << ": "sv << (!audio_client ? "unsupported"sv : "supported"sv) << std::endl;
-    std::cout << "Couldn't activate Device: [0x"sv << util::hex(status).to_string_view() << ']' << std::endl;
-
-    return;
   }
-
-  wave_format_t::pointer wave_format_p{};
-  status = audio_client->GetMixFormat(&wave_format_p);
-  wave_format_t wave_format { wave_format_p };
-
-  if (FAILED(status)) {
-    std::cout << "Couldn't acquire Wave Format [0x"sv << util::hex(status).to_string_view() << ']' << std::endl;
-
-    return;
-  }
-
-  switch(wave_format->wFormatTag) {
-    case WAVE_FORMAT_PCM:
-      break;
-    case WAVE_FORMAT_IEEE_FLOAT:
-      break;
-    case WAVE_FORMAT_EXTENSIBLE: {
-      auto wave_ex = (PWAVEFORMATEXTENSIBLE) wave_format.get();
-      if (IsEqualGUID(KSDATAFORMAT_SUBTYPE_IEEE_FLOAT, wave_ex->SubFormat)) {
-        break;
-      }
-
-      std::cout << "Unsupported Sub Format for WAVE_FORMAT_EXTENSIBLE: [0x"sv << util::hex(wave_ex->SubFormat).to_string_view() << ']' << std::endl;
-    }
-    default:
-      std::cout << "Unsupported Wave Format: [0x"sv << util::hex(wave_format->wFormatTag).to_string_view() << ']' << std::endl;
-  };
 }
 }
 
@@ -213,14 +275,13 @@ int main(int argc, char *argv[]) {
 
   HRESULT status;
 
-  audio::device_enum_t::pointer device_enum_p{};
+  audio::device_enum_t device_enum;
   status = CoCreateInstance(
     CLSID_MMDeviceEnumerator,
     nullptr,
     CLSCTX_ALL,
     IID_IMMDeviceEnumerator,
-    (void **) &device_enum_p);
+    (void **) &device_enum);
-  audio::device_enum_t device_enum { device_enum_p };
 
   if (FAILED(status)) {
     std::cout << "Couldn't create Device Enumerator: [0x"sv << util::hex(status).to_string_view() << ']' << std::endl;
@@ -228,9 +289,8 @@ int main(int argc, char *argv[]) {
     return -1;
   }
 
-  audio::collection_t::pointer collection_p {};
+  audio::collection_t collection;
-  status = device_enum->EnumAudioEndpoints(eRender, DEVICE_STATEMASK_ALL, &collection_p);
+  status = device_enum->EnumAudioEndpoints(eRender, DEVICE_STATEMASK_ALL, &collection);
-  audio::collection_t collection { collection_p };
 
   if (FAILED(status)) {
     std::cout << "Couldn't enumerate: [0x"sv << util::hex(status).to_string_view() << ']' << std::endl;
@@ -243,9 +303,8 @@ int main(int argc, char *argv[]) {
 
   std::cout << "====== Found "sv << count << " potential audio devices ======"sv << std::endl;
   for(auto x = 0; x < count; ++x) {
-    audio::device_t::pointer device_p {};
+    audio::device_t device;
-    collection->Item(x, &device_p);
+    collection->Item(x, &device);
-    audio::device_t device { device_p };
 
     audio::print_device(device);
   }