From ae1e71fffc67a010bd58de4fd2f73a1f20fbb21f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20S=C5=82omkowski?= Date: Tue, 17 Nov 2015 22:53:57 +0100 Subject: [PATCH] Add sequence and session numbers to audio callbacks. #3 --- include/mumlib.hpp | 2 +- include/mumlib/Audio.hpp | 18 ++++++-- include/mumlib/Callback.hpp | 12 ++++++ mumlib_example.cpp | 6 ++- src/Audio.cpp | 83 +++++++++++++++++++++++-------------- src/Callback.cpp | 16 +++++-- src/mumlib.cpp | 33 ++++++++++++--- 7 files changed, 124 insertions(+), 46 deletions(-) diff --git a/include/mumlib.hpp b/include/mumlib.hpp index 67e2855..b64f221 100644 --- a/include/mumlib.hpp +++ b/include/mumlib.hpp @@ -27,7 +27,7 @@ namespace mumlib { Mumlib(Callback &callback, io_service &ioService); - ~Mumlib(); + virtual ~Mumlib(); void connect(string host, int port, string user, string password); diff --git a/include/mumlib/Audio.hpp b/include/mumlib/Audio.hpp index 765bc9b..b00120c 100644 --- a/include/mumlib/Audio.hpp +++ b/include/mumlib/Audio.hpp @@ -17,15 +17,27 @@ namespace mumlib { AudioException(string message) : MumlibException(message) { } }; + struct IncomingAudioPacket { + AudioPacketType type; + int target; + int64_t sessionId; + int64_t sequenceNumber; + uint8_t *audioPayload; + int audioPayloadLength; + }; + class Audio : boost::noncopyable { public: Audio(); - ~Audio(); + virtual ~Audio(); + IncomingAudioPacket decodeIncomingAudioPacket(uint8_t *inputBuffer, int inputBufferLength); - int decodeAudioPacket(AudioPacketType type, uint8_t *inputBuffer, int inputLength, int16_t *pcmBuffer, - int pcmBufferSize); + std::pair decodeOpusPayload(uint8_t *inputBuffer, + int inputLength, + int16_t *pcmBuffer, + int pcmBufferSize); int encodeAudioPacket( int target, diff --git a/include/mumlib/Callback.hpp b/include/mumlib/Callback.hpp index ebb712e..bc668d9 100644 --- a/include/mumlib/Callback.hpp +++ b/include/mumlib/Callback.hpp @@ -19,10 +19,16 @@ namespace mumlib { string os_version) { }; virtual void audio( + int target, + int sessionId, + int sequenceNumber, int16_t *pcm_data, uint32_t pcm_data_size) { }; virtual void unsupportedAudio( + int target, + int sessionId, + int sequenceNumber, uint8_t *encoded_audio_data, uint32_t encoded_audio_data_size) { }; @@ -156,10 +162,16 @@ namespace mumlib { string os_version); virtual void audio( + int target, + int sessionId, + int sequenceNumber, int16_t *pcm_data, uint32_t pcm_data_size); virtual void unsupportedAudio( + int target, + int sessionId, + int sequenceNumber, uint8_t *encoded_audio_data, uint32_t encoded_audio_data_size); diff --git a/mumlib_example.cpp b/mumlib_example.cpp index e024f19..870e792 100644 --- a/mumlib_example.cpp +++ b/mumlib_example.cpp @@ -8,7 +8,11 @@ class MyCallback : public mumlib::BasicCallback { public: mumlib::Mumlib *mum; - virtual void audio(int16_t *pcm_data, uint32_t pcm_data_size) { + virtual void audio(int target, + int sessionId, + int sequenceNumber, + int16_t *pcm_data, + uint32_t pcm_data_size) { mum->sendAudioData(pcm_data, pcm_data_size); } diff --git a/src/Audio.cpp b/src/Audio.cpp index 091fc26..b117a4f 100644 --- a/src/Audio.cpp +++ b/src/Audio.cpp @@ -37,34 +37,25 @@ mumlib::Audio::~Audio() { } } -int mumlib::Audio::decodeAudioPacket(AudioPacketType type, - uint8_t *inputBuffer, - int inputLength, - int16_t *pcmBuffer, - int pcmBufferSize) { - - if (type != AudioPacketType::OPUS) { - throw AudioException("codecs other than OPUS are not supported"); - } - - int target = inputBuffer[0] & 0x1F; - - int64_t sessionId; - int64_t sequenceNumber; +std::pair mumlib::Audio::decodeOpusPayload(uint8_t *inputBuffer, + int inputLength, + int16_t *pcmBuffer, + int pcmBufferSize) { int64_t opusDataLength; - std::array varInts = {&sessionId, &sequenceNumber, &opusDataLength}; - - int dataPointer = 1; - for (int64_t *val : varInts) { - VarInt varInt(&inputBuffer[dataPointer]); - *val = varInt.getValue(); - dataPointer += varInt.getEncoded().size(); - } + int dataPointer = 0; + VarInt varInt(inputBuffer); + opusDataLength = varInt.getValue(); + dataPointer += varInt.getEncoded().size(); bool lastPacket = (opusDataLength & 0x2000) != 0; opusDataLength = opusDataLength & 0x1fff; + if (inputLength < opusDataLength + dataPointer) { + throw AudioException((boost::format("invalid Opus payload (%d B): header %d B, expected Opus data length %d B") + % inputLength % dataPointer % opusDataLength).str()); + } + int outputSize = opus_decode(opusDecoder, reinterpret_cast(&inputBuffer[dataPointer]), opusDataLength, @@ -77,17 +68,10 @@ int mumlib::Audio::decodeAudioPacket(AudioPacketType type, opus_strerror(outputSize)).str()); } - logger.debug( - "Received %d B of OPUS data, decoded to %d B (target: %d, sessionID: %ld, seq num: %ld, last: %d).", - opusDataLength, - outputSize, - target, - sessionId, - sequenceNumber, - lastPacket); + logger.debug("%d B of Opus data decoded to %d PCM samples, last packet: %d.", + opusDataLength, outputSize, lastPacket); - - return outputSize; + return std::make_pair(outputSize, lastPacket); } int mumlib::Audio::encodeAudioPacket(int target, int16_t *inputPcmBuffer, int inputLength, uint8_t *outputBuffer, @@ -147,3 +131,38 @@ void mumlib::Audio::resetEncoder() { outgoingSequenceNumber = 0; } + +mumlib::IncomingAudioPacket mumlib::Audio::decodeIncomingAudioPacket(uint8_t *inputBuffer, int inputBufferLength) { + mumlib::IncomingAudioPacket incomingAudioPacket; + + incomingAudioPacket.type = static_cast((inputBuffer[0] & 0xE0) >> 5); + incomingAudioPacket.target = inputBuffer[0] & 0x1F; + + std::array varInts = {&incomingAudioPacket.sessionId, &incomingAudioPacket.sequenceNumber}; + + int dataPointer = 1; + for (int64_t *val : varInts) { + VarInt varInt(&inputBuffer[dataPointer]); + *val = varInt.getValue(); + dataPointer += varInt.getEncoded().size(); + } + + incomingAudioPacket.audioPayload = &inputBuffer[dataPointer]; + incomingAudioPacket.audioPayloadLength = inputBufferLength - dataPointer; + + if (dataPointer >= inputBufferLength) { + throw AudioException((boost::format("invalid incoming audio packet (%d B): header %d B") % inputBufferLength % + dataPointer).str()); + } + + logger.debug( + "Received %d B of audio packet, %d B header, %d B payload (target: %d, sessionID: %ld, seq num: %ld).", + inputBufferLength, + dataPointer, + incomingAudioPacket.audioPayloadLength, + incomingAudioPacket.target, + incomingAudioPacket.sessionId, + incomingAudioPacket.sequenceNumber); + + return incomingAudioPacket; +} diff --git a/src/Callback.cpp b/src/Callback.cpp index ffe97e0..ea8fb80 100644 --- a/src/Callback.cpp +++ b/src/Callback.cpp @@ -36,13 +36,23 @@ void mumlib::BasicCallback::version( } void mumlib::BasicCallback::audio( + int target, + int sessionId, + int sequenceNumber, int16_t *pcmData, uint32_t pcm_data_size) { - impl->logger.debug("audio: %d bytes of raw PCM data.", pcm_data_size); + impl->logger.debug("audio: %d bytes of raw PCM data, target: %d, session: %d, seq: %d.", + pcm_data_size, target, sessionId, sequenceNumber); } -void BasicCallback::unsupportedAudio(uint8_t *encoded_audio_data, uint32_t encoded_audio_data_size) { - impl->logger.debug("unsupportedAudio: received %d bytes of encoded data.", encoded_audio_data_size); +void BasicCallback::unsupportedAudio( + int target, + int sessionId, + int sequenceNumber, + uint8_t *encoded_audio_data, + uint32_t encoded_audio_data_size) { + impl->logger.debug("unsupportedAudio: received %d bytes of encoded data, target: %d, session: %d, seq: %d.", + encoded_audio_data_size, target, sessionId, sequenceNumber); } void BasicCallback::serverSync(string welcome_text, int32_t session, int32_t max_bandwidth, int64_t permissions) { diff --git a/src/mumlib.cpp b/src/mumlib.cpp index 8c0d93f..f140a81 100644 --- a/src/mumlib.cpp +++ b/src/mumlib.cpp @@ -45,7 +45,7 @@ namespace mumlib { transport(ioService, boost::bind(&_Mumlib_Private::processIncomingTcpMessage, this, _1, _2, _3), boost::bind(&_Mumlib_Private::processAudioPacket, this, _1, _2, _3)) { } - ~_Mumlib_Private() { + virtual ~_Mumlib_Private() { if (not externalIoService) { delete &ioService; } @@ -54,13 +54,34 @@ namespace mumlib { bool processAudioPacket(AudioPacketType type, uint8_t *buffer, int length) { logger.info("Got %d B of encoded audio data.", length); try { - int16_t pcmData[5000]; - int pcmDataLength = audio.decodeAudioPacket(type, buffer, length, pcmData, 5000); - callback.audio(pcmData, pcmDataLength); + auto incomingAudioPacket = audio.decodeIncomingAudioPacket(buffer, length); + + if (type == AudioPacketType::OPUS) { + int16_t pcmData[5000]; + auto status = audio.decodeOpusPayload(incomingAudioPacket.audioPayload, + incomingAudioPacket.audioPayloadLength, + pcmData, + 5000); + + callback.audio(incomingAudioPacket.target, + incomingAudioPacket.sessionId, + incomingAudioPacket.sequenceNumber, + pcmData, + status.first); + } else { + logger.warn("Incoming audio packet doesn't contain Opus data, calling unsupportedAudio callback."); + callback.unsupportedAudio(incomingAudioPacket.target, + incomingAudioPacket.sessionId, + incomingAudioPacket.sequenceNumber, + incomingAudioPacket.audioPayload, + incomingAudioPacket.audioPayloadLength); + } + } catch (mumlib::AudioException &exp) { - logger.warn("Audio decode error: %s, calling unsupportedAudio callback.", exp.what()); - callback.unsupportedAudio(buffer, length); + logger.error("Audio decode error: %s.", exp.what()); } + + return true; } private: