From 8ea7113d2197ce801797a7b67cdd9301a86a0961 Mon Sep 17 00:00:00 2001 From: Auzan Muhammad Date: Fri, 25 Jan 2019 15:06:18 +0700 Subject: [PATCH] update --- CMakeLists.txt | 8 ++ include/mumlib.hpp | 8 +- include/mumlib/Audio.hpp | 31 +++++++- src/Audio.cpp | 168 ++++++++++++++++++++++++++++++++++++--- src/Transport.cpp | 2 +- src/mumlib.cpp | 109 +++++++++++++++---------- 6 files changed, 267 insertions(+), 59 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9fbbab3..aea43b2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -42,9 +42,15 @@ set(MUMLIB_SRC src/Audio.cpp ) +set(MUMLIB_LIBRARIES + speex + speexdsp + ) + PROTOBUF_GENERATE_CPP(PROTO_SRCS PROTO_HDRS Mumble.proto) add_library(mumlib SHARED ${MUMLIB_SRC} ${MUMLIB_PUBLIC_HEADERS} ${MUMLIB_PRIVATE_HEADERS} ${PROTO_SRCS} ${PROTO_HDRS}) +target_link_libraries(mumlib ${MUMLIB_LIBRARIES}) target_link_libraries(mumlib ${PROTOBUF_LIBRARIES}) target_link_libraries(mumlib ${Boost_LIBRARIES}) target_link_libraries(mumlib ${OPENSSL_LIBRARIES}) @@ -54,3 +60,5 @@ target_link_libraries(mumlib ${OPUS_LIBRARIES}) add_executable(mumlib_example mumlib_example.cpp) target_link_libraries(mumlib_example mumlib) + +install(TARGETS mumlib LIBRARY DESTINATION lib) diff --git a/include/mumlib.hpp b/include/mumlib.hpp index cab8e31..08297d1 100644 --- a/include/mumlib.hpp +++ b/include/mumlib.hpp @@ -79,9 +79,9 @@ namespace mumlib { void joinChannel(std::string channelName); - void sendVoiceTarget(mumlib::VoiceTargetType type, int targetId, int id); + void sendVoiceTarget(int targetId, mumlib::VoiceTargetType type, int sessionId); - bool sendVoiceTarget(mumlib::VoiceTargetType type, int targetId, std::string name); + void sendVoiceTarget(int targetId, mumlib::VoiceTargetType type, std::string name, int &error); void sendUserState(mumlib::UserState state, bool val); @@ -93,5 +93,9 @@ namespace mumlib { int getChannelIdBy(std::string channelName); int getUserIdBy(std::string userName); + + bool isSessionIdValid(int sessionId); + + bool isChannelIdValid(int channelId); }; } diff --git a/include/mumlib/Audio.hpp b/include/mumlib/Audio.hpp index 7e1433a..d75ac70 100644 --- a/include/mumlib/Audio.hpp +++ b/include/mumlib/Audio.hpp @@ -4,6 +4,8 @@ #include +#include + #include namespace mumlib { @@ -26,19 +28,30 @@ namespace mumlib { class Audio : boost::noncopyable { public: - explicit Audio(int opusSampleRate=DEFAULT_OPUS_SAMPLE_RATE, - int opusEncoderBitrate=DEFAULT_OPUS_ENCODER_BITRATE, + explicit Audio(int sampleRate=DEFAULT_OPUS_SAMPLE_RATE, + int bitrate=DEFAULT_OPUS_ENCODER_BITRATE, int channels=DEFAULT_OPUS_NUM_CHANNELS); virtual ~Audio(); IncomingAudioPacket decodeIncomingAudioPacket(uint8_t *inputBuffer, int inputBufferLength); + void addFrameToBuffer(uint8_t *inputBuffer, int inputLength, int sequence); + + void fetchAudio(uint8_t *inputBuffer, int bufferOffset, int inputLength); + + void mixAudio(uint8_t *inputBuffer, uint8_t *outputBuffer, int bufferOffset, int inputLength); + + void resizeBuffer(); + std::pair decodeOpusPayload(uint8_t *inputBuffer, int inputLength, int16_t *pcmBuffer, int pcmBufferSize); + std::pair decodeOpusPayload(int16_t *pcmBuffer, + int pcmBufferSize); + int encodeAudioPacket( int target, int16_t *inputPcmBuffer, @@ -52,14 +65,26 @@ namespace mumlib { void resetEncoder(); + void resetJitterBuffer(); + private: log4cpp::Category &logger; OpusDecoder *opusDecoder; OpusEncoder *opusEncoder; + JitterBuffer *jbBuffer; + + mutex m_jitter_mutex; int64_t outgoingSequenceNumber; - int sampleRate; + + unsigned int mSampleRate; + unsigned int mChannels; + unsigned int mFrameSize; + unsigned int mAudioBufferSize; + + float *mFadeIn; + float *mFadeOut; std::chrono::time_point lastEncodedAudioPacketTimestamp; }; diff --git a/src/Audio.cpp b/src/Audio.cpp index 60277df..80d8b72 100644 --- a/src/Audio.cpp +++ b/src/Audio.cpp @@ -4,30 +4,69 @@ static boost::posix_time::seconds RESET_SEQUENCE_NUMBER_INTERVAL(5); -mumlib::Audio::Audio(int opusSampleRate, int opusEncoderBitrate, int channels) +mumlib::Audio::Audio(int sampleRate, int bitrate, int channels) : logger(log4cpp::Category::getInstance("mumlib.Audio")), opusDecoder(nullptr), opusEncoder(nullptr), outgoingSequenceNumber(0) { - int error; - this->sampleRate = opusSampleRate; + int error, ret; + mSampleRate = sampleRate; + mChannels = channels; + mFrameSize = sampleRate / 100; + mAudioBufferSize = mFrameSize; + mAudioBufferSize *= 12; - opusDecoder = opus_decoder_create(opusSampleRate, channels, &error); + opusDecoder = opus_decoder_create(sampleRate, channels, &error); if (error != OPUS_OK) { throw AudioException((boost::format("failed to initialize OPUS decoder: %s") % opus_strerror(error)).str()); } - opusEncoder = opus_encoder_create(opusSampleRate, channels, OPUS_APPLICATION_VOIP, &error); + opusEncoder = opus_encoder_create(sampleRate, channels, OPUS_APPLICATION_VOIP, &error); if (error != OPUS_OK) { throw AudioException((boost::format("failed to initialize OPUS encoder: %s") % opus_strerror(error)).str()); } - opus_encoder_ctl(opusEncoder, OPUS_SET_VBR(0)); - - setOpusEncoderBitrate(opusEncoderBitrate); + ret = opus_encoder_ctl(opusEncoder, OPUS_SET_BITRATE(bitrate)); + if (ret != OPUS_OK) { + throw AudioException((boost::format("failed to initialize transmission bitrate to %d B/s: %s") + % bitrate % opus_strerror(ret)).str()); + } + ret = opus_encoder_ctl(opusEncoder, OPUS_SET_VBR(0)); + if (ret != OPUS_OK) { + throw AudioException((boost::format("failed to initialize variable bitrate: %s") + % opus_strerror(ret)).str()); + } + ret = opus_encoder_ctl(opusEncoder, OPUS_SET_VBR_CONSTRAINT(0)); + if (ret != OPUS_OK) { + throw AudioException((boost::format("failed to initialize variable bitrate constraint: %s") + % opus_strerror(ret)).str()); + } + ret = opus_encoder_ctl(opusEncoder, OPUS_SET_BANDWIDTH(OPUS_BANDWIDTH_NARROWBAND)); + if (ret != OPUS_OK) { + throw AudioException((boost::format("failed to initialize bandwidth narrow: %s") + % opus_strerror(ret)).str()); + } + ret = opus_encoder_ctl(opusEncoder, OPUS_SET_MAX_BANDWIDTH(OPUS_BANDWIDTH_NARROWBAND)); + if (ret != OPUS_OK) { + throw AudioException((boost::format("failed to initialize maximum bandwidth narrow: %s") + % opus_strerror(ret)).str()); + } resetEncoder(); + + jbBuffer = jitter_buffer_init(mFrameSize); + int margin = 10 * mFrameSize; + jitter_buffer_ctl(jbBuffer, JITTER_BUFFER_SET_MARGIN, &margin); + + mFadeIn = new float[mFrameSize]; + mFadeOut = new float[mFrameSize]; + + // Sine function to represent fade in/out. Period is FRAME_SIZE. + float mul = static_cast(M_PI / 2.0 * static_cast(mFrameSize)); + for(unsigned int i = 0; i < mFrameSize; i++) { + mFadeIn[i] = mFadeOut[mFrameSize - 1 - 1] = sinf(static_cast(i) * mul); + } } mumlib::Audio::~Audio() { @@ -38,6 +77,11 @@ mumlib::Audio::~Audio() { if (opusEncoder) { opus_encoder_destroy(opusEncoder); } + + jitter_buffer_destroy(jbBuffer); + + delete[] mFadeIn; + delete[] mFadeOut; } void mumlib::Audio::setOpusEncoderBitrate(int bitrate) { @@ -57,6 +101,99 @@ int mumlib::Audio::getOpusEncoderBitrate() { return bitrate; } +void mumlib::Audio::addFrameToBuffer(uint8_t *inputBuffer, int inputLength, int sequence) { + int dataPointer = 0; + VarInt varInt(inputBuffer); + int opusDataLength = varInt.getValue(); + dataPointer += varInt.getEncoded().size(); + bool lastPacket = (opusDataLength & 0x2000) != 0; + opusDataLength &= 0x1fff; + + auto *packet = reinterpret_cast(&inputBuffer[dataPointer]); + int frame = opus_packet_get_nb_frames(packet, opusDataLength); + int samples = frame * opus_packet_get_samples_per_frame(packet, mSampleRate); + + if(not sequence) { + resetJitterBuffer(); + } + + JitterBufferPacket jbPacket; + jbPacket.data = reinterpret_cast(&inputBuffer[dataPointer]); + jbPacket.len = opusDataLength; + jbPacket.span = samples; + jbPacket.timestamp = mFrameSize * sequence; + jbPacket.user_data = lastPacket; + + jitter_buffer_put(jbBuffer, &jbPacket); +} + +std::pair mumlib::Audio::decodeOpusPayload(int16_t *pcmBuffer, int pcmBufferSize) { + int avail = 0; + int ts = jitter_buffer_get_pointer_timestamp(jbBuffer); + jitter_buffer_ctl(jbBuffer, JITTER_BUFFER_GET_AVAILABLE_COUNT, &avail); + + char data[4096]; + JitterBufferPacket jbPacket; + jbPacket.data = data; + jbPacket.len = 4096; + + spx_int32_t startofs = 0; + int opusDataLength; + int outputSize; + spx_uint32_t lastPacket; + + if(jitter_buffer_get(jbBuffer, &jbPacket, mFrameSize, &startofs) == JITTER_BUFFER_OK) { + opusDataLength = jbPacket.len; + lastPacket = jbPacket.user_data; + + } else { + jitter_buffer_update_delay(jbBuffer, &jbPacket, NULL); + } + + if(opusDataLength) { + outputSize = opus_decode(opusDecoder, + reinterpret_cast(jbPacket.data), + jbPacket.len, + pcmBuffer, + pcmBufferSize, 0); + } else { + outputSize = opus_decode(opusDecoder, + NULL, 0, pcmBuffer, pcmBufferSize, 0); + } + + if(outputSize < 0) { + outputSize = mFrameSize; + memset(pcmBuffer, 0, mFrameSize * sizeof(float)); + } + + if(lastPacket) { + for(unsigned int i = 0; i < mFrameSize; i++) + pcmBuffer[i] *= mFadeOut[i]; + } + + for (int i = outputSize / mFrameSize; i > 0; --i) { + jitter_buffer_tick(jbBuffer); + } + + logger.debug("%d B of Opus data decoded to %d PCM samples, last packet: %d.", + opusDataLength, outputSize, lastPacket); + + return std::make_pair(outputSize, lastPacket); +} + +void mumlib::Audio::mixAudio(uint8_t *inputBuffer, uint8_t *outputBuffer, int bufferOffset, int inputLength) { + for(int i = 0; i < inputLength; i++) { + float mix = 0; + + // Clip to [-1,1] + if(mix > 1) + mix = 1; + else if(mix < -1) + mix = -1; + outputBuffer[i + bufferOffset] = mix; + } +} + std::pair mumlib::Audio::decodeOpusPayload(uint8_t *inputBuffer, int inputLength, int16_t *pcmBuffer, @@ -83,11 +220,11 @@ std::pair mumlib::Audio::decodeOpusPayload(uint8_t *inputBuffer, // Lost packets can be replaced with loss concealment by calling the decoder with a null pointer and zero length for the missing packet. // A single codec state may only be accessed from a single thread at a time and any required locking must be performed by the caller. // Separate streams must be decoded with separate decoder states and can be decoded in parallel unless the library was compiled with NONTHREADSAFE_PSEUDOSTACK defined. - - int frame = opus_packet_get_nb_frames(&inputBuffer[dataPointer], opusDataLength); - int samples = frame * opus_packet_get_samples_per_frame(&inputBuffer[dataPointer], sampleRate); + auto *packet = reinterpret_cast(&inputBuffer[dataPointer]); + int frame = opus_packet_get_nb_frames(packet, opusDataLength); + int samples = frame * opus_packet_get_samples_per_frame(packet, mSampleRate); int outputSize = opus_decode(opusDecoder, - reinterpret_cast(&inputBuffer[dataPointer]), + packet, opusDataLength, pcmBuffer, pcmBufferSize, @@ -143,7 +280,7 @@ int mumlib::Audio::encodeAudioPacket(int target, int16_t *inputPcmBuffer, int in memcpy(outputBuffer, &header[0], header.size()); memcpy(outputBuffer + header.size(), tmpOpusBuffer, (size_t) outputSize); - int incrementNumber = 100 * inputLength / sampleRate; + int incrementNumber = 100 * inputLength / mSampleRate; outgoingSequenceNumber += incrementNumber; @@ -162,6 +299,11 @@ void mumlib::Audio::resetEncoder() { outgoingSequenceNumber = 0; } +void mumlib::Audio::resetJitterBuffer() { + logger.debug("Last audio packet, resetting jitter buffer"); + jitter_buffer_reset(jbBuffer); +} + mumlib::IncomingAudioPacket mumlib::Audio::decodeIncomingAudioPacket(uint8_t *inputBuffer, int inputBufferLength) { mumlib::IncomingAudioPacket incomingAudioPacket{}; diff --git a/src/Transport.cpp b/src/Transport.cpp index 01b712a..83a383a 100644 --- a/src/Transport.cpp +++ b/src/Transport.cpp @@ -81,7 +81,7 @@ void mumlib::Transport::connect( doReceiveUdp(); } - + ip::tcp::resolver resolverTcp(ioService); ip::tcp::resolver::query queryTcp(host, to_string(port)); diff --git a/src/mumlib.cpp b/src/mumlib.cpp index c8101b5..eb33dad 100644 --- a/src/mumlib.cpp +++ b/src/mumlib.cpp @@ -8,7 +8,7 @@ #include #include -#include +#include #include #include @@ -33,6 +33,8 @@ namespace mumlib { int sessionId = 0; int channelId = 0; + int64_t seq = 0; + mutex jitter_mutex; std::vector listMumbleUser; std::vector listMumbleChannel; @@ -68,10 +70,21 @@ namespace mumlib { // todo: multiple users speaking simultaneously (Issue #3) // something weird while decoding the opus payload int16_t pcmData[5000]; - auto status = audio.decodeOpusPayload(incomingAudioPacket.audioPayload, - incomingAudioPacket.audioPayloadLength, - pcmData, - 5000); + + audio.addFrameToBuffer(incomingAudioPacket.audioPayload, + incomingAudioPacket.audioPayloadLength, + seq); + + auto status = audio.decodeOpusPayload(pcmData, 5000); + + // auto status = audio.decodeOpusPayload(incomingAudioPacket.audioPayload, + // incomingAudioPacket.audioPayloadLength, + // pcmData, + // 5000); + + if(status.second) seq = 0; else seq++; + + // logger.warn("Decode audio: %d , seq %d", incomingAudioPacket.sessionId, seq); callback.audio(incomingAudioPacket.target, incomingAudioPacket.sessionId, @@ -377,9 +390,12 @@ namespace mumlib { } void listUserRemovedBy(int sessionId) { - for(int i = 0; i < listMumbleUser.size(); i++) - if(listMumbleUser[i].sessionId == sessionId) + for(int i = 0; i < listMumbleUser.size(); i++) { + if(listMumbleUser[i].sessionId == sessionId) { listMumbleUser.erase(listMumbleUser.begin() + i); + return; + } + } } bool isListChannelContains(int channelId) { @@ -390,9 +406,12 @@ namespace mumlib { } void listChannelRemovedBy(int channelId) { - for(int i = 0; i < listMumbleChannel.size(); i++) - if(listMumbleChannel[i].channelId == channelId) + for(int i = 0; i < listMumbleChannel.size(); i++) { + if(listMumbleChannel[i].channelId == channelId) { listMumbleChannel.erase(listMumbleChannel.begin() + i); + return; + } + } } }; @@ -474,6 +493,8 @@ namespace mumlib { } void Mumlib::joinChannel(int channelId) { + if(!isChannelIdValid(channelId)) // when channel has not been registered / create + return; MumbleProto::UserState userState; userState.set_channel_id(channelId); impl->transport.sendControlMessage(MessageType::USERSTATE, userState); @@ -482,11 +503,10 @@ namespace mumlib { void Mumlib::joinChannel(string name) { int channelId = Mumlib::getChannelIdBy(name); - if(!channelId < 0) // when channel has not been registered / create - Mumlib::joinChannel(channelId); + Mumlib::joinChannel(channelId); } - void Mumlib::sendVoiceTarget(VoiceTargetType type, int targetId, int id) { + void Mumlib::sendVoiceTarget(int targetId, VoiceTargetType type, int id) { MumbleProto::VoiceTarget voiceTarget; MumbleProto::VoiceTarget_Target voiceTargetTarget; switch(type) { @@ -507,8 +527,8 @@ namespace mumlib { impl->transport.sendControlMessage(MessageType::VOICETARGET, voiceTarget); } - bool Mumlib::sendVoiceTarget(VoiceTargetType type, int targetId, string name) { - int id = -1; + void Mumlib::sendVoiceTarget(int targetId, VoiceTargetType type, string name, int &error) { + int id; switch(type) { case VoiceTargetType::CHANNEL: id = getChannelIdBy(name); @@ -519,10 +539,9 @@ namespace mumlib { default: break; } - if(id < 0) - return false; - sendVoiceTarget(type, targetId, id); - return true; + error = id < 0 ? 1: 0; + if(error) return; + sendVoiceTarget(targetId, type, id); } void Mumlib::sendUserState(mumlib::UserState field, bool val) { @@ -560,26 +579,22 @@ namespace mumlib { void Mumlib::sendUserState(mumlib::UserState field, std::string val) { MumbleProto::UserState userState; + + // if comment longer than 128 bytes, we need to set the SHA1 hash + // http://www.askyb.com/cpp/openssl-sha1-hashing-example-in-cpp/ + unsigned char digest[SHA_DIGEST_LENGTH]; + char mdString[SHA_DIGEST_LENGTH * 2 + 1]; + SHA1((unsigned char*) val.c_str(), val.size(), digest); + for(int i = 0; i < SHA_DIGEST_LENGTH; i++) + sprintf(&mdString[i*2], "%02x", (unsigned int) digest[i]); + switch (field) { case UserState::COMMENT: - - if(val.size() < 128) { + if(val.size() < 128) userState.set_comment(val); - } else { - // if comment longer than 128 bytes, we need to set the SHA1 hash - boost::uuids::detail::sha1 sha1; - uint hash[5]; - sha1.process_bytes(val.c_str(), val.size()); - sha1.get_digest(hash); - - std::stringstream valStream; - for(std::size_t i=0; i listMumbleChannel = impl->listMumbleChannel; - int channelId = -1; for(int i = 0; i < listMumbleChannel.size(); i++) if(listMumbleChannel[i].name == name) - channelId = listMumbleChannel[i].channelId; - return channelId; + return listMumbleChannel[i].channelId; + return -1; } int Mumlib::getUserIdBy(string name) { vector listMumbleUser = impl->listMumbleUser; - int sessionId = -1; for(int i = 0; i < listMumbleUser.size(); i++) if(listMumbleUser[i].name == name) - sessionId = listMumbleUser[i].sessionId; - return sessionId; + return listMumbleUser[i].sessionId; + return -1; + } + + bool Mumlib::isSessionIdValid(int sessionId) { + vector listMumbleUser = impl->listMumbleUser; + for(int i = 0; i < listMumbleUser.size(); i++) + if(listMumbleUser[i].sessionId == sessionId) + return true; + return false; + } + + bool Mumlib::isChannelIdValid(int channelId) { + vector listMumbleChannel = impl->listMumbleChannel; + for(int i = 0; i < listMumbleChannel.size(); i++) + if(listMumbleChannel[i].channelId == channelId) + return true; + return false; } }