diff --git a/CMakeLists.txt b/CMakeLists.txt index aea43b2..e4e177f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -42,7 +42,7 @@ set(MUMLIB_SRC src/Audio.cpp ) -set(MUMLIB_LIBRARIES +set(SPEEX_LIBRARIES speex speexdsp ) @@ -50,15 +50,15 @@ set(MUMLIB_LIBRARIES PROTOBUF_GENERATE_CPP(PROTO_SRCS PROTO_HDRS Mumble.proto) add_library(mumlib SHARED ${MUMLIB_SRC} ${MUMLIB_PUBLIC_HEADERS} ${MUMLIB_PRIVATE_HEADERS} ${PROTO_SRCS} ${PROTO_HDRS}) -target_link_libraries(mumlib ${MUMLIB_LIBRARIES}) -target_link_libraries(mumlib ${PROTOBUF_LIBRARIES}) -target_link_libraries(mumlib ${Boost_LIBRARIES}) -target_link_libraries(mumlib ${OPENSSL_LIBRARIES}) -target_link_libraries(mumlib ${LOG4CPP_LIBRARIES}) -target_link_libraries(mumlib ${OPUS_LIBRARIES}) +target_link_libraries(mumlib + ${SPEEX_LIBRARIES} + ${PROTOBUF_LIBRARIES} + ${Boost_LIBRARIES} + ${OPENSSL_LIBRARIES} + ${LOG4CPP_LIBRARIES} + ${OPUS_LIBRARIES}) - -add_executable(mumlib_example mumlib_example.cpp) -target_link_libraries(mumlib_example mumlib) +# add_executable(mumlib_example mumlib_example.cpp) +# target_link_libraries(mumlib_example mumlib) install(TARGETS mumlib LIBRARY DESTINATION lib) diff --git a/include/mumlib/Audio.hpp b/include/mumlib/Audio.hpp index d75ac70..862eeef 100644 --- a/include/mumlib/Audio.hpp +++ b/include/mumlib/Audio.hpp @@ -38,19 +38,19 @@ namespace mumlib { void addFrameToBuffer(uint8_t *inputBuffer, int inputLength, int sequence); - void fetchAudio(uint8_t *inputBuffer, int bufferOffset, int inputLength); - - void mixAudio(uint8_t *inputBuffer, uint8_t *outputBuffer, int bufferOffset, int inputLength); + // todo: mix audio + void mixAudio(uint8_t *dest, uint8_t *src, int bufferOffset, int inputLength); void resizeBuffer(); + + std::pair decodeOpusPayload(int16_t *pcmBuffer, + int pcmBufferSize); std::pair decodeOpusPayload(uint8_t *inputBuffer, int inputLength, int16_t *pcmBuffer, int pcmBufferSize); - std::pair decodeOpusPayload(int16_t *pcmBuffer, - int pcmBufferSize); int encodeAudioPacket( int target, @@ -74,17 +74,15 @@ namespace mumlib { OpusEncoder *opusEncoder; JitterBuffer *jbBuffer; - mutex m_jitter_mutex; - int64_t outgoingSequenceNumber; - unsigned int mSampleRate; - unsigned int mChannels; - unsigned int mFrameSize; - unsigned int mAudioBufferSize; + unsigned int iSampleRate; + unsigned int iChannels; + unsigned int iFrameSize; + unsigned int iAudioBufferSize; - float *mFadeIn; - float *mFadeOut; + float *fFadeIn; + float *fFadeOut; std::chrono::time_point lastEncodedAudioPacketTimestamp; }; diff --git a/src/Audio.cpp b/src/Audio.cpp index 80d8b72..393b674 100644 --- a/src/Audio.cpp +++ b/src/Audio.cpp @@ -8,14 +8,14 @@ mumlib::Audio::Audio(int sampleRate, int bitrate, int channels) : logger(log4cpp::Category::getInstance("mumlib.Audio")), opusDecoder(nullptr), opusEncoder(nullptr), - outgoingSequenceNumber(0) { + outgoingSequenceNumber(0), + iSampleRate(sampleRate), + iChannels(channels) { int error, ret; - mSampleRate = sampleRate; - mChannels = channels; - mFrameSize = sampleRate / 100; - mAudioBufferSize = mFrameSize; - mAudioBufferSize *= 12; + iFrameSize = sampleRate / 100; + iAudioBufferSize = iFrameSize; + iAudioBufferSize *= 12; opusDecoder = opus_decoder_create(sampleRate, channels, &error); if (error != OPUS_OK) { @@ -55,17 +55,17 @@ mumlib::Audio::Audio(int sampleRate, int bitrate, int channels) resetEncoder(); - jbBuffer = jitter_buffer_init(mFrameSize); - int margin = 10 * mFrameSize; + jbBuffer = jitter_buffer_init(iFrameSize); + int margin = 10 * iFrameSize; jitter_buffer_ctl(jbBuffer, JITTER_BUFFER_SET_MARGIN, &margin); - mFadeIn = new float[mFrameSize]; - mFadeOut = new float[mFrameSize]; + fFadeIn = new float[iFrameSize]; + fFadeOut = new float[iFrameSize]; // Sine function to represent fade in/out. Period is FRAME_SIZE. - float mul = static_cast(M_PI / 2.0 * static_cast(mFrameSize)); - for(unsigned int i = 0; i < mFrameSize; i++) { - mFadeIn[i] = mFadeOut[mFrameSize - 1 - 1] = sinf(static_cast(i) * mul); + float mul = static_cast(M_PI / 2.0 * static_cast(iFrameSize)); + for(unsigned int i = 0; i < iFrameSize; i++) { + fFadeIn[i] = fFadeOut[iFrameSize - 1 - 1] = sinf(static_cast(i) * mul); } } @@ -80,8 +80,8 @@ mumlib::Audio::~Audio() { jitter_buffer_destroy(jbBuffer); - delete[] mFadeIn; - delete[] mFadeOut; + delete[] fFadeIn; + delete[] fFadeOut; } void mumlib::Audio::setOpusEncoderBitrate(int bitrate) { @@ -111,17 +111,20 @@ void mumlib::Audio::addFrameToBuffer(uint8_t *inputBuffer, int inputLength, int auto *packet = reinterpret_cast(&inputBuffer[dataPointer]); int frame = opus_packet_get_nb_frames(packet, opusDataLength); - int samples = frame * opus_packet_get_samples_per_frame(packet, mSampleRate); + int samples = frame * opus_packet_get_samples_per_frame(packet, iSampleRate); + int channel = opus_packet_get_nb_channels(packet); if(not sequence) { resetJitterBuffer(); } + logger.info("Opus packet, frame: %d, samples: %d, channel: %d", frame, samples, channel); + JitterBufferPacket jbPacket; jbPacket.data = reinterpret_cast(&inputBuffer[dataPointer]); jbPacket.len = opusDataLength; jbPacket.span = samples; - jbPacket.timestamp = mFrameSize * sequence; + jbPacket.timestamp = iFrameSize * sequence; jbPacket.user_data = lastPacket; jitter_buffer_put(jbBuffer, &jbPacket); @@ -129,8 +132,12 @@ void mumlib::Audio::addFrameToBuffer(uint8_t *inputBuffer, int inputLength, int std::pair mumlib::Audio::decodeOpusPayload(int16_t *pcmBuffer, int pcmBufferSize) { int avail = 0; - int ts = jitter_buffer_get_pointer_timestamp(jbBuffer); + spx_uint32_t remaining = 0; jitter_buffer_ctl(jbBuffer, JITTER_BUFFER_GET_AVAILABLE_COUNT, &avail); + jitter_buffer_remaining_span(jbBuffer, remaining); + int timestamp = jitter_buffer_get_pointer_timestamp(jbBuffer); + + logger.warn("jbBufer, avail: %d, remain: %d, timestamp: %d", avail, remaining, timestamp); char data[4096]; JitterBufferPacket jbPacket; @@ -142,10 +149,9 @@ std::pair mumlib::Audio::decodeOpusPayload(int16_t *pcmBuffer, int pc int outputSize; spx_uint32_t lastPacket; - if(jitter_buffer_get(jbBuffer, &jbPacket, mFrameSize, &startofs) == JITTER_BUFFER_OK) { + if(jitter_buffer_get(jbBuffer, &jbPacket, iFrameSize, &startofs) == JITTER_BUFFER_OK) { opusDataLength = jbPacket.len; lastPacket = jbPacket.user_data; - } else { jitter_buffer_update_delay(jbBuffer, &jbPacket, NULL); } @@ -162,16 +168,16 @@ std::pair mumlib::Audio::decodeOpusPayload(int16_t *pcmBuffer, int pc } if(outputSize < 0) { - outputSize = mFrameSize; - memset(pcmBuffer, 0, mFrameSize * sizeof(float)); + outputSize = iFrameSize; + memset(pcmBuffer, 0, iFrameSize * sizeof(float)); } if(lastPacket) { - for(unsigned int i = 0; i < mFrameSize; i++) - pcmBuffer[i] *= mFadeOut[i]; + for(unsigned int i = 0; i < iFrameSize; i++) + pcmBuffer[i] *= fFadeOut[i]; } - for (int i = outputSize / mFrameSize; i > 0; --i) { + for (int i = outputSize / iFrameSize; i > 0; --i) { jitter_buffer_tick(jbBuffer); } @@ -181,7 +187,7 @@ std::pair mumlib::Audio::decodeOpusPayload(int16_t *pcmBuffer, int pc return std::make_pair(outputSize, lastPacket); } -void mumlib::Audio::mixAudio(uint8_t *inputBuffer, uint8_t *outputBuffer, int bufferOffset, int inputLength) { +void mumlib::Audio::mixAudio(uint8_t *dest, uint8_t *src, int bufferOffset, int inputLength) { for(int i = 0; i < inputLength; i++) { float mix = 0; @@ -190,7 +196,7 @@ void mumlib::Audio::mixAudio(uint8_t *inputBuffer, uint8_t *outputBuffer, int bu mix = 1; else if(mix < -1) mix = -1; - outputBuffer[i + bufferOffset] = mix; + dest[i + bufferOffset] = mix; } } @@ -222,7 +228,7 @@ std::pair mumlib::Audio::decodeOpusPayload(uint8_t *inputBuffer, // Separate streams must be decoded with separate decoder states and can be decoded in parallel unless the library was compiled with NONTHREADSAFE_PSEUDOSTACK defined. auto *packet = reinterpret_cast(&inputBuffer[dataPointer]); int frame = opus_packet_get_nb_frames(packet, opusDataLength); - int samples = frame * opus_packet_get_samples_per_frame(packet, mSampleRate); + int samples = frame * opus_packet_get_samples_per_frame(packet, iSampleRate); int outputSize = opus_decode(opusDecoder, packet, opusDataLength, @@ -280,7 +286,7 @@ int mumlib::Audio::encodeAudioPacket(int target, int16_t *inputPcmBuffer, int in memcpy(outputBuffer, &header[0], header.size()); memcpy(outputBuffer + header.size(), tmpOpusBuffer, (size_t) outputSize); - int incrementNumber = 100 * inputLength / mSampleRate; + int incrementNumber = 100 * inputLength / iSampleRate; outgoingSequenceNumber += incrementNumber; diff --git a/src/mumlib.cpp b/src/mumlib.cpp index eb33dad..3c85cec 100644 --- a/src/mumlib.cpp +++ b/src/mumlib.cpp @@ -34,7 +34,6 @@ namespace mumlib { int sessionId = 0; int channelId = 0; int64_t seq = 0; - mutex jitter_mutex; std::vector listMumbleUser; std::vector listMumbleChannel; @@ -67,8 +66,6 @@ namespace mumlib { auto incomingAudioPacket = audio.decodeIncomingAudioPacket(buffer, length); if (type == AudioPacketType::OPUS) { - // todo: multiple users speaking simultaneously (Issue #3) - // something weird while decoding the opus payload int16_t pcmData[5000]; audio.addFrameToBuffer(incomingAudioPacket.audioPayload, @@ -84,13 +81,12 @@ namespace mumlib { if(status.second) seq = 0; else seq++; - // logger.warn("Decode audio: %d , seq %d", incomingAudioPacket.sessionId, seq); - callback.audio(incomingAudioPacket.target, - incomingAudioPacket.sessionId, - incomingAudioPacket.sequenceNumber, - pcmData, - status.first); + incomingAudioPacket.sessionId, + incomingAudioPacket.sequenceNumber, + pcmData, + status.first); + } else { logger.warn("Incoming audio packet doesn't contain Opus data, calling unsupportedAudio callback."); callback.unsupportedAudio(incomingAudioPacket.target,