hack typo, adding frame to buffer

CMakeLists.txt

@@ -42,15 +42,23 @@ set(MUMLIB_SRC
         src/Audio.cpp
         )
 
+set(SPEEX_LIBRARIES
+        speex
+        speexdsp
+        )
+
 PROTOBUF_GENERATE_CPP(PROTO_SRCS PROTO_HDRS Mumble.proto)
 
 add_library(mumlib SHARED ${MUMLIB_SRC} ${MUMLIB_PUBLIC_HEADERS} ${MUMLIB_PRIVATE_HEADERS} ${PROTO_SRCS} ${PROTO_HDRS})
-target_link_libraries(mumlib ${PROTOBUF_LIBRARIES})
+target_link_libraries(mumlib 
-target_link_libraries(mumlib ${Boost_LIBRARIES})
+        ${SPEEX_LIBRARIES}
-target_link_libraries(mumlib ${OPENSSL_LIBRARIES})
+        ${PROTOBUF_LIBRARIES}
-target_link_libraries(mumlib ${LOG4CPP_LIBRARIES})
+        ${Boost_LIBRARIES}
-target_link_libraries(mumlib ${OPUS_LIBRARIES})
+        ${OPENSSL_LIBRARIES}
+        ${LOG4CPP_LIBRARIES}
+        ${OPUS_LIBRARIES})
 
+# add_executable(mumlib_example mumlib_example.cpp)
+# target_link_libraries(mumlib_example mumlib)
 
-add_executable(mumlib_example mumlib_example.cpp)
+install(TARGETS mumlib LIBRARY DESTINATION lib)
-target_link_libraries(mumlib_example mumlib)

include/mumlib.hpp

@@ -79,9 +79,9 @@ namespace mumlib {
 
         void joinChannel(std::string channelName);
 
-        void sendVoiceTarget(mumlib::VoiceTargetType type, int targetId, int id);
+        void sendVoiceTarget(int targetId, mumlib::VoiceTargetType type, int sessionId);
 
-        bool sendVoiceTarget(mumlib::VoiceTargetType type, int targetId, std::string name);
+        void sendVoiceTarget(int targetId, mumlib::VoiceTargetType type, std::string name, int &error);
 
         void sendUserState(mumlib::UserState state, bool val);
 
@@ -93,5 +93,9 @@ namespace mumlib {
         int getChannelIdBy(std::string channelName);
 
         int getUserIdBy(std::string userName);
+
+        bool isSessionIdValid(int sessionId);
+
+        bool isChannelIdValid(int channelId);
     };
 }

include/mumlib/Audio.hpp

@@ -4,6 +4,8 @@
 
 #include <opus/opus.h>
 
+#include <speex/speex_jitter.h>
+
 #include <chrono>
 
 namespace mumlib {
@@ -26,14 +28,24 @@ namespace mumlib {
 
     class Audio : boost::noncopyable {
     public:
-        explicit Audio(int opusSampleRate=DEFAULT_OPUS_SAMPLE_RATE,
+        explicit Audio(int sampleRate=DEFAULT_OPUS_SAMPLE_RATE,
-                       int opusEncoderBitrate=DEFAULT_OPUS_ENCODER_BITRATE,
+                       int bitrate=DEFAULT_OPUS_ENCODER_BITRATE,
                        int channels=DEFAULT_OPUS_NUM_CHANNELS);
 
         virtual ~Audio();
 
         IncomingAudioPacket decodeIncomingAudioPacket(uint8_t *inputBuffer, int inputBufferLength);
 
+        void addFrameToBuffer(uint8_t *inputBuffer, int inputLength, int sequence);
+
+        // todo: mix audio
+        void mixAudio(uint8_t *dest, uint8_t *src, int bufferOffset, int inputLength);
+
+        void resizeBuffer();
+
+        std::pair<int, bool> decodeOpusPayload(int16_t *pcmBuffer,
+                                               int pcmBufferSize);
+        
         std::pair<int, bool> decodeOpusPayload(uint8_t *inputBuffer,
                                                int inputLength,
                                                int16_t *pcmBuffer,
@@ -52,14 +64,24 @@ namespace mumlib {
 
         void resetEncoder();
 
+        void resetJitterBuffer();
+
     private:
         log4cpp::Category &logger;
 
         OpusDecoder *opusDecoder;
         OpusEncoder *opusEncoder;
+        JitterBuffer *jbBuffer;
 
         int64_t outgoingSequenceNumber;
-        int sampleRate;
+
+        unsigned int iSampleRate;
+        unsigned int iChannels;
+        unsigned int iFrameSize;
+        unsigned int iAudioBufferSize;
+
+        float *fFadeIn;
+        float *fFadeOut;
 
         std::chrono::time_point<std::chrono::system_clock> lastEncodedAudioPacketTimestamp;
     };

src/Audio.cpp

@@ -4,30 +4,69 @@
 
 static boost::posix_time::seconds RESET_SEQUENCE_NUMBER_INTERVAL(5);
 
-mumlib::Audio::Audio(int opusSampleRate, int opusEncoderBitrate, int channels)
+mumlib::Audio::Audio(int sampleRate, int bitrate, int channels)
         : logger(log4cpp::Category::getInstance("mumlib.Audio")),
           opusDecoder(nullptr),
           opusEncoder(nullptr),
-          outgoingSequenceNumber(0) {
+          outgoingSequenceNumber(0),
+          iSampleRate(sampleRate),
+          iChannels(channels) {
 
-    int error;
+    int error, ret;
-    this->sampleRate = opusSampleRate;
+    iFrameSize = sampleRate / 100;
+    iAudioBufferSize = iFrameSize;
+    iAudioBufferSize *= 12;
 
-    opusDecoder = opus_decoder_create(opusSampleRate, channels, &error);
+    opusDecoder = opus_decoder_create(sampleRate, channels, &error);
     if (error != OPUS_OK) {
         throw AudioException((boost::format("failed to initialize OPUS decoder: %s") % opus_strerror(error)).str());
     }
 
-    opusEncoder = opus_encoder_create(opusSampleRate, channels, OPUS_APPLICATION_VOIP, &error);
+    opusEncoder = opus_encoder_create(sampleRate, channels, OPUS_APPLICATION_VOIP, &error);
     if (error != OPUS_OK) {
         throw AudioException((boost::format("failed to initialize OPUS encoder: %s") % opus_strerror(error)).str());
     }
 
-    opus_encoder_ctl(opusEncoder, OPUS_SET_VBR(0));
+    ret = opus_encoder_ctl(opusEncoder, OPUS_SET_BITRATE(bitrate));
-
+    if (ret != OPUS_OK) {
-    setOpusEncoderBitrate(opusEncoderBitrate);
+        throw AudioException((boost::format("failed to initialize transmission bitrate to %d B/s: %s")
+                            % bitrate % opus_strerror(ret)).str());
+    }
+    ret = opus_encoder_ctl(opusEncoder, OPUS_SET_VBR(0));
+    if (ret != OPUS_OK) {
+        throw AudioException((boost::format("failed to initialize variable bitrate: %s") 
+                            % opus_strerror(ret)).str());
+    }
+    ret = opus_encoder_ctl(opusEncoder, OPUS_SET_VBR_CONSTRAINT(0));
+    if (ret != OPUS_OK) {
+        throw AudioException((boost::format("failed to initialize variable bitrate constraint: %s") 
+                            % opus_strerror(ret)).str());
+    }
+    ret = opus_encoder_ctl(opusEncoder, OPUS_SET_BANDWIDTH(OPUS_BANDWIDTH_NARROWBAND));
+    if (ret != OPUS_OK) {
+        throw AudioException((boost::format("failed to initialize bandwidth narrow: %s") 
+                            % opus_strerror(ret)).str());
+    }
+    ret = opus_encoder_ctl(opusEncoder, OPUS_SET_MAX_BANDWIDTH(OPUS_BANDWIDTH_NARROWBAND));
+    if (ret != OPUS_OK) {
+        throw AudioException((boost::format("failed to initialize maximum bandwidth narrow: %s") 
+                            % opus_strerror(ret)).str());
+    }
 
     resetEncoder();
+
+    jbBuffer = jitter_buffer_init(iFrameSize);
+    int margin = 10 * iFrameSize;
+    jitter_buffer_ctl(jbBuffer, JITTER_BUFFER_SET_MARGIN, &margin);
+
+    fFadeIn = new float[iFrameSize];
+    fFadeOut = new float[iFrameSize];
+
+    // Sine function to represent fade in/out. Period is FRAME_SIZE.
+    float mul = static_cast<float>(M_PI / 2.0 * static_cast<double>(iFrameSize));
+    for(unsigned int i = 0; i < iFrameSize; i++) {
+        fFadeIn[i] = fFadeOut[iFrameSize - 1 - 1] = sinf(static_cast<float>(i) * mul);
+    }
 }
 
 mumlib::Audio::~Audio() {
@@ -38,6 +77,11 @@ mumlib::Audio::~Audio() {
     if (opusEncoder) {
         opus_encoder_destroy(opusEncoder);
     }
+
+    jitter_buffer_destroy(jbBuffer);
+
+    delete[] fFadeIn;
+    delete[] fFadeOut;
 }
 
 void mumlib::Audio::setOpusEncoderBitrate(int bitrate) {
@@ -57,6 +101,105 @@ int mumlib::Audio::getOpusEncoderBitrate() {
     return bitrate;
 }
 
+void mumlib::Audio::addFrameToBuffer(uint8_t *inputBuffer, int inputLength, int sequence) {
+    int dataPointer = 0;
+    VarInt varInt(inputBuffer);
+    int opusDataLength = varInt.getValue();
+    dataPointer += varInt.getEncoded().size();
+    bool lastPacket = (opusDataLength & 0x2000) != 0;
+    opusDataLength &= 0x1fff;
+
+    auto *packet = reinterpret_cast<const unsigned char *>(&inputBuffer[dataPointer]);
+    int frame = opus_packet_get_nb_frames(packet, opusDataLength);
+    int samples = frame * opus_packet_get_samples_per_frame(packet, iSampleRate);
+    int channel = opus_packet_get_nb_channels(packet);
+
+    if(not sequence) {
+        resetJitterBuffer();
+    }
+
+    logger.info("Opus packet, frame: %d, samples: %d, channel: %d", frame, samples, channel);
+
+    JitterBufferPacket jbPacket;
+    jbPacket.data = reinterpret_cast<char *>(&inputBuffer[dataPointer]);
+    jbPacket.len = opusDataLength;
+    jbPacket.span = samples;
+    jbPacket.timestamp = iFrameSize * sequence;
+    jbPacket.user_data = lastPacket;
+        
+    jitter_buffer_put(jbBuffer, &jbPacket);
+}
+
+std::pair<int, bool> mumlib::Audio::decodeOpusPayload(int16_t *pcmBuffer, int pcmBufferSize) {
+    int avail = 0;
+    spx_uint32_t remaining = 0;
+    jitter_buffer_ctl(jbBuffer, JITTER_BUFFER_GET_AVAILABLE_COUNT, &avail);
+    jitter_buffer_remaining_span(jbBuffer, remaining);
+    int timestamp = jitter_buffer_get_pointer_timestamp(jbBuffer);
+
+    logger.warn("jbBufer, avail: %d, remain: %d, timestamp: %d", avail, remaining, timestamp);
+
+    char data[4096];
+    JitterBufferPacket jbPacket;
+    jbPacket.data = data;
+    jbPacket.len = 4096;
+
+    spx_int32_t startofs = 0;
+    int opusDataLength;
+    int outputSize;
+    spx_uint32_t lastPacket;
+
+    if(jitter_buffer_get(jbBuffer, &jbPacket, iFrameSize, &startofs) == JITTER_BUFFER_OK) {
+        opusDataLength = jbPacket.len;
+        lastPacket = jbPacket.user_data;
+    } else {
+        jitter_buffer_update_delay(jbBuffer, &jbPacket, NULL);
+    }
+    
+    if(opusDataLength) {
+        outputSize = opus_decode(opusDecoder, 
+                                    reinterpret_cast<const unsigned char *>(jbPacket.data),
+                                    jbPacket.len, 
+                                    pcmBuffer, 
+                                    pcmBufferSize, 0);
+    } else {
+        outputSize = opus_decode(opusDecoder, 
+                                    NULL, 0, pcmBuffer, pcmBufferSize, 0);        
+    }
+
+    if(outputSize < 0) {
+        outputSize = iFrameSize;
+        memset(pcmBuffer, 0, iFrameSize * sizeof(float));
+    }
+
+    if(lastPacket) {
+        for(unsigned int i = 0; i < iFrameSize; i++)
+            pcmBuffer[i] *= fFadeOut[i];
+    }
+
+    for (int i = outputSize / iFrameSize; i > 0; --i) {
+        jitter_buffer_tick(jbBuffer);
+    }
+    
+    logger.debug("%d B of Opus data decoded to %d PCM samples, last packet: %d.",
+                 opusDataLength, outputSize, lastPacket);
+
+    return std::make_pair(outputSize, lastPacket);
+}
+
+void mumlib::Audio::mixAudio(uint8_t *dest, uint8_t *src, int bufferOffset, int inputLength) {
+    for(int i = 0; i < inputLength; i++) {
+        float mix = 0;
+
+        // Clip to [-1,1]
+        if(mix > 1)
+            mix = 1;
+        else if(mix < -1)
+            mix = -1;
+        dest[i + bufferOffset] = mix;
+    }
+}
+
 std::pair<int, bool>  mumlib::Audio::decodeOpusPayload(uint8_t *inputBuffer,
                                                        int inputLength,
                                                        int16_t *pcmBuffer,
@@ -83,11 +226,11 @@ std::pair<int, bool>  mumlib::Audio::decodeOpusPayload(uint8_t *inputBuffer,
     // Lost packets can be replaced with loss concealment by calling the decoder with a null pointer and zero length for the missing packet.
     // A single codec state may only be accessed from a single thread at a time and any required locking must be performed by the caller. 
     // Separate streams must be decoded with separate decoder states and can be decoded in parallel unless the library was compiled with NONTHREADSAFE_PSEUDOSTACK defined.
-
+    auto *packet = reinterpret_cast<const unsigned char *>(&inputBuffer[dataPointer]);
-    int frame = opus_packet_get_nb_frames(&inputBuffer[dataPointer], opusDataLength);
+    int frame = opus_packet_get_nb_frames(packet, opusDataLength);
-    int samples = frame * opus_packet_get_samples_per_frame(&inputBuffer[dataPointer], sampleRate);
+    int samples = frame * opus_packet_get_samples_per_frame(packet, iSampleRate);
     int outputSize = opus_decode(opusDecoder,
-                                 reinterpret_cast<const unsigned char *>(&inputBuffer[dataPointer]),
+                                 packet,
                                  opusDataLength,
                                  pcmBuffer,
                                  pcmBufferSize,
@@ -143,7 +286,7 @@ int mumlib::Audio::encodeAudioPacket(int target, int16_t *inputPcmBuffer, int in
     memcpy(outputBuffer, &header[0], header.size());
     memcpy(outputBuffer + header.size(), tmpOpusBuffer, (size_t) outputSize);
 
-    int incrementNumber = 100 * inputLength / sampleRate;
+    int incrementNumber = 100 * inputLength / iSampleRate;
 
     outgoingSequenceNumber += incrementNumber;
 
@@ -162,6 +305,11 @@ void mumlib::Audio::resetEncoder() {
     outgoingSequenceNumber = 0;
 }
 
+void mumlib::Audio::resetJitterBuffer() {
+    logger.debug("Last audio packet, resetting jitter buffer");
+    jitter_buffer_reset(jbBuffer);
+}
+
 mumlib::IncomingAudioPacket mumlib::Audio::decodeIncomingAudioPacket(uint8_t *inputBuffer, int inputBufferLength) {
     mumlib::IncomingAudioPacket incomingAudioPacket{};
 

src/mumlib.cpp

@@ -8,7 +8,7 @@
 
 #include <boost/asio.hpp>
 #include <boost/bind.hpp>
-#include <boost/uuid/sha1.hpp>
+#include <openssl/sha.h>
 #include <log4cpp/Category.hh>
 
 #include <Mumble.pb.h>
@@ -33,6 +33,7 @@ namespace mumlib {
 
         int sessionId = 0;
         int channelId = 0;
+        int64_t seq = 0;
 
         std::vector<MumbleUser> listMumbleUser;
         std::vector<MumbleChannel> listMumbleChannel;
@@ -65,19 +66,27 @@ namespace mumlib {
                 auto incomingAudioPacket = audio.decodeIncomingAudioPacket(buffer, length);
 
                 if (type == AudioPacketType::OPUS) {
-                    // todo: multiple users speaking simultaneously (Issue #3)
-                    // something weird while decoding the opus payload
                     int16_t pcmData[5000];
-                    auto status = audio.decodeOpusPayload(incomingAudioPacket.audioPayload,
+
-                                                          incomingAudioPacket.audioPayloadLength,
+                    audio.addFrameToBuffer(incomingAudioPacket.audioPayload, 
-                                                          pcmData,
+                                        incomingAudioPacket.audioPayloadLength,
-                                                          5000);
+                                        seq);
+
+                    auto status = audio.decodeOpusPayload(pcmData, 5000);
+
+                    // auto status = audio.decodeOpusPayload(incomingAudioPacket.audioPayload,
+                    //                                       incomingAudioPacket.audioPayloadLength,
+                    //                                       pcmData,
+                    //                                       5000);
+
+                    if(status.second) seq = 0; else seq++;
 
                     callback.audio(incomingAudioPacket.target,
-                                   incomingAudioPacket.sessionId,
+                                incomingAudioPacket.sessionId,
-                                   incomingAudioPacket.sequenceNumber,
+                                incomingAudioPacket.sequenceNumber,
-                                   pcmData,
+                                pcmData,
-                                   status.first);
+                                status.first);
+                    
                 } else {
                     logger.warn("Incoming audio packet doesn't contain Opus data, calling unsupportedAudio callback.");
                     callback.unsupportedAudio(incomingAudioPacket.target,
@@ -377,9 +386,12 @@ namespace mumlib {
         }
 
         void listUserRemovedBy(int sessionId) {
-            for(int i = 0; i < listMumbleUser.size(); i++) 
+            for(int i = 0; i < listMumbleUser.size(); i++) {
-                if(listMumbleUser[i].sessionId == sessionId)
+                if(listMumbleUser[i].sessionId == sessionId) {
                     listMumbleUser.erase(listMumbleUser.begin() + i);
+                    return;
+                }
+            }
         }
 
         bool isListChannelContains(int channelId) {
@@ -390,9 +402,12 @@ namespace mumlib {
         }
 
         void listChannelRemovedBy(int channelId) {
-            for(int i = 0; i < listMumbleChannel.size(); i++) 
+            for(int i = 0; i < listMumbleChannel.size(); i++) {
-                if(listMumbleChannel[i].channelId == channelId)
+                if(listMumbleChannel[i].channelId == channelId) {
                     listMumbleChannel.erase(listMumbleChannel.begin() + i);
+                    return;
+                }
+            }
         }
     };
 
@@ -474,6 +489,8 @@ namespace mumlib {
     }
 
     void Mumlib::joinChannel(int channelId) {
+        if(!isChannelIdValid(channelId)) // when channel has not been registered / create
+            return;
         MumbleProto::UserState userState;
         userState.set_channel_id(channelId);
         impl->transport.sendControlMessage(MessageType::USERSTATE, userState);
@@ -482,11 +499,10 @@ namespace mumlib {
 
     void Mumlib::joinChannel(string name) {
         int channelId = Mumlib::getChannelIdBy(name);
-        if(!channelId < 0) // when channel has not been registered / create
+        Mumlib::joinChannel(channelId);
-            Mumlib::joinChannel(channelId);
     }
 
-    void Mumlib::sendVoiceTarget(VoiceTargetType type, int targetId, int id) {
+    void Mumlib::sendVoiceTarget(int targetId, VoiceTargetType type, int id) {
         MumbleProto::VoiceTarget voiceTarget;
         MumbleProto::VoiceTarget_Target voiceTargetTarget;
         switch(type) {
@@ -507,8 +523,8 @@ namespace mumlib {
         impl->transport.sendControlMessage(MessageType::VOICETARGET, voiceTarget);
     }
 
-    bool Mumlib::sendVoiceTarget(VoiceTargetType type, int targetId, string name) {
+    void Mumlib::sendVoiceTarget(int targetId, VoiceTargetType type, string name, int &error) {
-        int id = -1;
+        int id;
         switch(type) {
             case VoiceTargetType::CHANNEL: 
                 id = getChannelIdBy(name);
@@ -519,10 +535,9 @@ namespace mumlib {
             default:
                 break;
         }        
-        if(id < 0)
+        error = id < 0 ? 1: 0;
-            return false;
+        if(error) return;
-        sendVoiceTarget(type, targetId, id);
+        sendVoiceTarget(targetId, type, id);
-        return true;
     }
 
     void Mumlib::sendUserState(mumlib::UserState field, bool val) {
@@ -560,26 +575,22 @@ namespace mumlib {
 
     void Mumlib::sendUserState(mumlib::UserState field, std::string val) {
         MumbleProto::UserState userState;
+        
+        // if comment longer than 128 bytes, we need to set the SHA1 hash
+        // http://www.askyb.com/cpp/openssl-sha1-hashing-example-in-cpp/
+        unsigned char digest[SHA_DIGEST_LENGTH];
+        char mdString[SHA_DIGEST_LENGTH * 2 + 1];
 
+        SHA1((unsigned char*) val.c_str(), val.size(), digest);
+        for(int i = 0; i < SHA_DIGEST_LENGTH; i++)
+            sprintf(&mdString[i*2], "%02x", (unsigned int) digest[i]);
+	
         switch (field) {
             case UserState::COMMENT:
-                
+                if(val.size() < 128)
-                if(val.size() < 128) {
                     userState.set_comment(val);
-                } else {
+                else
-                    // if comment longer than 128 bytes, we need to set the SHA1 hash
+                    userState.set_comment_hash(mdString);
-                    boost::uuids::detail::sha1 sha1;
-                    uint hash[5];
-                    sha1.process_bytes(val.c_str(), val.size());
-                    sha1.get_digest(hash);
-
-                    std::stringstream valStream;
-                    for(std::size_t i=0; i<sizeof(hash)/sizeof(hash[0]); ++i) {
-                        valStream << std::hex << hash[i];
-                    }
-                    userState.set_comment_hash(valStream.str());
-                }
-                
                 break;
             default:
                 // in any other case, just ignore the command
@@ -591,19 +602,33 @@ namespace mumlib {
 
     int Mumlib::getChannelIdBy(string name) {
         vector<mumlib::MumbleChannel> listMumbleChannel = impl->listMumbleChannel;
-        int channelId = -1;
         for(int i = 0; i < listMumbleChannel.size(); i++)
             if(listMumbleChannel[i].name == name)
-                channelId = listMumbleChannel[i].channelId;
+                return listMumbleChannel[i].channelId;
-        return channelId;
+        return -1;
     }
 
     int Mumlib::getUserIdBy(string name) {
         vector<mumlib::MumbleUser> listMumbleUser = impl->listMumbleUser;
-        int sessionId = -1;
         for(int i = 0; i < listMumbleUser.size(); i++)
             if(listMumbleUser[i].name == name)
-                sessionId = listMumbleUser[i].sessionId;
+                return listMumbleUser[i].sessionId;
-        return sessionId;
+        return -1;
+    }
+
+    bool Mumlib::isSessionIdValid(int sessionId) {
+        vector<mumlib::MumbleUser> listMumbleUser = impl->listMumbleUser;
+        for(int i = 0; i < listMumbleUser.size(); i++)
+            if(listMumbleUser[i].sessionId == sessionId)
+                return true;
+        return false;
+    }
+
+    bool Mumlib::isChannelIdValid(int channelId) {
+        vector<mumlib::MumbleChannel> listMumbleChannel = impl->listMumbleChannel;
+        for(int i = 0; i < listMumbleChannel.size(); i++)
+            if(listMumbleChannel[i].channelId == channelId)
+                return true;
+        return false;
     }
 }