diff --git a/CMakeLists.txt b/CMakeLists.txt index 9fbbab3..e4e177f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -42,15 +42,23 @@ set(MUMLIB_SRC src/Audio.cpp ) +set(SPEEX_LIBRARIES + speex + speexdsp + ) + PROTOBUF_GENERATE_CPP(PROTO_SRCS PROTO_HDRS Mumble.proto) add_library(mumlib SHARED ${MUMLIB_SRC} ${MUMLIB_PUBLIC_HEADERS} ${MUMLIB_PRIVATE_HEADERS} ${PROTO_SRCS} ${PROTO_HDRS}) -target_link_libraries(mumlib ${PROTOBUF_LIBRARIES}) -target_link_libraries(mumlib ${Boost_LIBRARIES}) -target_link_libraries(mumlib ${OPENSSL_LIBRARIES}) -target_link_libraries(mumlib ${LOG4CPP_LIBRARIES}) -target_link_libraries(mumlib ${OPUS_LIBRARIES}) +target_link_libraries(mumlib + ${SPEEX_LIBRARIES} + ${PROTOBUF_LIBRARIES} + ${Boost_LIBRARIES} + ${OPENSSL_LIBRARIES} + ${LOG4CPP_LIBRARIES} + ${OPUS_LIBRARIES}) +# add_executable(mumlib_example mumlib_example.cpp) +# target_link_libraries(mumlib_example mumlib) -add_executable(mumlib_example mumlib_example.cpp) -target_link_libraries(mumlib_example mumlib) +install(TARGETS mumlib LIBRARY DESTINATION lib) diff --git a/Mumble.proto b/Mumble.proto index 92b8737..a74f247 100644 --- a/Mumble.proto +++ b/Mumble.proto @@ -1,3 +1,10 @@ +// Copyright 2005-2018 The Mumble Developers. All rights reserved. +// Use of this source code is governed by a BSD-style license +// that can be found in the LICENSE file at the root of the +// Mumble source tree or at . + +syntax = "proto2"; + package MumbleProto; option optimize_for = SPEED; @@ -63,7 +70,8 @@ message Ping { // Sent by the server when it rejects the user connection. message Reject { enum RejectType { - // TODO ?? + // The rejection reason is unknown (details should be available + // in Reject.reason). None = 0; // The client attempted to connect with an incompatible version. WrongVersion = 1; @@ -98,7 +106,7 @@ message ServerSync { optional uint32 max_bandwidth = 2; // Server welcome text. optional string welcome_text = 3; - // Current user permissions TODO: Confirm?? + // Current user permissions in the root channel. optional uint64 permissions = 4; } @@ -134,6 +142,10 @@ message ChannelState { optional int32 position = 9 [default = 0]; // SHA1 hash of the description if the description is 128 bytes or more. optional bytes description_hash = 10; + // Maximum number of users allowed in the channel. If this value is zero, + // the maximum number of users allowed in the channel is given by the + // server's "usersperchannel" setting. + optional uint32 max_users = 11; } // Used to communicate user leaving or being kicked. May be sent by the client @@ -180,9 +192,14 @@ message UserState { optional bool self_deaf = 10; // User image if it is less than 128 bytes. optional bytes texture = 11; - // TODO ?? + // The positional audio plugin identifier. + // Positional audio information is only sent to users who share + // identical plugin contexts. + // + // This value is not trasmitted to clients. optional bytes plugin_context = 12; - // TODO ?? + // The user's plugin-specific identity. + // This value is not transmitted to clients. optional string plugin_identity = 13; // User comment if it is less than 128 bytes. optional string comment = 14; @@ -209,7 +226,7 @@ message BanList { required uint32 mask = 2; // User name for identification purposes (does not affect the ban). optional string name = 3; - // TODO ?? + // The certificate hash of the banned user. optional string hash = 4; // Reason for the ban (does not affect the ban). optional string reason = 5; @@ -403,9 +420,9 @@ message VoiceTarget { message Target { // Users that are included as targets. repeated uint32 session = 1; - // Channels that are included as targets. + // Channel that is included as a target. optional uint32 channel_id = 2; - // TODO ?? + // ACL group that is included as a target. optional string group = 3; // True if the voice should follow links from the specified channel. optional bool links = 4 [default = false]; @@ -529,6 +546,8 @@ message ServerConfig { optional uint32 message_length = 4; // Maximum image message length. optional uint32 image_message_length = 5; + // The maximum number of users allowed on the server. + optional uint32 max_users = 6; } // Sent by the server to inform the clients of suggested client configuration @@ -541,4 +560,4 @@ message SuggestConfig { optional bool positional = 2; // True if the administrator suggests push to talk to be used on this server. optional bool push_to_talk = 3; -} +} \ No newline at end of file diff --git a/include/mumlib.hpp b/include/mumlib.hpp index 95afe02..08297d1 100644 --- a/include/mumlib.hpp +++ b/include/mumlib.hpp @@ -11,6 +11,8 @@ namespace mumlib { constexpr int DEFAULT_OPUS_ENCODER_BITRATE = 16000; + constexpr int DEFAULT_OPUS_SAMPLE_RATE = 48000; + constexpr int DEFAULT_OPUS_NUM_CHANNELS = 1; using namespace std; using namespace boost::asio; @@ -22,15 +24,28 @@ namespace mumlib { struct MumlibConfiguration { int opusEncoderBitrate = DEFAULT_OPUS_ENCODER_BITRATE; + int opusSampleRate = DEFAULT_OPUS_SAMPLE_RATE; + int opusChannels = DEFAULT_OPUS_NUM_CHANNELS; // additional fields will be added in the future }; + struct MumbleUser { + int32_t sessionId; + string name; + }; + + struct MumbleChannel { + int32_t channelId; + string name; + string description; + }; + struct _Mumlib_Private; class Mumlib : boost::noncopyable { public: - Mumlib(Callback &callback); + explicit Mumlib(Callback &callback); Mumlib(Callback &callback, io_service &ioService); @@ -48,13 +63,39 @@ namespace mumlib { ConnectionState getConnectionState(); + int getChannelId(); + + vector getListAllUser(); + + vector getListAllChannel(); + void sendAudioData(int16_t *pcmData, int pcmLength); + void sendAudioDataTarget(int targetId, int16_t *pcmData, int pcmLength); + void sendTextMessage(std::string message); void joinChannel(int channelId); + void joinChannel(std::string channelName); + + void sendVoiceTarget(int targetId, mumlib::VoiceTargetType type, int sessionId); + + void sendVoiceTarget(int targetId, mumlib::VoiceTargetType type, std::string name, int &error); + + void sendUserState(mumlib::UserState state, bool val); + + void sendUserState(mumlib::UserState state, std::string value); + private: _Mumlib_Private *impl; + + int getChannelIdBy(std::string channelName); + + int getUserIdBy(std::string userName); + + bool isSessionIdValid(int sessionId); + + bool isChannelIdValid(int channelId); }; } diff --git a/include/mumlib/Audio.hpp b/include/mumlib/Audio.hpp index d98f41a..862eeef 100644 --- a/include/mumlib/Audio.hpp +++ b/include/mumlib/Audio.hpp @@ -2,19 +2,19 @@ #include "Transport.hpp" -#include +#include + +#include #include namespace mumlib { - constexpr int SAMPLE_RATE = 48000; - class MumlibException; class AudioException : public MumlibException { public: - AudioException(string message) : MumlibException(message) { } + explicit AudioException(string message) : MumlibException(message) { } }; struct IncomingAudioPacket { @@ -28,17 +28,30 @@ namespace mumlib { class Audio : boost::noncopyable { public: - Audio(int opusEncoderBitrate = DEFAULT_OPUS_ENCODER_BITRATE); + explicit Audio(int sampleRate=DEFAULT_OPUS_SAMPLE_RATE, + int bitrate=DEFAULT_OPUS_ENCODER_BITRATE, + int channels=DEFAULT_OPUS_NUM_CHANNELS); virtual ~Audio(); IncomingAudioPacket decodeIncomingAudioPacket(uint8_t *inputBuffer, int inputBufferLength); + void addFrameToBuffer(uint8_t *inputBuffer, int inputLength, int sequence); + + // todo: mix audio + void mixAudio(uint8_t *dest, uint8_t *src, int bufferOffset, int inputLength); + + void resizeBuffer(); + + std::pair decodeOpusPayload(int16_t *pcmBuffer, + int pcmBufferSize); + std::pair decodeOpusPayload(uint8_t *inputBuffer, int inputLength, int16_t *pcmBuffer, int pcmBufferSize); + int encodeAudioPacket( int target, int16_t *inputPcmBuffer, @@ -52,14 +65,25 @@ namespace mumlib { void resetEncoder(); + void resetJitterBuffer(); + private: log4cpp::Category &logger; OpusDecoder *opusDecoder; OpusEncoder *opusEncoder; + JitterBuffer *jbBuffer; int64_t outgoingSequenceNumber; + unsigned int iSampleRate; + unsigned int iChannels; + unsigned int iFrameSize; + unsigned int iAudioBufferSize; + + float *fFadeIn; + float *fFadeOut; + std::chrono::time_point lastEncodedAudioPacketTimestamp; }; } \ No newline at end of file diff --git a/include/mumlib/Callback.hpp b/include/mumlib/Callback.hpp index a11b7b4..3e3ac4c 100644 --- a/include/mumlib/Callback.hpp +++ b/include/mumlib/Callback.hpp @@ -153,7 +153,7 @@ namespace mumlib { ~BasicCallback(); - virtual void version( + void version( uint16_t major, uint8_t minor, uint8_t patch, @@ -161,29 +161,29 @@ namespace mumlib { string os, string os_version) override; - virtual void audio( + void audio( int target, int sessionId, int sequenceNumber, int16_t *pcm_data, uint32_t pcm_data_size) override; - virtual void unsupportedAudio( + void unsupportedAudio( int target, int sessionId, int sequenceNumber, uint8_t *encoded_audio_data, uint32_t encoded_audio_data_size) override; - virtual void serverSync( + void serverSync( string welcome_text, int32_t session, int32_t max_bandwidth, int64_t permissions) override; - virtual void channelRemove(uint32_t channel_id) override; + void channelRemove(uint32_t channel_id) override; - virtual void channelState( + void channelState( string name, int32_t channel_id, int32_t parent, @@ -194,13 +194,13 @@ namespace mumlib { bool temporary, int32_t position) override; - virtual void userRemove( + void userRemove( uint32_t session, int32_t actor, string reason, bool ban) override; - virtual void userState( + void userState( int32_t session, int32_t actor, string name, @@ -215,7 +215,7 @@ namespace mumlib { int32_t priority_speaker, int32_t recording) override; - virtual void banList( + void banList( const uint8_t *ip_data, uint32_t ip_data_size, uint32_t mask, @@ -225,14 +225,14 @@ namespace mumlib { string start, int32_t duration) override; - virtual void textMessage( + void textMessage( uint32_t actor, std::vector session, std::vector channel_id, std::vector tree_id, string message) override; - virtual void permissionDenied( + void permissionDenied( int32_t permission, int32_t channel_id, int32_t session, @@ -240,48 +240,48 @@ namespace mumlib { int32_t deny_type, string name) override; - virtual void queryUsers( + void queryUsers( uint32_t n_ids, uint32_t *ids, uint32_t n_names, string *names) override; - virtual void contextActionModify( + void contextActionModify( string action, string text, uint32_t m_context, uint32_t operation) override; - virtual void contextAction( + void contextAction( int32_t session, int32_t channel_id, string action) override; - virtual void userList( + void userList( uint32_t user_id, string name, string last_seen, int32_t last_channel) override; - virtual void permissionQuery( + void permissionQuery( int32_t channel_id, uint32_t permissions, int32_t flush) override; - virtual void codecVersion( + void codecVersion( int32_t alpha, int32_t beta, uint32_t prefer_alpha, int32_t opus) override; - virtual void serverConfig( + void serverConfig( uint32_t max_bandwidth, string welcome_text, uint32_t allow_html, uint32_t message_length, uint32_t image_message_length) override; - virtual void suggestConfig( + void suggestConfig( uint32_t version, uint32_t positional, uint32_t push_to_talk) override; diff --git a/include/mumlib/CryptState.hpp b/include/mumlib/CryptState.hpp index cf5b61f..5d53c73 100644 --- a/include/mumlib/CryptState.hpp +++ b/include/mumlib/CryptState.hpp @@ -36,7 +36,7 @@ namespace mumlib { class CryptState : boost::noncopyable { - public: + private: unsigned char raw_key[AES_BLOCK_SIZE]; unsigned char encrypt_iv[AES_BLOCK_SIZE]; unsigned char decrypt_iv[AES_BLOCK_SIZE]; @@ -56,14 +56,19 @@ namespace mumlib { AES_KEY decrypt_key; bool bInit; + public: CryptState(); bool isValid() const; + void genKey(); + void setKey(const unsigned char *rkey, const unsigned char *eiv, const unsigned char *div); void setDecryptIV(const unsigned char *iv); + const unsigned char* getEncryptIV() const; + void ocb_encrypt(const unsigned char *plain, unsigned char *encrypted, unsigned int len, const unsigned char *nonce, unsigned char *tag); diff --git a/include/mumlib/Transport.hpp b/include/mumlib/Transport.hpp index ccf94a7..346728b 100644 --- a/include/mumlib/Transport.hpp +++ b/include/mumlib/Transport.hpp @@ -14,6 +14,7 @@ #include #include +#include namespace mumlib { @@ -30,7 +31,7 @@ namespace mumlib { class TransportException : public MumlibException { public: - TransportException(string message) : MumlibException(message) { } + TransportException(string message) : MumlibException(std::move(message)) { } }; class Transport : boost::noncopyable { diff --git a/include/mumlib/VarInt.hpp b/include/mumlib/VarInt.hpp index 451c61e..7e301c7 100644 --- a/include/mumlib/VarInt.hpp +++ b/include/mumlib/VarInt.hpp @@ -29,6 +29,6 @@ namespace mumlib { private: const int64_t value; - int64_t parseVariant(uint8_t *buffer); + int64_t parseVariant(const uint8_t *buffer); }; } \ No newline at end of file diff --git a/include/mumlib/enums.hpp b/include/mumlib/enums.hpp index b4a9b43..740d83b 100644 --- a/include/mumlib/enums.hpp +++ b/include/mumlib/enums.hpp @@ -45,4 +45,19 @@ namespace mumlib { OPUS }; + enum class UserState { + MUTE, + DEAF, + SUPPRESS, + SELF_MUTE, + SELF_DEAF, + COMMENT, + PRIORITY_SPEAKER, + RECORDING + }; + + enum class VoiceTargetType { + CHANNEL, + USER + }; } \ No newline at end of file diff --git a/src/Audio.cpp b/src/Audio.cpp index 3ce7594..393b674 100644 --- a/src/Audio.cpp +++ b/src/Audio.cpp @@ -4,29 +4,69 @@ static boost::posix_time::seconds RESET_SEQUENCE_NUMBER_INTERVAL(5); -mumlib::Audio::Audio(int opusEncoderBitrate) +mumlib::Audio::Audio(int sampleRate, int bitrate, int channels) : logger(log4cpp::Category::getInstance("mumlib.Audio")), opusDecoder(nullptr), opusEncoder(nullptr), - outgoingSequenceNumber(0) { + outgoingSequenceNumber(0), + iSampleRate(sampleRate), + iChannels(channels) { - int error; + int error, ret; + iFrameSize = sampleRate / 100; + iAudioBufferSize = iFrameSize; + iAudioBufferSize *= 12; - opusDecoder = opus_decoder_create(SAMPLE_RATE, 1, &error); + opusDecoder = opus_decoder_create(sampleRate, channels, &error); if (error != OPUS_OK) { throw AudioException((boost::format("failed to initialize OPUS decoder: %s") % opus_strerror(error)).str()); } - opusEncoder = opus_encoder_create(SAMPLE_RATE, 1, OPUS_APPLICATION_VOIP, &error); + opusEncoder = opus_encoder_create(sampleRate, channels, OPUS_APPLICATION_VOIP, &error); if (error != OPUS_OK) { throw AudioException((boost::format("failed to initialize OPUS encoder: %s") % opus_strerror(error)).str()); } - opus_encoder_ctl(opusEncoder, OPUS_SET_VBR(0)); - - setOpusEncoderBitrate(opusEncoderBitrate); + ret = opus_encoder_ctl(opusEncoder, OPUS_SET_BITRATE(bitrate)); + if (ret != OPUS_OK) { + throw AudioException((boost::format("failed to initialize transmission bitrate to %d B/s: %s") + % bitrate % opus_strerror(ret)).str()); + } + ret = opus_encoder_ctl(opusEncoder, OPUS_SET_VBR(0)); + if (ret != OPUS_OK) { + throw AudioException((boost::format("failed to initialize variable bitrate: %s") + % opus_strerror(ret)).str()); + } + ret = opus_encoder_ctl(opusEncoder, OPUS_SET_VBR_CONSTRAINT(0)); + if (ret != OPUS_OK) { + throw AudioException((boost::format("failed to initialize variable bitrate constraint: %s") + % opus_strerror(ret)).str()); + } + ret = opus_encoder_ctl(opusEncoder, OPUS_SET_BANDWIDTH(OPUS_BANDWIDTH_NARROWBAND)); + if (ret != OPUS_OK) { + throw AudioException((boost::format("failed to initialize bandwidth narrow: %s") + % opus_strerror(ret)).str()); + } + ret = opus_encoder_ctl(opusEncoder, OPUS_SET_MAX_BANDWIDTH(OPUS_BANDWIDTH_NARROWBAND)); + if (ret != OPUS_OK) { + throw AudioException((boost::format("failed to initialize maximum bandwidth narrow: %s") + % opus_strerror(ret)).str()); + } resetEncoder(); + + jbBuffer = jitter_buffer_init(iFrameSize); + int margin = 10 * iFrameSize; + jitter_buffer_ctl(jbBuffer, JITTER_BUFFER_SET_MARGIN, &margin); + + fFadeIn = new float[iFrameSize]; + fFadeOut = new float[iFrameSize]; + + // Sine function to represent fade in/out. Period is FRAME_SIZE. + float mul = static_cast(M_PI / 2.0 * static_cast(iFrameSize)); + for(unsigned int i = 0; i < iFrameSize; i++) { + fFadeIn[i] = fFadeOut[iFrameSize - 1 - 1] = sinf(static_cast(i) * mul); + } } mumlib::Audio::~Audio() { @@ -37,6 +77,11 @@ mumlib::Audio::~Audio() { if (opusEncoder) { opus_encoder_destroy(opusEncoder); } + + jitter_buffer_destroy(jbBuffer); + + delete[] fFadeIn; + delete[] fFadeOut; } void mumlib::Audio::setOpusEncoderBitrate(int bitrate) { @@ -56,6 +101,105 @@ int mumlib::Audio::getOpusEncoderBitrate() { return bitrate; } +void mumlib::Audio::addFrameToBuffer(uint8_t *inputBuffer, int inputLength, int sequence) { + int dataPointer = 0; + VarInt varInt(inputBuffer); + int opusDataLength = varInt.getValue(); + dataPointer += varInt.getEncoded().size(); + bool lastPacket = (opusDataLength & 0x2000) != 0; + opusDataLength &= 0x1fff; + + auto *packet = reinterpret_cast(&inputBuffer[dataPointer]); + int frame = opus_packet_get_nb_frames(packet, opusDataLength); + int samples = frame * opus_packet_get_samples_per_frame(packet, iSampleRate); + int channel = opus_packet_get_nb_channels(packet); + + if(not sequence) { + resetJitterBuffer(); + } + + logger.info("Opus packet, frame: %d, samples: %d, channel: %d", frame, samples, channel); + + JitterBufferPacket jbPacket; + jbPacket.data = reinterpret_cast(&inputBuffer[dataPointer]); + jbPacket.len = opusDataLength; + jbPacket.span = samples; + jbPacket.timestamp = iFrameSize * sequence; + jbPacket.user_data = lastPacket; + + jitter_buffer_put(jbBuffer, &jbPacket); +} + +std::pair mumlib::Audio::decodeOpusPayload(int16_t *pcmBuffer, int pcmBufferSize) { + int avail = 0; + spx_uint32_t remaining = 0; + jitter_buffer_ctl(jbBuffer, JITTER_BUFFER_GET_AVAILABLE_COUNT, &avail); + jitter_buffer_remaining_span(jbBuffer, remaining); + int timestamp = jitter_buffer_get_pointer_timestamp(jbBuffer); + + logger.warn("jbBufer, avail: %d, remain: %d, timestamp: %d", avail, remaining, timestamp); + + char data[4096]; + JitterBufferPacket jbPacket; + jbPacket.data = data; + jbPacket.len = 4096; + + spx_int32_t startofs = 0; + int opusDataLength; + int outputSize; + spx_uint32_t lastPacket; + + if(jitter_buffer_get(jbBuffer, &jbPacket, iFrameSize, &startofs) == JITTER_BUFFER_OK) { + opusDataLength = jbPacket.len; + lastPacket = jbPacket.user_data; + } else { + jitter_buffer_update_delay(jbBuffer, &jbPacket, NULL); + } + + if(opusDataLength) { + outputSize = opus_decode(opusDecoder, + reinterpret_cast(jbPacket.data), + jbPacket.len, + pcmBuffer, + pcmBufferSize, 0); + } else { + outputSize = opus_decode(opusDecoder, + NULL, 0, pcmBuffer, pcmBufferSize, 0); + } + + if(outputSize < 0) { + outputSize = iFrameSize; + memset(pcmBuffer, 0, iFrameSize * sizeof(float)); + } + + if(lastPacket) { + for(unsigned int i = 0; i < iFrameSize; i++) + pcmBuffer[i] *= fFadeOut[i]; + } + + for (int i = outputSize / iFrameSize; i > 0; --i) { + jitter_buffer_tick(jbBuffer); + } + + logger.debug("%d B of Opus data decoded to %d PCM samples, last packet: %d.", + opusDataLength, outputSize, lastPacket); + + return std::make_pair(outputSize, lastPacket); +} + +void mumlib::Audio::mixAudio(uint8_t *dest, uint8_t *src, int bufferOffset, int inputLength) { + for(int i = 0; i < inputLength; i++) { + float mix = 0; + + // Clip to [-1,1] + if(mix > 1) + mix = 1; + else if(mix < -1) + mix = -1; + dest[i + bufferOffset] = mix; + } +} + std::pair mumlib::Audio::decodeOpusPayload(uint8_t *inputBuffer, int inputLength, int16_t *pcmBuffer, @@ -75,8 +219,18 @@ std::pair mumlib::Audio::decodeOpusPayload(uint8_t *inputBuffer, % inputLength % dataPointer % opusDataLength).str()); } + // Issue #3 (Users speaking simultaneously) + // https://mf4.xiph.org/jenkins/view/opus/job/opus/ws/doc/html/group__opus__decoder.html + // Opus is a stateful codec with overlapping blocks and as a result Opus packets are not coded independently of each other. + // Packets must be passed into the decoder serially and in the correct order for a correct decode. + // Lost packets can be replaced with loss concealment by calling the decoder with a null pointer and zero length for the missing packet. + // A single codec state may only be accessed from a single thread at a time and any required locking must be performed by the caller. + // Separate streams must be decoded with separate decoder states and can be decoded in parallel unless the library was compiled with NONTHREADSAFE_PSEUDOSTACK defined. + auto *packet = reinterpret_cast(&inputBuffer[dataPointer]); + int frame = opus_packet_get_nb_frames(packet, opusDataLength); + int samples = frame * opus_packet_get_samples_per_frame(packet, iSampleRate); int outputSize = opus_decode(opusDecoder, - reinterpret_cast(&inputBuffer[dataPointer]), + packet, opusDataLength, pcmBuffer, pcmBufferSize, @@ -108,7 +262,7 @@ int mumlib::Audio::encodeAudioPacket(int target, int16_t *inputPcmBuffer, int in std::vector header; - header.push_back(0x80 | target); + header.push_back(static_cast(0x80 | target)); auto sequenceNumberEnc = VarInt(outgoingSequenceNumber).getEncoded(); header.insert(header.end(), sequenceNumberEnc.begin(), sequenceNumberEnc.end()); @@ -130,15 +284,15 @@ int mumlib::Audio::encodeAudioPacket(int target, int16_t *inputPcmBuffer, int in header.insert(header.end(), outputSizeEnc.begin(), outputSizeEnc.end()); memcpy(outputBuffer, &header[0], header.size()); - memcpy(outputBuffer + header.size(), tmpOpusBuffer, outputSize); + memcpy(outputBuffer + header.size(), tmpOpusBuffer, (size_t) outputSize); - int incrementNumber = 100 * inputLength / SAMPLE_RATE; + int incrementNumber = 100 * inputLength / iSampleRate; outgoingSequenceNumber += incrementNumber; lastEncodedAudioPacketTimestamp = std::chrono::system_clock::now(); - return outputSize + header.size(); + return static_cast(outputSize + header.size()); } void mumlib::Audio::resetEncoder() { @@ -151,8 +305,13 @@ void mumlib::Audio::resetEncoder() { outgoingSequenceNumber = 0; } +void mumlib::Audio::resetJitterBuffer() { + logger.debug("Last audio packet, resetting jitter buffer"); + jitter_buffer_reset(jbBuffer); +} + mumlib::IncomingAudioPacket mumlib::Audio::decodeIncomingAudioPacket(uint8_t *inputBuffer, int inputBufferLength) { - mumlib::IncomingAudioPacket incomingAudioPacket; + mumlib::IncomingAudioPacket incomingAudioPacket{}; incomingAudioPacket.type = static_cast((inputBuffer[0] & 0xE0) >> 5); incomingAudioPacket.target = inputBuffer[0] & 0x1F; diff --git a/src/Callback.cpp b/src/Callback.cpp index ea8fb80..5bc69e3 100644 --- a/src/Callback.cpp +++ b/src/Callback.cpp @@ -35,7 +35,7 @@ void mumlib::BasicCallback::version( os_version.c_str()); } -void mumlib::BasicCallback::audio( +void BasicCallback::audio( int target, int sessionId, int sequenceNumber, diff --git a/src/CryptState.cpp b/src/CryptState.cpp index 9c5937a..e1763c3 100644 --- a/src/CryptState.cpp +++ b/src/CryptState.cpp @@ -40,6 +40,7 @@ #include "mumlib/CryptState.hpp" +#include #include #include @@ -57,6 +58,15 @@ bool mumlib::CryptState::isValid() const { return bInit; } +void mumlib::CryptState::genKey() { + RAND_bytes(raw_key, AES_BLOCK_SIZE); + RAND_bytes(encrypt_iv, AES_BLOCK_SIZE); + RAND_bytes(decrypt_iv, AES_BLOCK_SIZE); + AES_set_encrypt_key(raw_key, 128, &encrypt_key); + AES_set_decrypt_key(raw_key, 128, &decrypt_key); + bInit = true; +} + void mumlib::CryptState::setKey(const unsigned char *rkey, const unsigned char *eiv, const unsigned char *div) { memcpy(raw_key, rkey, AES_BLOCK_SIZE); memcpy(encrypt_iv, eiv, AES_BLOCK_SIZE); @@ -70,6 +80,10 @@ void mumlib::CryptState::setDecryptIV(const unsigned char *iv) { memcpy(decrypt_iv, iv, AES_BLOCK_SIZE); } +const unsigned char* mumlib::CryptState::getEncryptIV() const { + return encrypt_iv; +} + void mumlib::CryptState::encrypt(const unsigned char *source, unsigned char *dst, unsigned int plain_length) { unsigned char tag[AES_BLOCK_SIZE]; diff --git a/src/Transport.cpp b/src/Transport.cpp index f134512..83a383a 100644 --- a/src/Transport.cpp +++ b/src/Transport.cpp @@ -32,15 +32,15 @@ mumlib::Transport::Transport( bool noUdp) : logger(log4cpp::Category::getInstance("mumlib.Transport")), ioService(ioService), - processMessageFunction(processMessageFunc), - processEncodedAudioPacketFunction(processEncodedAudioPacketFunction), + processMessageFunction(std::move(processMessageFunc)), + processEncodedAudioPacketFunction(std::move(processEncodedAudioPacketFunction)), noUdp(noUdp), state(ConnectionState::NOT_CONNECTED), udpSocket(ioService), sslContext(ssl::context::sslv23), sslSocket(ioService, sslContext), pingTimer(ioService, PING_INTERVAL), - asyncBufferPool(max(MAX_UDP_LENGTH, MAX_TCP_LENGTH)) { + asyncBufferPool(static_cast(max(MAX_UDP_LENGTH, MAX_TCP_LENGTH))) { sslIncomingBuffer = new uint8_t[MAX_TCP_LENGTH]; @@ -81,7 +81,7 @@ void mumlib::Transport::connect( doReceiveUdp(); } - + ip::tcp::resolver resolverTcp(ioService); ip::tcp::resolver::query queryTcp(host, to_string(port)); @@ -118,7 +118,6 @@ void mumlib::Transport::disconnect() { } } - void mumlib::Transport::sendVersion() { MumbleProto::Version version; @@ -185,10 +184,10 @@ void mumlib::Transport::doReceiveUdp() { } uint8_t plainBuffer[1024]; - const int plainBufferLength = bytesTransferred - 4; + const int plainBufferLength = static_cast(bytesTransferred - 4); bool success = cryptState.decrypt( - udpIncomingBuffer, plainBuffer, bytesTransferred); + udpIncomingBuffer, plainBuffer, static_cast(bytesTransferred)); if (not success) { throwTransportException("UDP packet decryption failed"); @@ -263,12 +262,12 @@ void mumlib::Transport::sendUdpAsync(uint8_t *buff, int length) { auto *encryptedMsgBuff = asyncBufferPool.malloc(); const int encryptedMsgLength = length + 4; - cryptState.encrypt(buff, reinterpret_cast(encryptedMsgBuff), length); + cryptState.encrypt(buff, reinterpret_cast(encryptedMsgBuff), static_cast(length)); logger.debug("Sending %d B of data UDP asynchronously.", encryptedMsgLength); udpSocket.async_send_to( - boost::asio::buffer(encryptedMsgBuff, length + 4), + boost::asio::buffer(encryptedMsgBuff, static_cast(length + 4)), udpReceiverEndpoint, [this, encryptedMsgBuff](const boost::system::error_code &ec, size_t bytesTransferred) { asyncBufferPool.free(encryptedMsgBuff); @@ -318,7 +317,7 @@ void mumlib::Transport::doReceiveSsl() { processMessageInternal( static_cast(messageType), &sslIncomingBuffer[6], - bytesTransferred - 6); + static_cast(bytesTransferred - 6)); doReceiveSsl(); } else { @@ -438,10 +437,10 @@ void mumlib::Transport::sendUdpPing() { vector message; message.push_back(0x20); - auto timestampVarint = VarInt(time(nullptr)).getEncoded(); + auto timestampVarint = VarInt(static_cast(time(nullptr))).getEncoded(); message.insert(message.end(), timestampVarint.begin(), timestampVarint.end()); - sendUdpAsync(&message[0], message.size()); + sendUdpAsync(&message[0], static_cast(message.size())); } void mumlib::Transport::sendSsl(uint8_t *buff, int length) { @@ -453,7 +452,7 @@ void mumlib::Transport::sendSsl(uint8_t *buff, int length) { logger.debug("Sending %d bytes of data.", length); try { - write(sslSocket, boost::asio::buffer(buff, length)); + write(sslSocket, boost::asio::buffer(buff, static_cast(length))); } catch (boost::system::system_error &err) { throwTransportException(std::string("SSL send failed: ") + err.what()); } @@ -467,13 +466,13 @@ void mumlib::Transport::sendSslAsync(uint8_t *buff, int length) { auto *asyncBuff = asyncBufferPool.malloc(); - memcpy(asyncBuff, buff, length); + memcpy(asyncBuff, buff, static_cast(length)); logger.debug("Sending %d B of data asynchronously.", length); async_write( sslSocket, - boost::asio::buffer(asyncBuff, length), + boost::asio::buffer(asyncBuff, static_cast(length)), [this, asyncBuff](const boost::system::error_code &ec, size_t bytesTransferred) { asyncBufferPool.free(asyncBuff); logger.debug("Sent %d B.", bytesTransferred); @@ -499,7 +498,7 @@ void mumlib::Transport::sendControlMessagePrivate(MessageType type, google::prot const uint16_t type_network = htons(static_cast(type)); const int size = message.ByteSize(); - const uint32_t size_network = htonl(size); + const uint32_t size_network = htonl((uint32_t) size); const int length = sizeof(type_network) + sizeof(size_network) + size; @@ -517,7 +516,7 @@ void mumlib::Transport::sendControlMessagePrivate(MessageType type, google::prot void mumlib::Transport::throwTransportException(string message) { state = ConnectionState::FAILED; - throw TransportException(message); + throw TransportException(std::move(message)); } void mumlib::Transport::sendEncodedAudioPacket(uint8_t *buffer, int length) { @@ -534,7 +533,7 @@ void mumlib::Transport::sendEncodedAudioPacket(uint8_t *buffer, int length) { const uint16_t netUdptunnelType = htons(static_cast(MessageType::UDPTUNNEL)); - const uint32_t netLength = htonl(length); + const uint32_t netLength = htonl(static_cast(length)); const int packet = sizeof(netUdptunnelType) + sizeof(netLength) + length; @@ -542,14 +541,14 @@ void mumlib::Transport::sendEncodedAudioPacket(uint8_t *buffer, int length) { memcpy(packetBuff, &netUdptunnelType, sizeof(netUdptunnelType)); memcpy(packetBuff + sizeof(netUdptunnelType), &netLength, sizeof(netLength)); - memcpy(packetBuff + sizeof(netUdptunnelType) + sizeof(netLength), buffer, length); + memcpy(packetBuff + sizeof(netUdptunnelType) + sizeof(netLength), buffer, static_cast(length)); sendSslAsync(packetBuff, length + sizeof(netUdptunnelType) + sizeof(netLength)); } } void mumlib::Transport::processAudioPacket(uint8_t *buff, int length) { - AudioPacketType type = static_cast((buff[0] & 0xE0) >> 5); + auto type = static_cast((buff[0] & 0xE0) >> 5); switch (type) { case AudioPacketType::CELT_Alpha: case AudioPacketType::Speex: diff --git a/src/VarInt.cpp b/src/VarInt.cpp index 2eeebf8..bd489da 100644 --- a/src/VarInt.cpp +++ b/src/VarInt.cpp @@ -12,7 +12,7 @@ mumlib::VarInt::VarInt(std::vector encoded) : value(parseVariant(&encod * This code was taken from Mumble source code * https://github.com/mumble-voip/mumble/blob/master/src/PacketDataStream.h */ -int64_t mumlib::VarInt::parseVariant(uint8_t *buffer) { +int64_t mumlib::VarInt::parseVariant(const uint8_t *buffer) { int64_t v = buffer[0]; if ((v & 0x80) == 0x00) { return (v & 0x7F); @@ -36,6 +36,7 @@ int64_t mumlib::VarInt::parseVariant(uint8_t *buffer) { return (v & 0x1F) << 16 | buffer[1] << 8 | buffer[2]; } + throw VarIntException("invalid varint"); } @@ -46,7 +47,7 @@ std::vector mumlib::VarInt::getEncoded() const { if ((i & 0x8000000000000000LL) && (~i < 0x100000000LL)) { i = ~i; if (i <= 0x3) { - encoded.push_back(0xFC | i); + encoded.push_back(static_cast(0xFC | i)); return encoded; } else { encoded.push_back(0xF8); @@ -54,29 +55,29 @@ std::vector mumlib::VarInt::getEncoded() const { } if (i < 0x80) { - encoded.push_back(i); + encoded.push_back(static_cast(i)); } else if (i < 0x4000) { - encoded.push_back(0x80 | (i >> 8)); - encoded.push_back(i & 0xFF); + encoded.push_back(static_cast(0x80 | (i >> 8))); + encoded.push_back(static_cast(i & 0xFF)); } else if (i < 0x200000) { - encoded.push_back(0xC0 | (i >> 16)); - encoded.push_back((i >> 8) & 0xFF); - encoded.push_back(i & 0xFF); + encoded.push_back(static_cast(0xC0 | (i >> 16))); + encoded.push_back(static_cast((i >> 8) & 0xFF)); + encoded.push_back(static_cast(i & 0xFF)); } else if (i < 0x10000000) { - encoded.push_back(0xE0 | (i >> 24)); - encoded.push_back((i >> 16) & 0xFF); - encoded.push_back((i >> 8) & 0xFF); - encoded.push_back(i & 0xFF); + encoded.push_back(static_cast(0xE0 | (i >> 24))); + encoded.push_back(static_cast((i >> 16) & 0xFF)); + encoded.push_back(static_cast((i >> 8) & 0xFF)); + encoded.push_back(static_cast(i & 0xFF)); } else { encoded.push_back(0xF4); - encoded.push_back((i >> 56) & 0xFF); - encoded.push_back((i >> 48) & 0xFF); - encoded.push_back((i >> 40) & 0xFF); - encoded.push_back((i >> 32) & 0xFF); - encoded.push_back((i >> 24) & 0xFF); - encoded.push_back((i >> 16) & 0xFF); - encoded.push_back((i >> 8) & 0xFF); - encoded.push_back(i & 0xFF); + encoded.push_back(static_cast((i >> 56) & 0xFF)); + encoded.push_back(static_cast((i >> 48) & 0xFF)); + encoded.push_back(static_cast((i >> 40) & 0xFF)); + encoded.push_back(static_cast((i >> 32) & 0xFF)); + encoded.push_back(static_cast((i >> 24) & 0xFF)); + encoded.push_back(static_cast((i >> 16) & 0xFF)); + encoded.push_back(static_cast((i >> 8) & 0xFF)); + encoded.push_back(static_cast(i & 0xFF)); } return encoded; diff --git a/src/mumlib.cpp b/src/mumlib.cpp index 08197d1..3c85cec 100644 --- a/src/mumlib.cpp +++ b/src/mumlib.cpp @@ -8,6 +8,7 @@ #include #include +#include #include #include @@ -32,6 +33,10 @@ namespace mumlib { int sessionId = 0; int channelId = 0; + int64_t seq = 0; + + std::vector listMumbleUser; + std::vector listMumbleChannel; _Mumlib_Private(Callback &callback, MumlibConfiguration &configuration) : _Mumlib_Private(callback, *(new io_service()), configuration) { @@ -43,7 +48,8 @@ namespace mumlib { ioService(ioService), externalIoService(true), transport(ioService, boost::bind(&_Mumlib_Private::processIncomingTcpMessage, this, _1, _2, _3), - boost::bind(&_Mumlib_Private::processAudioPacket, this, _1, _2, _3)) { + boost::bind(&_Mumlib_Private::processAudioPacket, this, _1, _2, _3)), + audio(configuration.opusSampleRate, configuration.opusEncoderBitrate, configuration.opusChannels) { audio.setOpusEncoderBitrate(configuration.opusEncoderBitrate); } @@ -61,16 +67,26 @@ namespace mumlib { if (type == AudioPacketType::OPUS) { int16_t pcmData[5000]; - auto status = audio.decodeOpusPayload(incomingAudioPacket.audioPayload, - incomingAudioPacket.audioPayloadLength, - pcmData, - 5000); + + audio.addFrameToBuffer(incomingAudioPacket.audioPayload, + incomingAudioPacket.audioPayloadLength, + seq); + + auto status = audio.decodeOpusPayload(pcmData, 5000); + + // auto status = audio.decodeOpusPayload(incomingAudioPacket.audioPayload, + // incomingAudioPacket.audioPayloadLength, + // pcmData, + // 5000); + + if(status.second) seq = 0; else seq++; callback.audio(incomingAudioPacket.target, - incomingAudioPacket.sessionId, - incomingAudioPacket.sequenceNumber, - pcmData, - status.first); + incomingAudioPacket.sessionId, + incomingAudioPacket.sequenceNumber, + pcmData, + status.first); + } else { logger.warn("Incoming audio packet doesn't contain Opus data, calling unsupportedAudio callback."); callback.unsupportedAudio(incomingAudioPacket.target, @@ -122,6 +138,11 @@ namespace mumlib { case MessageType::CHANNELREMOVE: { MumbleProto::ChannelRemove channelRemove; channelRemove.ParseFromArray(buffer, length); + + if(isListChannelContains(channelRemove.channel_id())) { + listChannelRemovedBy(channelRemove.channel_id()); + } + callback.channelRemove(channelRemove.channel_id()); } break; @@ -152,7 +173,14 @@ namespace mumlib { links_remove.push_back(channelState.links_remove(i)); } - this->channelId = channel_id; + MumbleChannel mumbleChannel; + mumbleChannel.channelId = channel_id; + mumbleChannel.name = channelState.name(); + mumbleChannel.description = channelState.description(); + + if(not isListChannelContains(channel_id)) { + listMumbleChannel.push_back(mumbleChannel); + } callback.channelState( channelState.name(), @@ -175,6 +203,10 @@ namespace mumlib { bool ban = user_remove.has_ban() ? user_remove.ban() : false; //todo make sure it's correct to assume it's false + if(isListUserContains(user_remove.session())) { + listUserRemovedBy(user_remove.session()); + } + callback.userRemove( user_remove.session(), actor, @@ -200,6 +232,18 @@ namespace mumlib { int32_t priority_speaker = userState.has_priority_speaker() ? userState.priority_speaker() : -1; int32_t recording = userState.has_recording() ? userState.recording() : -1; + if(session == this->sessionId) { + this->channelId = channel_id; + } + + MumbleUser mumbleUser; + mumbleUser.name = userState.name(); + mumbleUser.sessionId = session; + + if(not isListUserContains(session)) { + listMumbleUser.push_back(mumbleUser); + } + callback.userState(session, actor, userState.name(), @@ -334,7 +378,37 @@ namespace mumlib { return true; } + bool isListUserContains(int sessionId) { + for(int i = 0; i < listMumbleUser.size(); i++) + if(listMumbleUser[i].sessionId == sessionId) + return true; + return false; + } + void listUserRemovedBy(int sessionId) { + for(int i = 0; i < listMumbleUser.size(); i++) { + if(listMumbleUser[i].sessionId == sessionId) { + listMumbleUser.erase(listMumbleUser.begin() + i); + return; + } + } + } + + bool isListChannelContains(int channelId) { + for(int i = 0; i < listMumbleChannel.size(); i++) + if(listMumbleChannel[i].channelId == channelId) + return true; + return false; + } + + void listChannelRemovedBy(int channelId) { + for(int i = 0; i < listMumbleChannel.size(); i++) { + if(listMumbleChannel[i].channelId == channelId) { + listMumbleChannel.erase(listMumbleChannel.begin() + i); + return; + } + } + } }; Mumlib::Mumlib(Callback &callback) { @@ -363,13 +437,25 @@ namespace mumlib { return impl->transport.getConnectionState(); } + int Mumlib::getChannelId() { + return impl->channelId; + } + + vector Mumlib::getListAllUser() { + return impl->listMumbleUser; + } + + vector Mumlib::getListAllChannel() { + return impl->listMumbleChannel; + } + void Mumlib::connect(string host, int port, string user, string password) { impl->transport.connect(host, port, user, password); } void Mumlib::disconnect() { if (not impl->externalIoService) { - impl->ioService.reset(); + impl->ioService.stop(); } if (impl->transport.getConnectionState() != ConnectionState::NOT_CONNECTED) { impl->transport.disconnect(); @@ -385,8 +471,12 @@ namespace mumlib { } void Mumlib::sendAudioData(int16_t *pcmData, int pcmLength) { + Mumlib::sendAudioDataTarget(0, pcmData, pcmLength); + } + + void Mumlib::sendAudioDataTarget(int targetId, int16_t *pcmData, int pcmLength) { uint8_t encodedData[5000]; - int length = impl->audio.encodeAudioPacket(0, pcmData, pcmLength, encodedData, 5000); + int length = impl->audio.encodeAudioPacket(targetId, pcmData, pcmLength, encodedData, 5000); impl->transport.sendEncodedAudioPacket(encodedData, length); } @@ -399,9 +489,146 @@ namespace mumlib { } void Mumlib::joinChannel(int channelId) { + if(!isChannelIdValid(channelId)) // when channel has not been registered / create + return; MumbleProto::UserState userState; userState.set_channel_id(channelId); impl->transport.sendControlMessage(MessageType::USERSTATE, userState); impl->channelId = channelId; } + + void Mumlib::joinChannel(string name) { + int channelId = Mumlib::getChannelIdBy(name); + Mumlib::joinChannel(channelId); + } + + void Mumlib::sendVoiceTarget(int targetId, VoiceTargetType type, int id) { + MumbleProto::VoiceTarget voiceTarget; + MumbleProto::VoiceTarget_Target voiceTargetTarget; + switch(type) { + case VoiceTargetType::CHANNEL: { + voiceTargetTarget.set_channel_id(id); + voiceTargetTarget.set_children(true); + } + break; + case VoiceTargetType::USER: { + voiceTargetTarget.add_session(id); + } + break; + default: + return; + } + voiceTarget.set_id(targetId); + voiceTarget.add_targets()->CopyFrom(voiceTargetTarget); + impl->transport.sendControlMessage(MessageType::VOICETARGET, voiceTarget); + } + + void Mumlib::sendVoiceTarget(int targetId, VoiceTargetType type, string name, int &error) { + int id; + switch(type) { + case VoiceTargetType::CHANNEL: + id = getChannelIdBy(name); + break; + case VoiceTargetType::USER: + id = getUserIdBy(name); + break; + default: + break; + } + error = id < 0 ? 1: 0; + if(error) return; + sendVoiceTarget(targetId, type, id); + } + + void Mumlib::sendUserState(mumlib::UserState field, bool val) { + MumbleProto::UserState userState; + + switch (field) { + case UserState::MUTE: + userState.set_mute(val); + break; + case UserState::DEAF: + userState.set_deaf(val); + break; + case UserState::SUPPRESS: + userState.set_suppress(val); + break; + case UserState::SELF_MUTE: + userState.set_self_mute(val); + break; + case UserState::SELF_DEAF: + userState.set_self_deaf(val); + break; + case UserState::PRIORITY_SPEAKER: + userState.set_priority_speaker(val); + break; + case UserState::RECORDING: + userState.set_recording(val); + break; + default: + // in any other case, just ignore the command + return; + } + + impl->transport.sendControlMessage(MessageType::USERSTATE, userState); + } + + void Mumlib::sendUserState(mumlib::UserState field, std::string val) { + MumbleProto::UserState userState; + + // if comment longer than 128 bytes, we need to set the SHA1 hash + // http://www.askyb.com/cpp/openssl-sha1-hashing-example-in-cpp/ + unsigned char digest[SHA_DIGEST_LENGTH]; + char mdString[SHA_DIGEST_LENGTH * 2 + 1]; + + SHA1((unsigned char*) val.c_str(), val.size(), digest); + for(int i = 0; i < SHA_DIGEST_LENGTH; i++) + sprintf(&mdString[i*2], "%02x", (unsigned int) digest[i]); + + switch (field) { + case UserState::COMMENT: + if(val.size() < 128) + userState.set_comment(val); + else + userState.set_comment_hash(mdString); + break; + default: + // in any other case, just ignore the command + return; + } + + impl->transport.sendControlMessage(MessageType::USERSTATE, userState); + } + + int Mumlib::getChannelIdBy(string name) { + vector listMumbleChannel = impl->listMumbleChannel; + for(int i = 0; i < listMumbleChannel.size(); i++) + if(listMumbleChannel[i].name == name) + return listMumbleChannel[i].channelId; + return -1; + } + + int Mumlib::getUserIdBy(string name) { + vector listMumbleUser = impl->listMumbleUser; + for(int i = 0; i < listMumbleUser.size(); i++) + if(listMumbleUser[i].name == name) + return listMumbleUser[i].sessionId; + return -1; + } + + bool Mumlib::isSessionIdValid(int sessionId) { + vector listMumbleUser = impl->listMumbleUser; + for(int i = 0; i < listMumbleUser.size(); i++) + if(listMumbleUser[i].sessionId == sessionId) + return true; + return false; + } + + bool Mumlib::isChannelIdValid(int channelId) { + vector listMumbleChannel = impl->listMumbleChannel; + for(int i = 0; i < listMumbleChannel.size(); i++) + if(listMumbleChannel[i].channelId == channelId) + return true; + return false; + } }