From 2d88ee85cd295b268ece34c1dac8519544eaeaa1 Mon Sep 17 00:00:00 2001 From: badaix Date: Mon, 3 Aug 2020 22:52:42 +0200 Subject: [PATCH] Opus encoder resamples to 48000:16:2 --- CMakeLists.txt | 36 +++++-- client/CMakeLists.txt | 16 --- client/Makefile | 2 +- client/decoder/opus_decoder.cpp | 19 ++-- client/stream.cpp | 135 +++--------------------- client/stream.hpp | 10 +- common/CMakeLists.txt | 18 +++- common/message/pcm_chunk.hpp | 22 +++- common/resampler.cpp | 180 ++++++++++++++++++++++++++++++++ common/resampler.hpp | 50 +++++++++ server/Makefile | 4 +- server/encoder/opus_encoder.cpp | 42 ++++++-- server/encoder/opus_encoder.hpp | 7 +- 13 files changed, 367 insertions(+), 174 deletions(-) create mode 100644 common/resampler.cpp create mode 100644 common/resampler.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 2fdcc7d1..550de843 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -123,7 +123,6 @@ if(NOT WIN32) add_definitions(-DFREEBSD -DHAS_DAEMON) link_directories("/usr/local/lib") list(APPEND INCLUDE_DIRS "/usr/local/include") - list(APPEND CMAKE_REQUIRED_INCLUDES "${INCLUDE_DIRS}") elseif(ANDROID) # add_definitions("-DNO_CPP11_STRING") else() @@ -138,6 +137,8 @@ if(NOT WIN32) pkg_search_module(AVAHI avahi-client) if (AVAHI_FOUND) add_definitions(-DHAS_AVAHI) + else() + message(STATUS "avahi-client not found") endif (AVAHI_FOUND) endif(BUILD_WITH_AVAHI) @@ -147,15 +148,22 @@ if(NOT WIN32) add_definitions(-DFREEBSD) link_directories("/usr/local/lib") list(APPEND INCLUDE_DIRS "/usr/local/include") - list(APPEND CMAKE_REQUIRED_INCLUDES "${INCLUDE_DIRS}") endif() endif() + pkg_search_module(SOXR soxr) + if (SOXR_FOUND) + add_definitions("-DHAS_SOXR") + else() + message(STATUS "soxr not found") + endif (SOXR_FOUND) if(BUILD_WITH_FLAC) pkg_search_module(FLAC flac) if (FLAC_FOUND) add_definitions("-DHAS_FLAC") + else() + message(STATUS "flac not found") endif (FLAC_FOUND) endif() @@ -163,6 +171,8 @@ if(NOT WIN32) pkg_search_module(OGG ogg) if (OGG_FOUND) add_definitions("-DHAS_OGG") + else() + message(STATUS "ogg not found") endif (OGG_FOUND) endif() @@ -180,10 +190,16 @@ if(NOT WIN32) endif (TREMOR_FOUND) endif() + if ((BUILD_WITH_VORBIS OR BUILD_WITH_TREMOR) AND NOT VORBIS_FOUND AND NOT TREMOR_FOUND) + message(STATUS "tremor and vorbis not found") + endif() + if(BUILD_WITH_VORBIS) pkg_search_module(VORBISENC vorbisenc) if (VORBISENC_FOUND) add_definitions("-DHAS_VORBIS_ENC") + else() + message(STATUS "vorbisenc not found") endif(VORBISENC_FOUND) endif() @@ -191,6 +207,8 @@ if(NOT WIN32) pkg_search_module(OPUS opus) if (OPUS_FOUND) add_definitions("-DHAS_OPUS") + else() + message(STATUS "opus not found") endif (OPUS_FOUND) endif() @@ -211,24 +229,30 @@ if(WIN32) find_path(FLAC_INCLUDE_DIRS FLAC/all.h) find_library(FLAC_LIBRARIES FLAC) - find_package_handle_standard_args(FLAC DEFAULT_MSG FLAC_INCLUDE_DIRS FLAC_LIBRARIES) + find_package_handle_standard_args(FLAC REQUIRED FLAC_INCLUDE_DIRS FLAC_LIBRARIES) find_path(OGG_INCLUDE_DIRS ogg/ogg.h) find_library(OGG_LIBRARIES ogg) - find_package_handle_standard_args(Ogg DEFAULT_MSG OGG_INCLUDE_DIRS OGG_LIBRARIES) + find_package_handle_standard_args(Ogg REQUIRED OGG_INCLUDE_DIRS OGG_LIBRARIES) find_path(VORBIS_INCLUDE_DIRS vorbis/vorbisenc.h) find_library(VORBIS_LIBRARIES vorbis) - find_package_handle_standard_args(Vorbis DEFAULT_MSG VORBIS_INCLUDE_DIRS VORBIS_LIBRARIES) + find_package_handle_standard_args(Vorbis REQUIRED VORBIS_INCLUDE_DIRS VORBIS_LIBRARIES) find_path(OPUS_INCLUDE_DIRS opus/opus.h) find_library(OPUS_LIBRARIES opus) find_package_handle_standard_args(Opus REQUIRED OPUS_INCLUDE_DIRS OPUS_LIBRARIES) + find_path(SOXR_INCLUDE_DIRS soxr.h) + find_library(SOXR_LIBRARIES soxr) + find_package_handle_standard_args(Soxr REQUIRED SOXR_INCLUDE_DIRS SOXR_LIBRARIES) + add_definitions(-DNTDDI_VERSION=0x06020000 -D_WIN32_WINNT=0x0602 -DWINVER=0x0602 -DWINDOWS -DWIN32_LEAN_AND_MEAN -DUNICODE -D_UNICODE -D_CRT_SECURE_NO_WARNINGS ) - add_definitions(-DHAS_OGG -DHAS_VORBIS -DHAS_FLAC -DHAS_VORBIS_ENC -DHAS_OPUS -DHAS_WASAPI) + add_definitions(-DHAS_OGG -DHAS_VORBIS -DHAS_FLAC -DHAS_VORBIS_ENC -DHAS_OPUS -DHAS_WASAPI -DHAS_SOXR) endif() +list(APPEND CMAKE_REQUIRED_INCLUDES "${INCLUDE_DIRS}") + add_subdirectory(common) if (BUILD_SERVER) diff --git a/client/CMakeLists.txt b/client/CMakeLists.txt index e526324e..d5971801 100644 --- a/client/CMakeLists.txt +++ b/client/CMakeLists.txt @@ -48,22 +48,6 @@ else() endif (ALSA_FOUND) endif (MACOSX) -#pkg_search_module(SOXR soxr) -if(WIN32) - find_path(SOXR_INCLUDE_DIRS soxr.h) - find_library(SOXR_LIBRARIES soxr) - find_package_handle_standard_args(Soxr DEFAULT_MSG SOXR_INCLUDE_DIRS SOXR_LIBRARIES) - add_definitions("-DHAS_SOXR") -else() - find_package(soxr) -endif() - -if (SOXR_FOUND) - add_definitions("-DHAS_SOXR") - list(APPEND CLIENT_LIBRARIES ${SOXR_LIBRARIES}) - list(APPEND CLIENT_INCLUDE ${SOXR_INCLUDE_DIRS}) -endif (SOXR_FOUND) - # if OGG then tremor or vorbis if (OGG_FOUND) list(APPEND CLIENT_SOURCES decoder/ogg_decoder.cpp) diff --git a/client/Makefile b/client/Makefile index 16ed0a6f..09ab9bb2 100644 --- a/client/Makefile +++ b/client/Makefile @@ -44,7 +44,7 @@ endif CXXFLAGS += $(ADD_CFLAGS) -std=c++14 -Wall -Wextra -Wpedantic -Wno-unused-function -DBOOST_ERROR_CODE_HEADER_ONLY -DHAS_FLAC -DHAS_OGG -DHAS_OPUS -DVERSION=\"$(VERSION)\" -I. -I.. -I../common LDFLAGS += $(ADD_LDFLAGS) -logg -lFLAC -lopus -lsoxr -OBJ = snapclient.o stream.o client_connection.o time_provider.o player/player.o player/file_player.o decoder/pcm_decoder.o decoder/ogg_decoder.o decoder/flac_decoder.o decoder/opus_decoder.o controller.o ../common/sample_format.o +OBJ = snapclient.o stream.o client_connection.o time_provider.o player/player.o player/file_player.o decoder/pcm_decoder.o decoder/ogg_decoder.o decoder/flac_decoder.o decoder/opus_decoder.o controller.o ../common/sample_format.o ../common/resampler.o ifneq (,$(TARGET)) diff --git a/client/decoder/opus_decoder.cpp b/client/decoder/opus_decoder.cpp index 75d74020..4cc86651 100644 --- a/client/decoder/opus_decoder.cpp +++ b/client/decoder/opus_decoder.cpp @@ -49,10 +49,10 @@ OpusDecoder::~OpusDecoder() bool OpusDecoder::decode(msg::PcmChunk* chunk) { - int frame_size = 0; + int decoded_frames = 0; - while ((frame_size = opus_decode(dec_, (unsigned char*)chunk->payload, chunk->payloadSize, pcm_.data(), - static_cast(pcm_.size()) / sample_format_.channels(), 0)) == OPUS_BUFFER_TOO_SMALL) + while ((decoded_frames = opus_decode(dec_, (unsigned char*)chunk->payload, chunk->payloadSize, pcm_.data(), + static_cast(pcm_.size()) / sample_format_.channels(), 0)) == OPUS_BUFFER_TOO_SMALL) { if (pcm_.size() < const_max_frame_size * sample_format_.channels()) { @@ -63,18 +63,19 @@ bool OpusDecoder::decode(msg::PcmChunk* chunk) break; } - if (frame_size < 0) + if (decoded_frames < 0) { - LOG(ERROR, LOG_TAG) << "Failed to decode chunk: " << opus_strerror(frame_size) << ", IN size: " << chunk->payloadSize << ", OUT size: " << pcm_.size() - << '\n'; + LOG(ERROR, LOG_TAG) << "Failed to decode chunk: " << opus_strerror(decoded_frames) << ", IN size: " << chunk->payloadSize + << ", OUT size: " << pcm_.size() << '\n'; return false; } else { - LOG(DEBUG, LOG_TAG) << "Decoded chunk: size " << chunk->payloadSize << " bytes, decoded " << frame_size << " samples" << '\n'; + LOG(DEBUG, LOG_TAG) << "Decode chunk: " << decoded_frames << " frames, size: " << chunk->payloadSize + << " bytes, decoded: " << decoded_frames * sample_format_.frameSize() << " bytes\n"; // copy encoded data to chunk - chunk->payloadSize = frame_size * sample_format_.channels() * sizeof(opus_int16); + chunk->payloadSize = decoded_frames * sample_format_.frameSize(); // decoded_frames * sample_format_.channels() * sizeof(opus_int16); chunk->payload = (char*)realloc(chunk->payload, chunk->payloadSize); memcpy(chunk->payload, (char*)pcm_.data(), chunk->payloadSize); return true; @@ -96,7 +97,7 @@ SampleFormat OpusDecoder::setHeader(msg::CodecHeader* chunk) // decode the sampleformat uint32_t rate; - memcpy(&rate, chunk->payload + 4, sizeof(id_opus)); + memcpy(&rate, chunk->payload + 4, sizeof(rate)); uint16_t bits; memcpy(&bits, chunk->payload + 8, sizeof(bits)); uint16_t channels; diff --git a/client/stream.cpp b/client/stream.cpp index 2389d094..2b34f75e 100644 --- a/client/stream.cpp +++ b/client/stream.cpp @@ -50,49 +50,20 @@ Stream::Stream(const SampleFormat& in_format, const SampleFormat& out_format) out_format.channels() != 0 ? out_format.channels() : format_.channels()); } -/* -48000 x -------- = ----- -47999,2 x - 1 + /* + 48000 x + ------- = ----- + 47999,2 x - 1 -x = 1,000016667 / (1,000016667 - 1) -*/ -// setRealSampleRate(format_.rate()); -#ifdef HAS_SOXR - soxr_ = nullptr; - if ((format_.rate() != in_format_.rate()) || (format_.bits() != in_format_.bits())) - { - LOG(INFO, LOG_TAG) << "Resampling from " << in_format_.toString() << " to " << format_.toString() << "\n"; - soxr_error_t error; - - soxr_datatype_t in_type = SOXR_INT16_I; - soxr_datatype_t out_type = SOXR_INT16_I; - if (in_format_.sampleSize() > 2) - in_type = SOXR_INT32_I; - if (format_.sampleSize() > 2) - out_type = SOXR_INT32_I; - soxr_io_spec_t iospec = soxr_io_spec(in_type, out_type); - // HQ should be fine: http://sox.sourceforge.net/Docs/FAQ - soxr_quality_spec_t q_spec = soxr_quality_spec(SOXR_HQ, 0); - soxr_ = soxr_create(static_cast(in_format_.rate()), static_cast(format_.rate()), format_.channels(), &error, &iospec, &q_spec, NULL); - if (error) - { - LOG(ERROR, LOG_TAG) << "Error soxr_create: " << error << "\n"; - soxr_ = nullptr; - } - // initialize the buffer with 20ms (~latency of the reampler) - resample_buffer_.resize(format_.frameSize() * static_cast(ceil(format_.msRate() * 20))); - } -#endif + x = 1,000016667 / (1,000016667 - 1) + */ + // setRealSampleRate(format_.rate()); + resampler_ = std::make_unique(in_format_, format_); } Stream::~Stream() { -#ifdef HAS_SOXR - if (soxr_) - soxr_delete(soxr_); -#endif } @@ -127,92 +98,15 @@ void Stream::clearChunks() void Stream::addChunk(unique_ptr chunk) { // drop chunk if it's too old. Just in case, this shouldn't happen. - cs::usec age = std::chrono::duration_cast(TimeProvider::serverNow() - chunk->start()); + auto age = std::chrono::duration_cast(TimeProvider::serverNow() - chunk->start()); if (age > 5s + bufferMs_) return; -// LOG(DEBUG, LOG_TAG) << "new chunk: " << chunk->durationMs() << " ms, Chunks: " << chunks_.size() << "\n"; + LOG(DEBUG, LOG_TAG) << "new chunk: " << chunk->durationMs() << " ms, age: " << age.count() << " ms, Chunks: " << chunks_.size() << "\n"; -#ifndef HAS_SOXR - chunks_.push(move(chunk)); -#else - if (soxr_ == nullptr) - { - chunks_.push(move(chunk)); - } - else - { - if (in_format_.bits() == 24) - { - // sox expects 32 bit input, shift 8 bits left - int32_t* frames = (int32_t*)chunk->payload; - for (size_t n = 0; n < chunk->getSampleCount(); ++n) - frames[n] = frames[n] << 8; - } - - size_t idone; - size_t odone; - auto resample_buffer_framesize = resample_buffer_.size() / format_.frameSize(); - auto error = soxr_process(soxr_, chunk->payload, chunk->getFrameCount(), &idone, resample_buffer_.data(), resample_buffer_framesize, &odone); - if (error) - { - LOG(ERROR, LOG_TAG) << "Error soxr_process: " << error << "\n"; - } - else - { - LOG(TRACE, LOG_TAG) << "Resample idone: " << idone << "/" << chunk->getFrameCount() << ", odone: " << odone << "/" - << resample_buffer_.size() / format_.frameSize() << ", delay: " << soxr_delay(soxr_) << "\n"; - - // some data has been resampled (odone frames) and some is still in the pipe (soxr_delay frames) - if (odone > 0) - { - // get the resamples ts from the input ts - auto input_end_ts = chunk->start() + chunk->duration(); - double resampled_ms = (odone + soxr_delay(soxr_)) / format_.msRate(); - auto resampled_start = input_end_ts - std::chrono::microseconds(static_cast(resampled_ms * 1000.)); - - auto resampled_chunk = new msg::PcmChunk(format_, 0); - auto us = chrono::duration_cast(resampled_start.time_since_epoch()).count(); - resampled_chunk->timestamp.sec = static_cast(us / 1000000); - resampled_chunk->timestamp.usec = static_cast(us % 1000000); - - // copy from the resample_buffer to the resampled chunk - resampled_chunk->payloadSize = static_cast(odone * format_.frameSize()); - resampled_chunk->payload = (char*)realloc(resampled_chunk->payload, resampled_chunk->payloadSize); - memcpy(resampled_chunk->payload, resample_buffer_.data(), resampled_chunk->payloadSize); - - if (format_.bits() == 24) - { - // sox has quantized to 32 bit, shift 8 bits right - int32_t* frames = (int32_t*)resampled_chunk->payload; - for (size_t n = 0; n < resampled_chunk->getSampleCount(); ++n) - { - // +128 to round to the nearest so that quantisation steps are distributed evenly - frames[n] = (frames[n] + 128) >> 8; - if (frames[n] > 0x7fffffff) - frames[n] = 0x7fffffff; - } - } - chunks_.push(shared_ptr(resampled_chunk)); - - // check if the resample_buffer is large enough, or if soxr was using all available space - if (odone == resample_buffer_framesize) - { - // buffer for resampled data too small, add space for 5ms - resample_buffer_.resize(resample_buffer_.size() + format_.frameSize() * static_cast(ceil(format_.msRate() * 5))); - LOG(DEBUG, LOG_TAG) << "Resample buffer completely filled, adding space for 5ms; new buffer size: " << resample_buffer_.size() - << " bytes\n"; - } - - // //LOG(TRACE, LOG_TAG) << "ts: " << out->timestamp.sec << "s, " << out->timestamp.usec/1000.f << " ms, duration: " << odone / format_.msRate() - // << "\n"; - // int64_t next_us = us + static_cast(odone / format_.msRate() * 1000); - // LOG(TRACE, LOG_TAG) << "ts: " << us << ", next: " << next_us << ", diff: " << next_us_ - us << "\n"; - // next_us_ = next_us; - } - } - } -#endif + auto resampled = resampler_->resample(std::move(chunk)); + if (resampled) + chunks_.push(move(resampled)); } @@ -372,6 +266,9 @@ bool Stream::getPlayerChunk(void* outputBuffer, const cs::usec& outputBufferDacT { if (age.count() > 0) { + // TODO: should be enough to check if "age.count() > chunk->duration" + // if "age.count > 0 && age.count < chunk->duration" then + // the current chunk could be fast forwarded by age.count, instead of dropping the whole chunk LOG(DEBUG, LOG_TAG) << "age > 0: " << age.count() / 1000 << "ms\n"; // age > 0: the top of the stream is too old. We must fast foward. // delete the current chunk, it's too old. This will avoid an endless loop if there is no chunk in the queue. diff --git a/client/stream.hpp b/client/stream.hpp index a5b75079..1d018a0b 100644 --- a/client/stream.hpp +++ b/client/stream.hpp @@ -16,14 +16,15 @@ along with this program. If not, see . ***/ -#ifndef STREAM_H -#define STREAM_H +#ifndef STREAM_HPP +#define STREAM_HPP #include "common/queue.h" #include "common/sample_format.hpp" #include "double_buffer.hpp" #include "message/message.hpp" #include "message/pcm_chunk.hpp" +#include "resampler.hpp" #include #include #ifdef HAS_SOXR @@ -102,9 +103,8 @@ private: int32_t correctAfterXFrames_; chronos::msec bufferMs_; -#ifdef HAS_SOXR - soxr_t soxr_; -#endif + std::unique_ptr resampler_; + std::vector resample_buffer_; std::vector read_buffer_; int frame_delta_; diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt index 28d64138..d0f2c03e 100644 --- a/common/CMakeLists.txt +++ b/common/CMakeLists.txt @@ -1,5 +1,17 @@ +set(SOURCES + resampler.cpp + sample_format.cpp) + if(NOT WIN32) - add_library(common STATIC daemon.cpp sample_format.cpp) -else() - add_library(common STATIC sample_format.cpp) + list(APPEND SOURCES daemon.cpp) endif() + +if (SOXR_FOUND) + include_directories(${SOXR_INCLUDE_DIRS}) +endif (SOXR_FOUND) + +add_library(common STATIC ${SOURCES}) + +if (SOXR_FOUND) + target_link_libraries(common ${SOXR_LIBRARIES}) +endif (SOXR_FOUND) diff --git a/common/message/pcm_chunk.hpp b/common/message/pcm_chunk.hpp index 187f45b9..c3a9b954 100644 --- a/common/message/pcm_chunk.hpp +++ b/common/message/pcm_chunk.hpp @@ -58,6 +58,19 @@ public: } #endif + // std::unique_ptr consume(uint32_t frameCount) + // { + // auto result = std::make_unique(format, 0); + // if (frameCount * format.frameSize() > payloadSize) + // frameCount = payloadSize / format.frameSize(); + // result->payload = payload; + // result->payloadSize = frameCount * format.frameSize(); + // payloadSize -= result->payloadSize; + // payload = (char*)realloc(payload + result->payloadSize, payloadSize); + // // payload += result->payloadSize; + // return result; + // } + int readFrames(void* outputBuffer, uint32_t frameCount) { // logd << "read: " << frameCount << ", total: " << (wireChunk->length / format.frameSize()) << ", idx: " << idx;// << std::endl; @@ -87,7 +100,6 @@ public: return idx_; } - chronos::time_point_clk start() const override { return chronos::time_point_clk(chronos::sec(timestamp.sec) + chronos::usec(timestamp.usec) + @@ -105,6 +117,14 @@ public: return std::chrono::duration_cast(chronos::nsec(static_cast(1000000 * getFrameCount() / format.msRate()))); } + // void append(const PcmChunk& chunk) + // { + // auto newSize = payloadSize + chunk.payloadSize; + // payload = (char*)realloc(payload, newSize); + // memcpy(payload + payloadSize, chunk.payload, chunk.payloadSize); + // payloadSize = newSize; + // } + double durationMs() const { return static_cast(getFrameCount()) / format.msRate(); diff --git a/common/resampler.cpp b/common/resampler.cpp new file mode 100644 index 00000000..9f737c37 --- /dev/null +++ b/common/resampler.cpp @@ -0,0 +1,180 @@ +/*** + This file is part of snapcast + Copyright (C) 2014-2020 Johannes Pohl + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +***/ + +#include "resampler.hpp" +#include "common/aixlog.hpp" + +using namespace std; + +static constexpr auto LOG_TAG = "Resampler"; + +Resampler::Resampler(const SampleFormat& in_format, const SampleFormat& out_format) : in_format_(in_format), out_format_(out_format) +{ +#ifdef HAS_SOXR + soxr_ = nullptr; + if ((out_format_.rate() != in_format_.rate()) || (out_format_.bits() != in_format_.bits())) + { + LOG(INFO, LOG_TAG) << "Resampling from " << in_format_.toString() << " to " << out_format_.toString() << "\n"; + soxr_error_t error; + + soxr_datatype_t in_type = SOXR_INT16_I; + soxr_datatype_t out_type = SOXR_INT16_I; + if (in_format_.sampleSize() > 2) + in_type = SOXR_INT32_I; + if (out_format_.sampleSize() > 2) + out_type = SOXR_INT32_I; + soxr_io_spec_t iospec = soxr_io_spec(in_type, out_type); + // HQ should be fine: http://sox.sourceforge.net/Docs/FAQ + soxr_quality_spec_t q_spec = soxr_quality_spec(SOXR_HQ, 0); + soxr_ = + soxr_create(static_cast(in_format_.rate()), static_cast(out_format_.rate()), in_format_.channels(), &error, &iospec, &q_spec, NULL); + if (error) + { + LOG(ERROR, LOG_TAG) << "Error soxr_create: " << error << "\n"; + soxr_ = nullptr; + } + // initialize the buffer with 20ms (~latency of the reampler) + resample_buffer_.resize(out_format_.frameSize() * static_cast(ceil(out_format_.msRate() * 20))); + } +#endif + // resampled_chunk_ = std::make_unique(out_format_, 0); +} + + +// std::shared_ptr Resampler::resample(std::shared_ptr chunk, chronos::usec duration) +// { +// auto resampled_chunk = resample(chunk); +// if (!resampled_chunk) +// return nullptr; +// std::cerr << "1\n"; +// resampled_chunk_->append(*resampled_chunk); +// std::cerr << "2\n"; +// while (resampled_chunk_->duration() >= duration) +// { +// LOG(DEBUG, LOG_TAG) << "resampled duration: " << resampled_chunk_->durationMs() << ", consuming: " << out_format_.usRate() * duration.count() << +// "\n"; +// auto chunk = resampled_chunk_->consume(out_format_.usRate() * duration.count()); +// LOG(DEBUG, LOG_TAG) << "consumed: " << chunk->durationMs() << ", resampled duration: " << resampled_chunk_->durationMs() << "\n"; +// return chunk; +// } +// // resampled_chunks_.push_back(resampled_chunk); +// // chronos::usec avail; +// // for (const auto& chunk: resampled_chunks_) +// // { +// // avail += chunk->durationLeft(); +// // if (avail >= duration) +// // { + +// // } +// // } +// } + +shared_ptr Resampler::resample(shared_ptr chunk) +{ +#ifndef HAS_SOXR + return chunk; +#else + if (soxr_ == nullptr) + { + return chunk; + } + else + { + if (in_format_.bits() == 24) + { + // sox expects 32 bit input, shift 8 bits left + int32_t* frames = (int32_t*)chunk->payload; + for (size_t n = 0; n < chunk->getSampleCount(); ++n) + frames[n] = frames[n] << 8; + } + + size_t idone; + size_t odone; + auto resample_buffer_framesize = resample_buffer_.size() / out_format_.frameSize(); + auto error = soxr_process(soxr_, chunk->payload, chunk->getFrameCount(), &idone, resample_buffer_.data(), resample_buffer_framesize, &odone); + if (error) + { + LOG(ERROR, LOG_TAG) << "Error soxr_process: " << error << "\n"; + } + else + { + LOG(TRACE, LOG_TAG) << "Resample idone: " << idone << "/" << chunk->getFrameCount() << ", odone: " << odone << "/" + << resample_buffer_.size() / out_format_.frameSize() << ", delay: " << soxr_delay(soxr_) << "\n"; + + // some data has been resampled (odone frames) and some is still in the pipe (soxr_delay frames) + if (odone > 0) + { + // get the resampled ts from the input ts + auto input_end_ts = chunk->start() + chunk->duration(); + double resampled_ms = (odone + soxr_delay(soxr_)) / out_format_.msRate(); + auto resampled_start = input_end_ts - std::chrono::microseconds(static_cast(resampled_ms * 1000.)); + + auto resampled_chunk = std::make_shared(out_format_, 0); + auto us = chrono::duration_cast(resampled_start.time_since_epoch()).count(); + resampled_chunk->timestamp.sec = static_cast(us / 1000000); + resampled_chunk->timestamp.usec = static_cast(us % 1000000); + + // copy from the resample_buffer to the resampled chunk + resampled_chunk->payloadSize = static_cast(odone * out_format_.frameSize()); + resampled_chunk->payload = (char*)realloc(resampled_chunk->payload, resampled_chunk->payloadSize); + memcpy(resampled_chunk->payload, resample_buffer_.data(), resampled_chunk->payloadSize); + + if (out_format_.bits() == 24) + { + // sox has quantized to 32 bit, shift 8 bits right + int32_t* frames = (int32_t*)resampled_chunk->payload; + for (size_t n = 0; n < resampled_chunk->getSampleCount(); ++n) + { + // +128 to round to the nearest so that quantisation steps are distributed evenly + frames[n] = (frames[n] + 128) >> 8; + if (frames[n] > 0x7fffffff) + frames[n] = 0x7fffffff; + } + } + + // check if the resample_buffer is large enough, or if soxr was using all available space + if (odone == resample_buffer_framesize) + { + // buffer for resampled data too small, add space for 5ms + resample_buffer_.resize(resample_buffer_.size() + out_format_.frameSize() * static_cast(ceil(out_format_.msRate() * 5))); + LOG(DEBUG, LOG_TAG) << "Resample buffer completely filled, adding space for 5ms; new buffer size: " << resample_buffer_.size() + << " bytes\n"; + } + + // //LOG(TRACE, LOG_TAG) << "ts: " << out->timestamp.sec << "s, " << out->timestamp.usec/1000.f << " ms, duration: " << odone / format_.msRate() + // << "\n"; + // int64_t next_us = us + static_cast(odone / format_.msRate() * 1000); + // LOG(TRACE, LOG_TAG) << "ts: " << us << ", next: " << next_us << ", diff: " << next_us_ - us << "\n"; + // next_us_ = next_us; + + return resampled_chunk; + } + } + } + return nullptr; +#endif +} + + +Resampler::~Resampler() +{ +#ifdef HAS_SOXR + if (soxr_) + soxr_delete(soxr_); +#endif +} diff --git a/common/resampler.hpp b/common/resampler.hpp new file mode 100644 index 00000000..b496df6e --- /dev/null +++ b/common/resampler.hpp @@ -0,0 +1,50 @@ +/*** + This file is part of snapcast + Copyright (C) 2014-2020 Johannes Pohl + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +***/ + +#ifndef RESAMPLER_HPP +#define RESAMPLER_HPP + +#include "common/message/pcm_chunk.hpp" +#include "common/sample_format.hpp" +#include +#include +#ifdef HAS_SOXR +#include +#endif + + +class Resampler +{ +public: + Resampler(const SampleFormat& in_format, const SampleFormat& out_format); + virtual ~Resampler(); + + // std::shared_ptr resample(std::shared_ptr chunk, chronos::usec duration); + std::shared_ptr resample(std::shared_ptr chunk); + +private: + std::vector resample_buffer_; + // std::unique_ptr resampled_chunk_; + SampleFormat in_format_; + SampleFormat out_format_; +#ifdef HAS_SOXR + soxr_t soxr_; +#endif +}; + +#endif diff --git a/server/Makefile b/server/Makefile index 1e4ab6e8..ac904f77 100644 --- a/server/Makefile +++ b/server/Makefile @@ -43,8 +43,8 @@ ifneq ($(SANITIZE), ) endif CXXFLAGS += $(ADD_CFLAGS) -std=c++14 -Wall -Wextra -Wpedantic -Wno-unused-function -DBOOST_ERROR_CODE_HEADER_ONLY -DHAS_FLAC -DHAS_OGG -DHAS_VORBIS -DHAS_VORBIS_ENC -DHAS_OPUS -DVERSION=\"$(VERSION)\" -I. -I.. -I../common -LDFLAGS += $(ADD_LDFLAGS) -lvorbis -lvorbisenc -logg -lFLAC -lopus -OBJ = snapserver.o server.o config.o control_server.o control_session_tcp.o control_session_http.o control_session_ws.o stream_server.o stream_session.o stream_session_tcp.o stream_session_ws.o streamreader/stream_uri.o streamreader/base64.o streamreader/stream_manager.o streamreader/pcm_stream.o streamreader/posix_stream.o streamreader/pipe_stream.o streamreader/file_stream.o streamreader/tcp_stream.o streamreader/process_stream.o streamreader/airplay_stream.o streamreader/librespot_stream.o streamreader/watchdog.o encoder/encoder_factory.o encoder/flac_encoder.o encoder/opus_encoder.o encoder/pcm_encoder.o encoder/ogg_encoder.o ../common/sample_format.o +LDFLAGS += $(ADD_LDFLAGS) -lvorbis -lvorbisenc -logg -lFLAC -lopus -lsoxr +OBJ = snapserver.o server.o config.o control_server.o control_session_tcp.o control_session_http.o control_session_ws.o stream_server.o stream_session.o stream_session_tcp.o stream_session_ws.o streamreader/stream_uri.o streamreader/base64.o streamreader/stream_manager.o streamreader/pcm_stream.o streamreader/posix_stream.o streamreader/pipe_stream.o streamreader/file_stream.o streamreader/tcp_stream.o streamreader/process_stream.o streamreader/airplay_stream.o streamreader/librespot_stream.o streamreader/watchdog.o encoder/encoder_factory.o encoder/flac_encoder.o encoder/opus_encoder.o encoder/pcm_encoder.o encoder/ogg_encoder.o ../common/sample_format.o ../common/resampler.o ifneq (,$(TARGET)) CXXFLAGS += -D$(TARGET) diff --git a/server/encoder/opus_encoder.cpp b/server/encoder/opus_encoder.cpp index e80f8c42..abaf5a73 100644 --- a/server/encoder/opus_encoder.cpp +++ b/server/encoder/opus_encoder.cpp @@ -31,6 +31,8 @@ namespace encoder static constexpr opus_int32 const_min_bitrate = 6000; static constexpr opus_int32 const_max_bitrate = 512000; +static constexpr auto LOG_TAG = "OpusEncoder"; + namespace { template @@ -77,8 +79,16 @@ void OpusEncoder::initEncoder() { // Opus is quite restrictive in sample rate and bit depth // It can handle mono signals, but we will check for stereo - if ((sampleFormat_.rate() != 48000) || (sampleFormat_.bits() != 16) || (sampleFormat_.channels() != 2)) - throw SnapException("Opus sampleformat must be 48000:16:2"); + // if ((sampleFormat_.rate() != 48000) || (sampleFormat_.bits() != 16) || (sampleFormat_.channels() != 2)) + // throw SnapException("Opus sampleformat must be 48000:16:2"); + if (sampleFormat_.channels() != 2) + throw SnapException("Opus requires a stereo signal"); + SampleFormat out{48000, 16, 2}; + if ((sampleFormat_.rate() != 48000) || (sampleFormat_.bits() != 16)) + LOG(INFO, LOG_TAG) << "Resampling input from " << sampleFormat_.toString() << " to " << out.toString() << " as required by Opus\n"; + + resampler_ = make_unique(sampleFormat_, out); + sampleFormat_ = out; opus_int32 bitrate = 192000; opus_int32 complexity = 10; @@ -132,7 +142,7 @@ void OpusEncoder::initEncoder() throw SnapException("Opus error parsing options: " + codecOptions_); } - LOG(INFO) << "Opus bitrate: " << bitrate << " bps, complexity: " << complexity << "\n"; + LOG(INFO, LOG_TAG) << "Opus bitrate: " << bitrate << " bps, complexity: " << complexity << "\n"; int error; enc_ = opus_encoder_create(sampleFormat_.rate(), sampleFormat_.channels(), OPUS_APPLICATION_RESTRICTED_LOWDELAY, &error); @@ -166,7 +176,15 @@ void OpusEncoder::initEncoder() // and encode the buffer content in the next iteration void OpusEncoder::encode(const msg::PcmChunk* chunk) { - // LOG(TRACE) << "encode " << chunk->duration().count() << "ms\n"; + // chunk = + // resampler_->resample(std::make_shared(chunk)).get(); + auto in = std::make_shared(*chunk); + auto out = resampler_->resample(in); //, std::chrono::milliseconds(20)); + if (out == nullptr) + return; + chunk = out.get(); + + // LOG(TRACE, LOG_TAG) << "encode " << chunk->duration().count() << "ms\n"; uint32_t offset = 0; // check if there is something left from the last call to encode and fill the remainder buffer to @@ -175,13 +193,13 @@ void OpusEncoder::encode(const msg::PcmChunk* chunk) { offset = std::min(static_cast(remainder_max_size_ - remainder_->payloadSize), chunk->payloadSize); memcpy(remainder_->payload + remainder_->payloadSize, chunk->payload, offset); - // LOG(TRACE) << "remainder buffer size: " << remainder_->payloadSize << "/" << remainder_max_size_ << ", appending " << offset << " bytes\n"; + // LOG(TRACE, LOG_TAG) << "remainder buffer size: " << remainder_->payloadSize << "/" << remainder_max_size_ << ", appending " << offset << " bytes\n"; remainder_->payloadSize += offset; if (remainder_->payloadSize < remainder_max_size_) { - LOG(DEBUG) << "not enough data to encode (" << remainder_->payloadSize << " of " << remainder_max_size_ << " bytes)" - << "\n"; + LOG(DEBUG, LOG_TAG) << "not enough data to encode (" << remainder_->payloadSize << " of " << remainder_max_size_ << " bytes)" + << "\n"; return; } encode(chunk->format, remainder_->payload, remainder_->payloadSize); @@ -196,7 +214,8 @@ void OpusEncoder::encode(const msg::PcmChunk* chunk) uint32_t bytes = ms2bytes(duration); while (chunk->payloadSize - offset >= bytes) { - // LOG(TRACE) << "encoding " << duration << "ms (" << bytes << "), offset: " << offset << ", chunk size: " << chunk->payloadSize - offset << "\n"; + // LOG(TRACE, LOG_TAG) << "encoding " << duration << "ms (" << bytes << "), offset: " << offset << ", chunk size: " << chunk->payloadSize - offset + // << "\n"; encode(chunk->format, chunk->payload + offset, bytes); offset += bytes; } @@ -216,13 +235,13 @@ void OpusEncoder::encode(const msg::PcmChunk* chunk) void OpusEncoder::encode(const SampleFormat& format, const char* data, size_t size) { // void* buffer; - // LOG(INFO) << "frames: " << chunk->readFrames(buffer, std::chrono::milliseconds(10)) << "\n"; + // LOG(INFO, LOG_TAG) << "frames: " << chunk->readFrames(buffer, std::chrono::milliseconds(10)) << "\n"; int samples_per_channel = size / format.frameSize(); if (encoded_.size() < size) encoded_.resize(size); opus_int32 len = opus_encode(enc_, (opus_int16*)data, samples_per_channel, encoded_.data(), size); - // LOG(TRACE) << "Encode " << samples_per_channel << " frames, size " << size << " bytes, encoded: " << len << " bytes" << '\n'; + LOG(TRACE, LOG_TAG) << "Encode " << samples_per_channel << " frames, size " << size << " bytes, encoded: " << len << " bytes" << '\n'; if (len > 0) { @@ -235,7 +254,8 @@ void OpusEncoder::encode(const SampleFormat& format, const char* data, size_t si } else { - LOG(ERROR) << "Failed to encode chunk: " << opus_strerror(len) << ", samples / channel: " << samples_per_channel << ", bytes: " << size << '\n'; + LOG(ERROR, LOG_TAG) << "Failed to encode chunk: " << opus_strerror(len) << ", samples / channel: " << samples_per_channel << ", bytes: " << size + << '\n'; } } diff --git a/server/encoder/opus_encoder.hpp b/server/encoder/opus_encoder.hpp index 18ad9068..59713289 100644 --- a/server/encoder/opus_encoder.hpp +++ b/server/encoder/opus_encoder.hpp @@ -16,8 +16,10 @@ along with this program. If not, see . ***/ -#pragma once +#ifndef OPUS_ENCODER_HPP +#define OPUS_ENCODER_HPP +#include "common/resampler.hpp" #include "encoder.hpp" #include @@ -43,6 +45,9 @@ protected: std::vector encoded_; std::unique_ptr remainder_; size_t remainder_max_size_; + std::unique_ptr resampler_; }; } // namespace encoder + +#endif