Skip to content

Commit

Permalink
Merge pull request #1096 from overte-org/protocol_changes
Browse files Browse the repository at this point in the history 
Protocol changes -> master
  • Loading branch information
@ksuprynowicz
ksuprynowicz authored Oct 21, 2024
2 parents b78d0b0 + e7f7314 commit ce8d57b
Show file tree
Hide file tree
Showing 348 changed files with 15,279 additions and 17,195 deletions.
1 change: 1 addition & 0 deletions assignment-client/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ link_hifi_libraries(
material-networking model-networking ktx shaders
)
include_hifi_library_headers(procedural)
include_hifi_library_headers(entities)

add_crashpad()
target_breakpad()
Expand Down
174 changes: 138 additions & 36 deletions assignment-client/src/audio/AudioMixer.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
#include <StDev.h>
#include <UUID.h>
#include <CPUDetect.h>
#include <ZoneEntityItem.h>

#include "AudioLogging.h"
#include "AudioHelpers.h"
Expand All @@ -39,6 +40,8 @@
#include "AvatarAudioStream.h"
#include "InjectedAudioStream.h"
#include "crash-handler/CrashHandler.h"
#include "../AssignmentDynamicFactory.h"
#include "../entities/AssignmentParentFinder.h"

using namespace std;

Expand All @@ -56,14 +59,15 @@ float AudioMixer::_noiseMutingThreshold{ DEFAULT_NOISE_MUTING_THRESHOLD };
float AudioMixer::_attenuationPerDoublingInDistance{ DEFAULT_ATTENUATION_PER_DOUBLING_IN_DISTANCE };
map<QString, shared_ptr<CodecPlugin>> AudioMixer::_availableCodecs{ };
QStringList AudioMixer::_codecPreferenceOrder{};
vector<AudioMixer::ZoneDescription> AudioMixer::_audioZones;
vector<AudioMixer::ZoneSettings> AudioMixer::_zoneSettings;
vector<AudioMixer::ReverbSettings> AudioMixer::_zoneReverbSettings;
unordered_map<QString, AudioMixer::ZoneSettings> AudioMixer::_audioZones;

AudioMixer::AudioMixer(ReceivedMessage& message) :
ThreadedAssignment(message)
{

DependencyManager::registerInheritance<EntityDynamicFactoryInterface, AssignmentDynamicFactory>();
DependencyManager::set<AssignmentDynamicFactory>();

// Always clear settings first
// This prevents previous assignment settings from sticking around
clearDomainSettings();
Expand Down Expand Up @@ -112,6 +116,8 @@ AudioMixer::AudioMixer(ReceivedMessage& message) :
PacketReceiver::makeSourcedListenerReference<AudioMixer>(this, &AudioMixer::handleNodeMuteRequestPacket));
packetReceiver.registerListener(PacketType::KillAvatar,
PacketReceiver::makeSourcedListenerReference<AudioMixer>(this, &AudioMixer::handleKillAvatarPacket));
packetReceiver.registerListenerForTypes({ PacketType::OctreeStats, PacketType::EntityData, PacketType::EntityErase },
PacketReceiver::makeSourcedListenerReference<AudioMixer>(this, &AudioMixer::handleOctreePacket));

packetReceiver.registerListenerForTypes({
PacketType::ReplicatedMicrophoneAudioNoEcho,
Expand Down Expand Up @@ -405,7 +411,7 @@ void AudioMixer::start() {

// prepare the NodeList
nodeList->addSetOfNodeTypesToNodeInterestSet({
NodeType::Agent, NodeType::EntityScriptServer,
NodeType::Agent, NodeType::EntityScriptServer, NodeType::EntityServer,
NodeType::UpstreamAudioMixer, NodeType::DownstreamAudioMixer
});
nodeList->linkedDataCreateCallback = [&](Node* node) { getOrCreateClientData(node); };
Expand All @@ -417,12 +423,19 @@ void AudioMixer::start() {
parseSettingsObject(settingsObject);
}

setupEntityQuery();

// mix state
unsigned int frame = 1;

while (!_isFinished) {
auto ticTimer = _ticTiming.timer();

// Set our query each frame
{
_entityViewer.queryOctree();
}

if (_startFrameTimestamp.time_since_epoch().count() == 0) {
_startFrameTimestamp = _idealFrameTimestamp = p_high_resolution_clock::now();
} else {
Expand Down Expand Up @@ -555,8 +568,6 @@ void AudioMixer::clearDomainSettings() {
_noiseMutingThreshold = DEFAULT_NOISE_MUTING_THRESHOLD;
_codecPreferenceOrder.clear();
_audioZones.clear();
_zoneSettings.clear();
_zoneReverbSettings.clear();
}

void AudioMixer::parseSettingsObject(const QJsonObject& settingsObject) {
Expand Down Expand Up @@ -727,8 +738,13 @@ void AudioMixer::parseSettingsObject(const QJsonObject& settingsObject) {
if (allOk) {
glm::vec3 corner(xMin, yMin, zMin);
glm::vec3 dimensions(xMax - xMin, yMax - yMin, zMax - zMin);
AABox zoneAABox(corner, dimensions);
_audioZones.push_back({ zoneName, zoneAABox });

Transform t;
t.setTranslation(corner + 0.5f * dimensions);
t.setScale(dimensions);
_audioZones[zoneName].inverseTransform = t.getInverseMatrix();
_audioZones[zoneName].volume = dimensions.x * dimensions.y * dimensions.z;

qCDebug(audio) << "Added zone:" << zoneName << "(corner:" << corner << ", dimensions:" << dimensions << ")";
}
}
Expand All @@ -749,28 +765,17 @@ void AudioMixer::parseSettingsObject(const QJsonObject& settingsObject) {
coefficientObject.contains(LISTENER) &&
coefficientObject.contains(COEFFICIENT)) {

auto itSource = find_if(begin(_audioZones), end(_audioZones), [&](const ZoneDescription& description) {
return description.name == coefficientObject.value(SOURCE).toString();
});
auto itListener = find_if(begin(_audioZones), end(_audioZones), [&](const ZoneDescription& description) {
return description.name == coefficientObject.value(LISTENER).toString();
});
auto itSource = _audioZones.find(coefficientObject.value(SOURCE).toString());

bool ok;
float coefficient = coefficientObject.value(COEFFICIENT).toString().toFloat(&ok);

if (ok && coefficient <= 1.0f && itSource != _audioZones.end()) {
auto listener = coefficientObject.value(LISTENER).toString();
itSource->second.listeners.emplace_back(listener);
itSource->second.coefficients.emplace_back(coefficient);

if (ok && coefficient <= 1.0f &&
itSource != end(_audioZones) &&
itListener != end(_audioZones)) {

ZoneSettings settings;
settings.source = itSource - begin(_audioZones);
settings.listener = itListener - begin(_audioZones);
settings.coefficient = coefficient;

_zoneSettings.push_back(settings);
qCDebug(audio) << "Added Coefficient:" << itSource->name << itListener->name << settings.coefficient;
qCDebug(audio) << "Added Coefficient:" << itSource->first << listener << coefficient;
}
}
}
Expand All @@ -791,28 +796,125 @@ void AudioMixer::parseSettingsObject(const QJsonObject& settingsObject) {
reverbObject.contains(WET_LEVEL)) {

bool okReverbTime, okWetLevel;
auto itZone = find_if(begin(_audioZones), end(_audioZones), [&](const ZoneDescription& description) {
return description.name == reverbObject.value(ZONE).toString();
});
auto itZone = _audioZones.find(reverbObject.value(ZONE).toString());
float reverbTime = reverbObject.value(REVERB_TIME).toString().toFloat(&okReverbTime);
float wetLevel = reverbObject.value(WET_LEVEL).toString().toFloat(&okWetLevel);

if (okReverbTime && okWetLevel && itZone != end(_audioZones)) {
ReverbSettings settings;
settings.zone = itZone - begin(_audioZones);
settings.reverbTime = reverbTime;
settings.wetLevel = wetLevel;
if (okReverbTime && okWetLevel && itZone != _audioZones.end()) {
itZone->second.reverbEnabled = true;
itZone->second.reverbTime = reverbTime;
itZone->second.wetLevel = wetLevel;

_zoneReverbSettings.push_back(settings);

qCDebug(audio) << "Added Reverb:" << itZone->name << reverbTime << wetLevel;
qCDebug(audio) << "Added Reverb:" << itZone->first << reverbTime << wetLevel;
}
}
}
}
}
}

void AudioMixer::setupEntityQuery() {
_entityViewer.init();
EntityTreePointer entityTree = _entityViewer.getTree();
entityTree->setIsServer(true);
DependencyManager::registerInheritance<SpatialParentFinder, AssignmentParentFinder>();
DependencyManager::set<AssignmentParentFinder>(entityTree);

connect(entityTree.get(), &EntityTree::addingEntityPointer, this, &AudioMixer::entityAdded);
connect(entityTree.get(), &EntityTree::deletingEntityPointer, this, &AudioMixer::entityRemoved);

// ES query: {"type": "Zone"}
QJsonObject zoneQuery;
zoneQuery["type"] = "Zone";

QJsonObject queryFlags;
queryFlags["includeAncestors"] = true;
queryFlags["includeDescendants"] = true;
zoneQuery["flags"] = queryFlags;
zoneQuery["name"] = true; // Handy for debugging.

_entityViewer.getOctreeQuery().setJSONParameters(zoneQuery);
}

void AudioMixer::handleOctreePacket(QSharedPointer<ReceivedMessage> message, SharedNodePointer senderNode) {
PacketType packetType = message->getType();

switch (packetType) {
case PacketType::OctreeStats:
{ // Ignore stats, but may have a different Entity packet appended.
OctreeHeadlessViewer::parseOctreeStats(message, senderNode);
const auto piggyBackedSizeWithHeader = message->getBytesLeftToRead();
if (piggyBackedSizeWithHeader > 0) {
// pull out the piggybacked packet and create a new QSharedPointer<NLPacket> for it
auto buffer = std::unique_ptr<char[]>(new char[piggyBackedSizeWithHeader]);
memcpy(buffer.get(), message->getRawMessage() + message->getPosition(), piggyBackedSizeWithHeader);

auto newPacket = NLPacket::fromReceivedPacket(std::move(buffer), piggyBackedSizeWithHeader, message->getSenderSockAddr());
auto newMessage = QSharedPointer<ReceivedMessage>::create(*newPacket);
handleOctreePacket(newMessage, senderNode);
}
break;
}

case PacketType::EntityData:
_entityViewer.processDatagram(*message, senderNode);
break;

case PacketType::EntityErase:
_entityViewer.processEraseMessage(*message, senderNode);
break;

default:
qCDebug(audio) << "Unexpected packet type:" << packetType;
break;
}
}

void updateAudioZone(EntityItem* entity, std::unordered_map<QString, AudioMixer::ZoneSettings>& audioZones) {
auto zoneEntity = (ZoneEntityItem*)entity;
auto& audioZone = audioZones[entity->getID().toString()];
auto& audioSettings = zoneEntity->getAudioProperties();

vec3 dimensions = entity->getScaledDimensions();
Transform t;
t.setTranslation(entity->getWorldPosition());
t.setScale(dimensions);
t.setRotation(entity->getWorldOrientation());
audioZone.inverseTransform = t.getInverseMatrix();
audioZone.volume = dimensions.x * dimensions.y * dimensions.z;

audioZone.reverbEnabled = audioSettings.getReverbEnabled();
audioZone.reverbTime = audioSettings.getReverbTime();
audioZone.wetLevel = audioSettings.getReverbWetLevel();

audioZone.listeners.clear();
auto listenerZones = audioSettings.getListenerZones();
audioZone.listeners.reserve(listenerZones.length());
for (auto& listener : listenerZones) {
audioZone.listeners.push_back(listener.toString());
}

audioZone.coefficients = audioSettings.getListenerAttenuationCoefficients().toStdVector();

/*qCDebug(audio) << "Updated audio zone:" << entity->getID().toString() << "(position:" << t.getTranslation()
<< ", dimensions:" << t.getScale() << ")";*/
}

void AudioMixer::entityAdded(EntityItem* entity) {
if (entity->getType() == EntityTypes::Zone) {
updateAudioZone(entity, _audioZones);
entity->registerChangeHandler([entity](const EntityItemID& entityItemID) {
updateAudioZone(entity, _audioZones);
});
}
}

void AudioMixer::entityRemoved(EntityItem* entity) {
if (entity->getType() == EntityTypes::Zone) {
_audioZones.erase(entity->getID().toString());
}
}

AudioMixer::Timer::Timing::Timing(uint64_t& sum) : _sum(sum) {
_timing = p_high_resolution_clock::now();
}
Expand Down
44 changes: 24 additions & 20 deletions assignment-client/src/audio/AudioMixer.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@

#include <QtCore/QSharedPointer>

#include <AABox.h>
#include <Transform.h>
#include <AudioHRTF.h>
#include <AudioRingBuffer.h>
#include <ThreadedAssignment.h>
Expand All @@ -25,6 +25,8 @@
#include "AudioMixerStats.h"
#include "AudioMixerWorkerPool.h"

#include "../entities/EntityTreeHeadlessViewer.h"

class PositionalAudioStream;
class AvatarAudioStream;
class AudioHRTF;
Expand All @@ -36,28 +38,22 @@ class AudioMixer : public ThreadedAssignment {
public:
AudioMixer(ReceivedMessage& message);


struct ZoneDescription {
QString name;
AABox area;
};
struct ZoneSettings {
int source;
int listener;
float coefficient;
};
struct ReverbSettings {
int zone;
float reverbTime;
float wetLevel;
glm::mat4 inverseTransform;
float volume { FLT_MAX };

bool reverbEnabled { false };
float reverbTime { 0.0f };
float wetLevel { 0.0f };

std::vector<QString> listeners;
std::vector<float> coefficients;
};

static int getStaticJitterFrames() { return _numStaticJitterFrames; }
static bool shouldMute(float quietestFrame) { return quietestFrame > _noiseMutingThreshold; }
static float getAttenuationPerDoublingInDistance() { return _attenuationPerDoublingInDistance; }
static const std::vector<ZoneDescription>& getAudioZones() { return _audioZones; }
static const std::vector<ZoneSettings>& getZoneSettings() { return _zoneSettings; }
static const std::vector<ReverbSettings>& getReverbSettings() { return _zoneReverbSettings; }
static const std::unordered_map<QString, ZoneSettings>& getAudioZones() { return _audioZones; }
static const std::pair<QString, CodecPluginPointer> negotiateCodec(std::vector<QString> codecs);

static bool shouldReplicateTo(const Node& from, const Node& to) {
Expand All @@ -72,12 +68,17 @@ public slots:
void run() override;
void sendStatsPacket() override;

// Audio zone possibly changed
void entityAdded(EntityItem* entity);
void entityRemoved(EntityItem* entity);

private slots:
// packet handlers
void handleMuteEnvironmentPacket(QSharedPointer<ReceivedMessage> packet, SharedNodePointer sendingNode);
void handleNodeMuteRequestPacket(QSharedPointer<ReceivedMessage> packet, SharedNodePointer sendingNode);
void handleNodeKilled(SharedNodePointer killedNode);
void handleKillAvatarPacket(QSharedPointer<ReceivedMessage> packet, SharedNodePointer sendingNode);
void handleOctreePacket(QSharedPointer<ReceivedMessage> message, SharedNodePointer senderNode);

void queueAudioPacket(QSharedPointer<ReceivedMessage> packet, SharedNodePointer sendingNode);
void queueReplicatedAudioPacket(QSharedPointer<ReceivedMessage> packet);
Expand Down Expand Up @@ -146,14 +147,17 @@ private slots:
static std::map<QString, CodecPluginPointer> _availableCodecs;
static QStringList _codecPreferenceOrder;

static std::vector<ZoneDescription> _audioZones;
static std::vector<ZoneSettings> _zoneSettings;
static std::vector<ReverbSettings> _zoneReverbSettings;
static std::unordered_map<QString, ZoneSettings> _audioZones;

float _throttleStartTarget = 0.9f;
float _throttleBackoffTarget = 0.44f;

AudioMixerWorker::SharedData _workerSharedData;

void setupEntityQuery();

// Attach to entity tree for audio zone info.
EntityTreeHeadlessViewer _entityViewer;
};

#endif // hifi_AudioMixer_h
Loading

0 comments on commit ce8d57b

Please sign in to comment.