Protocol changes -> master

assignment-client/CMakeLists.txt

@@ -22,6 +22,7 @@ link_hifi_libraries(
   material-networking model-networking ktx shaders
 )
 include_hifi_library_headers(procedural)
+include_hifi_library_headers(entities)
 
 add_crashpad()
 target_breakpad()

assignment-client/src/audio/AudioMixer.cpp

@@ -31,6 +31,7 @@
 #include <StDev.h>
 #include <UUID.h>
 #include <CPUDetect.h>
+#include <ZoneEntityItem.h>
 
 #include "AudioLogging.h"
 #include "AudioHelpers.h"
@@ -39,6 +40,8 @@
 #include "AvatarAudioStream.h"
 #include "InjectedAudioStream.h"
 #include "crash-handler/CrashHandler.h"
+#include "../AssignmentDynamicFactory.h"
+#include "../entities/AssignmentParentFinder.h"
 
 using namespace std;
 
@@ -56,14 +59,15 @@ float AudioMixer::_noiseMutingThreshold{ DEFAULT_NOISE_MUTING_THRESHOLD };
 float AudioMixer::_attenuationPerDoublingInDistance{ DEFAULT_ATTENUATION_PER_DOUBLING_IN_DISTANCE };
 map<QString, shared_ptr<CodecPlugin>> AudioMixer::_availableCodecs{ };
 QStringList AudioMixer::_codecPreferenceOrder{};
-vector<AudioMixer::ZoneDescription> AudioMixer::_audioZones;
-vector<AudioMixer::ZoneSettings> AudioMixer::_zoneSettings;
-vector<AudioMixer::ReverbSettings> AudioMixer::_zoneReverbSettings;
+unordered_map<QString, AudioMixer::ZoneSettings> AudioMixer::_audioZones;
 
 AudioMixer::AudioMixer(ReceivedMessage& message) :
     ThreadedAssignment(message)
 {
 
+    DependencyManager::registerInheritance<EntityDynamicFactoryInterface, AssignmentDynamicFactory>();
+    DependencyManager::set<AssignmentDynamicFactory>();
+
     // Always clear settings first
     // This prevents previous assignment settings from sticking around
     clearDomainSettings();
@@ -112,6 +116,8 @@ AudioMixer::AudioMixer(ReceivedMessage& message) :
         PacketReceiver::makeSourcedListenerReference<AudioMixer>(this, &AudioMixer::handleNodeMuteRequestPacket));
     packetReceiver.registerListener(PacketType::KillAvatar,
         PacketReceiver::makeSourcedListenerReference<AudioMixer>(this, &AudioMixer::handleKillAvatarPacket));
+    packetReceiver.registerListenerForTypes({ PacketType::OctreeStats, PacketType::EntityData, PacketType::EntityErase },
+        PacketReceiver::makeSourcedListenerReference<AudioMixer>(this, &AudioMixer::handleOctreePacket));
 
     packetReceiver.registerListenerForTypes({
         PacketType::ReplicatedMicrophoneAudioNoEcho,
@@ -405,7 +411,7 @@ void AudioMixer::start() {
 
     // prepare the NodeList
     nodeList->addSetOfNodeTypesToNodeInterestSet({
-        NodeType::Agent, NodeType::EntityScriptServer,
+        NodeType::Agent, NodeType::EntityScriptServer, NodeType::EntityServer,
         NodeType::UpstreamAudioMixer, NodeType::DownstreamAudioMixer
     });
     nodeList->linkedDataCreateCallback = [&](Node* node) { getOrCreateClientData(node); };
@@ -417,12 +423,19 @@ void AudioMixer::start() {
         parseSettingsObject(settingsObject);
     }
 
+    setupEntityQuery();
+
     // mix state
     unsigned int frame = 1;
 
     while (!_isFinished) {
         auto ticTimer = _ticTiming.timer();
 
+        // Set our query each frame
+        {
+            _entityViewer.queryOctree();
+        }
+
         if (_startFrameTimestamp.time_since_epoch().count() == 0) {
             _startFrameTimestamp = _idealFrameTimestamp = p_high_resolution_clock::now();
         } else {
@@ -555,8 +568,6 @@ void AudioMixer::clearDomainSettings() {
     _noiseMutingThreshold = DEFAULT_NOISE_MUTING_THRESHOLD;
     _codecPreferenceOrder.clear();
     _audioZones.clear();
-    _zoneSettings.clear();
-    _zoneReverbSettings.clear();
 }
 
 void AudioMixer::parseSettingsObject(const QJsonObject& settingsObject) {
@@ -727,8 +738,13 @@ void AudioMixer::parseSettingsObject(const QJsonObject& settingsObject) {
                     if (allOk) {
                         glm::vec3 corner(xMin, yMin, zMin);
                         glm::vec3 dimensions(xMax - xMin, yMax - yMin, zMax - zMin);
-                        AABox zoneAABox(corner, dimensions);
-                        _audioZones.push_back({ zoneName, zoneAABox });
+
+                        Transform t;
+                        t.setTranslation(corner + 0.5f * dimensions);
+                        t.setScale(dimensions);
+                        _audioZones[zoneName].inverseTransform = t.getInverseMatrix();
+                        _audioZones[zoneName].volume = dimensions.x * dimensions.y * dimensions.z;
+
                         qCDebug(audio) << "Added zone:" << zoneName << "(corner:" << corner << ", dimensions:" << dimensions << ")";
                     }
                 }
@@ -749,28 +765,17 @@ void AudioMixer::parseSettingsObject(const QJsonObject& settingsObject) {
                     coefficientObject.contains(LISTENER) &&
                     coefficientObject.contains(COEFFICIENT)) {
 
-                    auto itSource = find_if(begin(_audioZones), end(_audioZones), [&](const ZoneDescription& description) {
-                        return description.name == coefficientObject.value(SOURCE).toString();
-                    });
-                    auto itListener = find_if(begin(_audioZones), end(_audioZones), [&](const ZoneDescription& description) {
-                        return description.name == coefficientObject.value(LISTENER).toString();
-                    });
+                    auto itSource = _audioZones.find(coefficientObject.value(SOURCE).toString());
 
                     bool ok;
                     float coefficient = coefficientObject.value(COEFFICIENT).toString().toFloat(&ok);
 
+                    if (ok && coefficient <= 1.0f && itSource != _audioZones.end()) {
+                        auto listener = coefficientObject.value(LISTENER).toString();
+                        itSource->second.listeners.emplace_back(listener);
+                        itSource->second.coefficients.emplace_back(coefficient);
 
-                    if (ok && coefficient <= 1.0f &&
-                        itSource != end(_audioZones) &&
-                        itListener != end(_audioZones)) {
-
-                        ZoneSettings settings;
-                        settings.source = itSource - begin(_audioZones);
-                        settings.listener = itListener - begin(_audioZones);
-                        settings.coefficient = coefficient;
-
-                        _zoneSettings.push_back(settings);
-                        qCDebug(audio) << "Added Coefficient:" << itSource->name << itListener->name << settings.coefficient;
+                        qCDebug(audio) << "Added Coefficient:" << itSource->first << listener << coefficient;
                     }
                 }
             }
@@ -791,28 +796,125 @@ void AudioMixer::parseSettingsObject(const QJsonObject& settingsObject) {
                     reverbObject.contains(WET_LEVEL)) {
 
                     bool okReverbTime, okWetLevel;
-                    auto itZone = find_if(begin(_audioZones), end(_audioZones), [&](const ZoneDescription& description) {
-                        return description.name == reverbObject.value(ZONE).toString();
-                    });
+                    auto itZone = _audioZones.find(reverbObject.value(ZONE).toString());
                     float reverbTime = reverbObject.value(REVERB_TIME).toString().toFloat(&okReverbTime);
                     float wetLevel = reverbObject.value(WET_LEVEL).toString().toFloat(&okWetLevel);
 
-                    if (okReverbTime && okWetLevel && itZone != end(_audioZones)) {
-                        ReverbSettings settings;
-                        settings.zone = itZone - begin(_audioZones);
-                        settings.reverbTime = reverbTime;
-                        settings.wetLevel = wetLevel;
+                    if (okReverbTime && okWetLevel && itZone != _audioZones.end()) {
+                        itZone->second.reverbEnabled = true;
+                        itZone->second.reverbTime = reverbTime;
+                        itZone->second.wetLevel = wetLevel;
 
-                        _zoneReverbSettings.push_back(settings);
-
-                        qCDebug(audio) << "Added Reverb:" << itZone->name << reverbTime << wetLevel;
+                        qCDebug(audio) << "Added Reverb:" << itZone->first << reverbTime << wetLevel;
                     }
                 }
             }
         }
     }
 }
 
+void AudioMixer::setupEntityQuery() {
+    _entityViewer.init();
+    EntityTreePointer entityTree = _entityViewer.getTree();
+    entityTree->setIsServer(true);
+    DependencyManager::registerInheritance<SpatialParentFinder, AssignmentParentFinder>();
+    DependencyManager::set<AssignmentParentFinder>(entityTree);
+
+    connect(entityTree.get(), &EntityTree::addingEntityPointer, this, &AudioMixer::entityAdded);
+    connect(entityTree.get(), &EntityTree::deletingEntityPointer, this, &AudioMixer::entityRemoved);
+
+    // ES query: {"type": "Zone"}
+    QJsonObject zoneQuery;
+    zoneQuery["type"] = "Zone";
+
+    QJsonObject queryFlags;
+    queryFlags["includeAncestors"] = true;
+    queryFlags["includeDescendants"] = true;
+    zoneQuery["flags"] = queryFlags;
+    zoneQuery["name"] = true;  // Handy for debugging.
+
+    _entityViewer.getOctreeQuery().setJSONParameters(zoneQuery);
+}
+
+void AudioMixer::handleOctreePacket(QSharedPointer<ReceivedMessage> message, SharedNodePointer senderNode) {
+    PacketType packetType = message->getType();
+
+    switch (packetType) {
+    case PacketType::OctreeStats:
+    {   // Ignore stats, but may have a different Entity packet appended.
+        OctreeHeadlessViewer::parseOctreeStats(message, senderNode);
+        const auto piggyBackedSizeWithHeader = message->getBytesLeftToRead();
+        if (piggyBackedSizeWithHeader > 0) {
+            // pull out the piggybacked packet and create a new QSharedPointer<NLPacket> for it
+            auto buffer = std::unique_ptr<char[]>(new char[piggyBackedSizeWithHeader]);
+            memcpy(buffer.get(), message->getRawMessage() + message->getPosition(), piggyBackedSizeWithHeader);
+
+            auto newPacket = NLPacket::fromReceivedPacket(std::move(buffer), piggyBackedSizeWithHeader, message->getSenderSockAddr());
+            auto newMessage = QSharedPointer<ReceivedMessage>::create(*newPacket);
+            handleOctreePacket(newMessage, senderNode);
+        }
+        break;
+    }
+
+    case PacketType::EntityData:
+        _entityViewer.processDatagram(*message, senderNode);
+        break;
+
+    case PacketType::EntityErase:
+        _entityViewer.processEraseMessage(*message, senderNode);
+        break;
+
+    default:
+        qCDebug(audio) << "Unexpected packet type:" << packetType;
+        break;
+    }
+}
+
+void updateAudioZone(EntityItem* entity, std::unordered_map<QString, AudioMixer::ZoneSettings>& audioZones) {
+    auto zoneEntity = (ZoneEntityItem*)entity;
+    auto& audioZone = audioZones[entity->getID().toString()];
+    auto& audioSettings = zoneEntity->getAudioProperties();
+
+    vec3 dimensions = entity->getScaledDimensions();
+    Transform t;
+    t.setTranslation(entity->getWorldPosition());
+    t.setScale(dimensions);
+    t.setRotation(entity->getWorldOrientation());
+    audioZone.inverseTransform = t.getInverseMatrix();
+    audioZone.volume = dimensions.x * dimensions.y * dimensions.z;
+
+    audioZone.reverbEnabled = audioSettings.getReverbEnabled();
+    audioZone.reverbTime = audioSettings.getReverbTime();
+    audioZone.wetLevel = audioSettings.getReverbWetLevel();
+
+    audioZone.listeners.clear();
+    auto listenerZones = audioSettings.getListenerZones();
+    audioZone.listeners.reserve(listenerZones.length());
+    for (auto& listener : listenerZones) {
+        audioZone.listeners.push_back(listener.toString());
+    }
+
+    audioZone.coefficients = audioSettings.getListenerAttenuationCoefficients().toStdVector();
+
+    /*qCDebug(audio) << "Updated audio zone:" << entity->getID().toString() << "(position:" << t.getTranslation()
+                   << ", dimensions:" << t.getScale() << ")";*/
+}
+
+void AudioMixer::entityAdded(EntityItem* entity) {
+    if (entity->getType() == EntityTypes::Zone) {
+        updateAudioZone(entity, _audioZones);
+        entity->registerChangeHandler([entity](const EntityItemID& entityItemID) {
+            updateAudioZone(entity, _audioZones);
+        });
+    }
+}
+
+void AudioMixer::entityRemoved(EntityItem* entity) {
+    if (entity->getType() == EntityTypes::Zone) {
+        _audioZones.erase(entity->getID().toString());
+    }
+}
+
 AudioMixer::Timer::Timing::Timing(uint64_t& sum) : _sum(sum) {
     _timing = p_high_resolution_clock::now();
 }

assignment-client/src/audio/AudioMixer.h

@@ -14,7 +14,7 @@
 
 #include <QtCore/QSharedPointer>
 
-#include <AABox.h>
+#include <Transform.h>
 #include <AudioHRTF.h>
 #include <AudioRingBuffer.h>
 #include <ThreadedAssignment.h>
@@ -25,6 +25,8 @@
 #include "AudioMixerStats.h"
 #include "AudioMixerWorkerPool.h"
 
+#include "../entities/EntityTreeHeadlessViewer.h"
+
 class PositionalAudioStream;
 class AvatarAudioStream;
 class AudioHRTF;
@@ -36,28 +38,22 @@ class AudioMixer : public ThreadedAssignment {
 public:
     AudioMixer(ReceivedMessage& message);
 
-
-    struct ZoneDescription {
-        QString name;
-        AABox area;
-    };
     struct ZoneSettings {
-        int source;
-        int listener;
-        float coefficient;
-    };
-    struct ReverbSettings {
-        int zone;
-        float reverbTime;
-        float wetLevel;
+        glm::mat4 inverseTransform;
+        float volume { FLT_MAX };
+
+        bool reverbEnabled { false };
+        float reverbTime { 0.0f };
+        float wetLevel { 0.0f };
+
+        std::vector<QString> listeners;
+        std::vector<float> coefficients;
     };
 
     static int getStaticJitterFrames() { return _numStaticJitterFrames; }
     static bool shouldMute(float quietestFrame) { return quietestFrame > _noiseMutingThreshold; }
     static float getAttenuationPerDoublingInDistance() { return _attenuationPerDoublingInDistance; }
-    static const std::vector<ZoneDescription>& getAudioZones() { return _audioZones; }
-    static const std::vector<ZoneSettings>& getZoneSettings() { return _zoneSettings; }
-    static const std::vector<ReverbSettings>& getReverbSettings() { return _zoneReverbSettings; }
+    static const std::unordered_map<QString, ZoneSettings>& getAudioZones() { return _audioZones; }
     static const std::pair<QString, CodecPluginPointer> negotiateCodec(std::vector<QString> codecs);
 
     static bool shouldReplicateTo(const Node& from, const Node& to) {
@@ -72,12 +68,17 @@ public slots:
     void run() override;
     void sendStatsPacket() override;
 
+    // Audio zone possibly changed
+    void entityAdded(EntityItem* entity);
+    void entityRemoved(EntityItem* entity);
+
 private slots:
     // packet handlers
     void handleMuteEnvironmentPacket(QSharedPointer<ReceivedMessage> packet, SharedNodePointer sendingNode);
     void handleNodeMuteRequestPacket(QSharedPointer<ReceivedMessage> packet, SharedNodePointer sendingNode);
     void handleNodeKilled(SharedNodePointer killedNode);
     void handleKillAvatarPacket(QSharedPointer<ReceivedMessage> packet, SharedNodePointer sendingNode);
+    void handleOctreePacket(QSharedPointer<ReceivedMessage> message, SharedNodePointer senderNode);
 
     void queueAudioPacket(QSharedPointer<ReceivedMessage> packet, SharedNodePointer sendingNode);
     void queueReplicatedAudioPacket(QSharedPointer<ReceivedMessage> packet);
@@ -146,14 +147,17 @@ private slots:
     static std::map<QString, CodecPluginPointer> _availableCodecs;
     static QStringList _codecPreferenceOrder;
 
-    static std::vector<ZoneDescription> _audioZones;
-    static std::vector<ZoneSettings> _zoneSettings;
-    static std::vector<ReverbSettings> _zoneReverbSettings;
+    static std::unordered_map<QString, ZoneSettings> _audioZones;
 
     float _throttleStartTarget = 0.9f;
     float _throttleBackoffTarget = 0.44f;
 
     AudioMixerWorker::SharedData _workerSharedData;
+
+    void setupEntityQuery();
+
+    // Attach to entity tree for audio zone info.
+    EntityTreeHeadlessViewer _entityViewer;
 };
 
 #endif // hifi_AudioMixer_h