diff --git a/CondTools/RunInfo/interface/LHCInfoHelper.h b/CondTools/RunInfo/interface/LHCInfoHelper.h index 61d005c287a17..60d96ed89a475 100644 --- a/CondTools/RunInfo/interface/LHCInfoHelper.h +++ b/CondTools/RunInfo/interface/LHCInfoHelper.h @@ -11,6 +11,9 @@ namespace cond { // Large number of LS for the OMS query, covering around 25 hours static constexpr unsigned int kLumisectionsQueryLimit = 4000; + // last Run number and LS number of the specified Fill + std::pair getFillLastRunAndLS(const cond::OMSService& oms, unsigned short fillId); + // Returns lumi-type IOV from last LS of last Run of the specified Fill cond::Time_t getFillLastLumiIOV(const cond::OMSService& oms, unsigned short fillId); diff --git a/CondTools/RunInfo/interface/LHCInfoPerFillPopConSourceHandler.h b/CondTools/RunInfo/interface/LHCInfoPerFillPopConSourceHandler.h new file mode 100644 index 0000000000000..9a0a5185258fc --- /dev/null +++ b/CondTools/RunInfo/interface/LHCInfoPerFillPopConSourceHandler.h @@ -0,0 +1,55 @@ +#include "CondCore/PopCon/interface/PopConSourceHandler.h" +#include "CondFormats/RunInfo/interface/LHCInfoPerFill.h" +#include "CondTools/RunInfo/interface/OMSAccess.h" +#include "FWCore/ParameterSet/interface/ParameterSet.h" + +class LHCInfoPerFillPopConSourceHandler : public popcon::PopConSourceHandler { +public: + LHCInfoPerFillPopConSourceHandler(edm::ParameterSet const& pset); + ~LHCInfoPerFillPopConSourceHandler() override = default; + + void getNewObjects() override; + std::string id() const override; + +private: + void addEmptyPayload(cond::Time_t iov); + + // Add payload to buffer and store corresponding lumiid IOV in m_timestampToLumiid map + void addPayloadToBuffer(cond::OMSServiceResultRef& row); + void convertBufferedIovsToLumiid(std::map timestampToLumiid); + + size_t getLumiData(const cond::OMSService& oms, + unsigned short fillId, + const boost::posix_time::ptime& beginFillTime, + const boost::posix_time::ptime& endFillTime); + + void getDipData(const cond::OMSService& oms, + const boost::posix_time::ptime& beginFillTime, + const boost::posix_time::ptime& endFillTime); + + bool getCTPPSData(cond::persistency::Session& session, + const boost::posix_time::ptime& beginFillTime, + const boost::posix_time::ptime& endFillTime); + + bool getEcalData(cond::persistency::Session& session, + const boost::posix_time::ptime& lowerTime, + const boost::posix_time::ptime& upperTime); + +private: + bool m_debug; + // starting date for sampling + boost::posix_time::ptime m_startTime; + boost::posix_time::ptime m_endTime; + bool m_endFillMode = true; + std::string m_name; + //for reading from relational database source + std::string m_connectionString, m_ecalConnectionString; + std::string m_authpath; + std::string m_omsBaseUrl; + std::unique_ptr m_fillPayload; + std::shared_ptr m_prevPayload; + std::vector>> m_tmpBuffer; + bool m_lastPayloadEmpty = false; + // to hold correspondance between timestamp-type IOVs and lumiid-type IOVs + std::map m_timestampToLumiid; +}; diff --git a/CondTools/RunInfo/interface/LHCInfoPerLSPopConSourceHandler.h b/CondTools/RunInfo/interface/LHCInfoPerLSPopConSourceHandler.h new file mode 100644 index 0000000000000..0578663626030 --- /dev/null +++ b/CondTools/RunInfo/interface/LHCInfoPerLSPopConSourceHandler.h @@ -0,0 +1,82 @@ +#ifndef LHCInfoPerLSPopConSourceHandler_h +#define LHCInfoPerLSPopConSourceHandler_h + +#include "CondCore/PopCon/interface/PopConSourceHandler.h" +#include "CondFormats/RunInfo/interface/LHCInfoPerLS.h" +#include "FWCore/ParameterSet/interface/ParameterSet.h" +#include "CondTools/RunInfo/interface/OMSAccess.h" +#include +#include +#include +#include +#include + +class LHCInfoPerLSPopConSourceHandler : public popcon::PopConSourceHandler { +public: + LHCInfoPerLSPopConSourceHandler(edm::ParameterSet const& pset); + ~LHCInfoPerLSPopConSourceHandler() override; + void getNewObjects() override; + std::string id() const override; + +private: + void populateIovs(); + void filterInvalidPayloads(); + bool isPayloadValid(const LHCInfoPerLS& payload) const; + void addEmptyPayload(cond::Time_t iov); + void addDefaultPayload(cond::Time_t iov, unsigned short fill, const cond::OMSService& oms); + void addDefaultPayload(cond::Time_t iov, unsigned short fill, int run, unsigned short lumi); + bool makeFillPayload(std::unique_ptr& targetPayload, const cond::OMSServiceResult& queryResult); + void addPayloadToBuffer(cond::OMSServiceResultRef& row); + size_t bufferAllLS(const cond::OMSServiceResult& queryResult); + size_t getLumiData(const cond::OMSService& oms, + unsigned short fillId, + const boost::posix_time::ptime& beginFillTime, + const boost::posix_time::ptime& endFillTime); + bool getCTPPSData(cond::persistency::Session& session, + const boost::posix_time::ptime& beginFillTime, + const boost::posix_time::ptime& endFillTime); + + bool m_debug; + // starting date for sampling + boost::posix_time::ptime m_startTime; + boost::posix_time::ptime m_endTime; + bool m_endFillMode = true; + std::string m_name; + // for reading from relational database source + std::string m_connectionString; + std::string m_authpath; + std::string m_omsBaseUrl; + // Allows for basic test of durigFill mode when there is no Stable Beams in LHC + // makes duringFill interpret fills as ongoing fill and writing their last LS + // (disabling the check if the last LS is in stable beams, + // although still only fills with stable beams are being processed + // also, still only up to one payload will be written) + const bool m_debugLogic; + // values for the default payload which is inserted after the last processed fill + // has ended and there's no ongoing stable beam yet: + float m_defaultCrossingAngleX; + float m_defaultCrossingAngleY; + float m_defaultBetaStarX; + float m_defaultBetaStarY; + float m_minBetaStar; // meters + float m_maxBetaStar; // meters + float m_minCrossingAngle; // urad + float m_maxCrossingAngle; // urad + + std::unique_ptr m_fillPayload; + std::shared_ptr m_prevPayload; + cond::Time_t m_startFillTime; + cond::Time_t m_endFillTime; + cond::Time_t m_prevEndFillTime = 0; + cond::Time_t m_prevStartFillTime; + cond::Time_t m_startStableBeamTime; + cond::Time_t m_endStableBeamTime; + std::vector>> m_tmpBuffer; + bool m_lastPayloadEmpty = false; + // mapping of lumisections IDs (pairs of runnumber an LS number) found in OMS to + // the IDs they've been assignd from PPS DB value pair(-1, -1) means lumisection + // corresponding to the key exists in OMS but no lumisection was matched from PPS + std::map, std::pair> m_lsIdMap; +}; + +#endif \ No newline at end of file diff --git a/CondTools/RunInfo/interface/LHCInfoPopConSourceHandler.h b/CondTools/RunInfo/interface/LHCInfoPopConSourceHandler.h index 6139ba4becd26..4f5a2dd8a4e0b 100644 --- a/CondTools/RunInfo/interface/LHCInfoPopConSourceHandler.h +++ b/CondTools/RunInfo/interface/LHCInfoPopConSourceHandler.h @@ -18,8 +18,6 @@ class LHCInfoPopConSourceHandler : public popcon::PopConSourceHandler { void getNewObjects() override; std::string id() const override; - static constexpr unsigned int kLumisectionsQueryLimit = 4000; // enough for fills not exceeding 25h - private: void addEmptyPayload(cond::Time_t iov); diff --git a/CondTools/RunInfo/plugins/BuildFile.xml b/CondTools/RunInfo/plugins/BuildFile.xml index 9616cb9213904..fddb5283b8496 100644 --- a/CondTools/RunInfo/plugins/BuildFile.xml +++ b/CondTools/RunInfo/plugins/BuildFile.xml @@ -59,6 +59,10 @@ + + + + @@ -67,10 +71,14 @@ + + + + - + diff --git a/CondTools/RunInfo/plugins/LHCInfoPerFillOnlinePopConAnalyzer.cc b/CondTools/RunInfo/plugins/LHCInfoPerFillOnlinePopConAnalyzer.cc new file mode 100644 index 0000000000000..700052917ccde --- /dev/null +++ b/CondTools/RunInfo/plugins/LHCInfoPerFillOnlinePopConAnalyzer.cc @@ -0,0 +1,7 @@ +#include "CondCore/PopCon/interface/OnlinePopConAnalyzer.h" +#include "CondTools/RunInfo/interface/LHCInfoPerFillPopConSourceHandler.h" +#include "FWCore/Framework/interface/MakerMacros.h" + +typedef popcon::OnlinePopConAnalyzer LHCInfoPerFillOnlinePopConAnalyzer; + +DEFINE_FWK_MODULE(LHCInfoPerFillOnlinePopConAnalyzer); diff --git a/CondTools/RunInfo/plugins/LHCInfoPerFillPopConAnalyzer.cc b/CondTools/RunInfo/plugins/LHCInfoPerFillPopConAnalyzer.cc index c87e36815e557..673ea289e788f 100644 --- a/CondTools/RunInfo/plugins/LHCInfoPerFillPopConAnalyzer.cc +++ b/CondTools/RunInfo/plugins/LHCInfoPerFillPopConAnalyzer.cc @@ -1,753 +1,7 @@ -#include "CondCore/CondDB/interface/ConnectionPool.h" #include "CondCore/PopCon/interface/PopConAnalyzer.h" -#include "CondCore/PopCon/interface/PopConSourceHandler.h" -#include "CondFormats/Common/interface/TimeConversions.h" -#include "CondFormats/RunInfo/interface/LHCInfoPerFill.h" -#include "CondTools/RunInfo/interface/LumiSectionFilter.h" -#include "CondTools/RunInfo/interface/LHCInfoHelper.h" -#include "CondTools/RunInfo/interface/OMSAccess.h" -#include "CoralBase/Attribute.h" -#include "CoralBase/AttributeList.h" -#include "CoralBase/AttributeSpecification.h" -#include "CoralBase/TimeStamp.h" +#include "CondTools/RunInfo/interface/LHCInfoPerFillPopConSourceHandler.h" #include "FWCore/Framework/interface/MakerMacros.h" -#include "FWCore/MessageLogger/interface/MessageLogger.h" -#include "FWCore/ParameterSet/interface/ParameterSet.h" -#include "FWCore/ParameterSet/interface/ParameterSetfwd.h" -#include "RelationalAccess/ICursor.h" -#include "RelationalAccess/IQuery.h" -#include "RelationalAccess/ISchema.h" -#include "RelationalAccess/ISessionProxy.h" - -class LHCInfoPerFillPopConSourceHandler; typedef popcon::PopConAnalyzer LHCInfoPerFillPopConAnalyzer; //define this as a plug-in DEFINE_FWK_MODULE(LHCInfoPerFillPopConAnalyzer); - -namespace cond { - namespace theLHCInfoPerFillPopConImpl { - - static const std::pair s_fillTypeMap[] = { - std::make_pair("PROTONS", LHCInfoPerFill::PROTONS), - std::make_pair("IONS", LHCInfoPerFill::IONS), - std::make_pair("COSMICS", LHCInfoPerFill::COSMICS), - std::make_pair("GAP", LHCInfoPerFill::GAP)}; - - static const std::pair s_particleTypeMap[] = { - std::make_pair("PROTON", LHCInfoPerFill::PROTON), - std::make_pair("PB82", LHCInfoPerFill::PB82), - std::make_pair("AR18", LHCInfoPerFill::AR18), - std::make_pair("D", LHCInfoPerFill::D), - std::make_pair("XE54", LHCInfoPerFill::XE54)}; - - LHCInfoPerFill::FillType fillTypeFromString(const std::string& s_fill_type) { - for (auto const& i : s_fillTypeMap) - if (s_fill_type == i.first) - return i.second; - return LHCInfoPerFill::UNKNOWN; - } - - LHCInfoPerFill::ParticleType particleTypeFromString(const std::string& s_particle_type) { - for (auto const& i : s_particleTypeMap) - if (s_particle_type == i.first) - return i.second; - return LHCInfoPerFill::NONE; - } - } // namespace theLHCInfoPerFillPopConImpl - - namespace impl { - - template <> - LHCInfoPerFill::FillType from_string(const std::string& attributeValue) { - return from_string_impl( - attributeValue, LHCInfoPerFill::UNKNOWN); - } - - template <> - LHCInfoPerFill::ParticleType from_string(const std::string& attributeValue) { - return from_string_impl( - attributeValue, LHCInfoPerFill::NONE); - } - - } // namespace impl -} // namespace cond - -namespace theLHCInfoPerFillImpl { - - bool makeFillPayload(std::unique_ptr& targetPayload, const cond::OMSServiceResult& queryResult) { - bool ret = false; - if (!queryResult.empty()) { - auto row = *queryResult.begin(); - auto currentFill = row.get("fill_number"); - auto bunches1 = row.get("bunches_beam1"); - auto bunches2 = row.get("bunches_beam2"); - auto collidingBunches = row.get("bunches_colliding"); - auto targetBunches = row.get("bunches_target"); - auto fillType = row.get("fill_type_runtime"); - auto particleType1 = row.get("fill_type_party1"); - auto particleType2 = row.get("fill_type_party2"); - auto intensityBeam1 = row.get("intensity_beam1"); - auto intensityBeam2 = row.get("intensity_beam2"); - auto energy = row.get("energy"); - auto creationTime = row.get("start_time"); - auto stableBeamStartTime = row.get("start_stable_beam"); - std::string endTimeStr = row.get("end_time"); - auto beamDumpTime = - (endTimeStr == "null") ? 0 : cond::time::from_boost(row.get("end_time")); - auto injectionScheme = row.get("injection_scheme"); - targetPayload = std::make_unique(); - targetPayload->setFillNumber(currentFill); - targetPayload->setBunchesInBeam1(bunches1); - targetPayload->setBunchesInBeam2(bunches2); - targetPayload->setCollidingBunches(collidingBunches); - targetPayload->setTargetBunches(targetBunches); - targetPayload->setFillType(fillType); - targetPayload->setParticleTypeForBeam1(particleType1); - targetPayload->setParticleTypeForBeam2(particleType2); - targetPayload->setIntensityForBeam1(intensityBeam1); - targetPayload->setIntensityForBeam2(intensityBeam2); - targetPayload->setEnergy(energy); - targetPayload->setCreationTime(cond::time::from_boost(creationTime)); - targetPayload->setBeginTime(cond::time::from_boost(stableBeamStartTime)); - targetPayload->setEndTime(beamDumpTime); - targetPayload->setInjectionScheme(injectionScheme); - ret = true; - } - return ret; - } -} // namespace theLHCInfoPerFillImpl - -namespace theLHCInfoPerFillImpl { - static const std::map vecMap = { - {"Beam1/beamPhaseMean", 1}, {"Beam2/beamPhaseMean", 2}, {"Beam1/cavPhaseMean", 3}, {"Beam2/cavPhaseMean", 4}}; - void setElementData(cond::Time_t since, - const std::string& dipVal, - unsigned int elementNr, - float value, - LHCInfoPerFill& payload, - std::set& initList) { - if (initList.find(since) == initList.end()) { - payload.beam1VC().resize(LHCInfoPerFill::bunchSlots, 0.); - payload.beam2VC().resize(LHCInfoPerFill::bunchSlots, 0.); - payload.beam1RF().resize(LHCInfoPerFill::bunchSlots, 0.); - payload.beam2RF().resize(LHCInfoPerFill::bunchSlots, 0.); - initList.insert(since); - } - // set the current values to all of the payloads of the lumi section samples after the current since - if (elementNr < LHCInfoPerFill::bunchSlots) { - switch (vecMap.at(dipVal)) { - case 1: - payload.beam1VC()[elementNr] = value; - break; - case 2: - payload.beam2VC()[elementNr] = value; - break; - case 3: - payload.beam1RF()[elementNr] = value; - break; - case 4: - payload.beam2RF()[elementNr] = value; - break; - default: - break; - } - } - } -} // namespace theLHCInfoPerFillImpl - -namespace theLHCInfoPerFillImpl { - bool comparePayloads(const LHCInfoPerFill& rhs, const LHCInfoPerFill& lhs) { - if (rhs.fillNumber() != lhs.fillNumber() || rhs.delivLumi() != lhs.delivLumi() || rhs.recLumi() != lhs.recLumi() || - rhs.instLumi() != lhs.instLumi() || rhs.instLumiError() != lhs.instLumiError() || - rhs.lhcState() != lhs.lhcState() || rhs.lhcComment() != lhs.lhcComment() || - rhs.ctppsStatus() != lhs.ctppsStatus()) { - return false; - } - return true; - } - - size_t transferPayloads(const std::vector>>& buffer, - std::map>& iovsToTransfer, - std::shared_ptr& prevPayload) { - size_t niovs = 0; - std::stringstream condIovs; - std::stringstream formattedIovs; - for (auto& iov : buffer) { - bool add = false; - auto payload = iov.second; - cond::Time_t since = iov.first; - if (iovsToTransfer.empty()) { - add = true; - } else { - LHCInfoPerFill& lastAdded = *iovsToTransfer.rbegin()->second; - if (!comparePayloads(lastAdded, *payload)) { - add = true; - } - } - if (add) { - niovs++; - condIovs << since << " "; - formattedIovs << boost::posix_time::to_iso_extended_string(cond::time::to_boost(since)) << " "; - iovsToTransfer.insert(std::make_pair(since, payload)); - prevPayload = iov.second; - } - } - edm::LogInfo("transferPayloads") << "TRANSFERED IOVS: " << condIovs.str(); - edm::LogInfo("transferPayloads") << "FORMATTED TRANSFERED IOVS: " << formattedIovs.str(); - return niovs; - } - -} // namespace theLHCInfoPerFillImpl -class LHCInfoPerFillPopConSourceHandler : public popcon::PopConSourceHandler { -public: - LHCInfoPerFillPopConSourceHandler(edm::ParameterSet const& pset) - : m_debug(pset.getUntrackedParameter("debug", false)), - m_startTime(), - m_endTime(), - m_endFillMode(pset.getUntrackedParameter("endFill", true)), - m_name(pset.getUntrackedParameter("name", "LHCInfoPerFillPopConSourceHandler")), - m_connectionString(pset.getUntrackedParameter("connectionString", "")), - m_ecalConnectionString(pset.getUntrackedParameter("ecalConnectionString", "")), - m_authpath(pset.getUntrackedParameter("authenticationPath", "")), - m_omsBaseUrl(pset.getUntrackedParameter("omsBaseUrl", "")), - m_fillPayload(), - m_prevPayload(), - m_tmpBuffer() { - if (!pset.getUntrackedParameter("startTime").empty()) { - m_startTime = boost::posix_time::time_from_string(pset.getUntrackedParameter("startTime")); - } - boost::posix_time::ptime now = boost::posix_time::second_clock::local_time(); - m_endTime = now; - if (!pset.getUntrackedParameter("endTime").empty()) { - m_endTime = boost::posix_time::time_from_string(pset.getUntrackedParameter("endTime")); - if (m_endTime > now) - m_endTime = now; - } - } - - ~LHCInfoPerFillPopConSourceHandler() override = default; - - void getNewObjects() override { - //if a new tag is created, transfer fake fill from 1 to the first fill for the first time - if (tagInfo().size == 0) { - edm::LogInfo(m_name) << "New tag " << tagInfo().name << "; from " << m_name << "::getNewObjects"; - } else { - //check what is already inside the database - edm::LogInfo(m_name) << "got info for tag " << tagInfo().name << ": size " << tagInfo().size - << ", last object valid since " << tagInfo().lastInterval.since << " ( " - << boost::posix_time::to_iso_extended_string( - cond::time::to_boost(tagInfo().lastInterval.since)) - << " ); from " << m_name << "::getNewObjects"; - } - - cond::Time_t lastSince = tagInfo().lastInterval.since; - if (tagInfo().isEmpty()) { - // for a new or empty tag, an empty payload should be added on top with since=1 - if (m_endFillMode) { - addEmptyPayload(1); - lastSince = 1; - } else { - addEmptyPayload(cond::time::lumiTime(1, 1)); - lastSince = cond::time::lumiTime(1, 1); - } - } else { - edm::LogInfo(m_name) << "The last Iov in tag " << tagInfo().name << " valid since " << lastSince << "from " - << m_name << "::getNewObjects"; - } - - //retrieve the data from the relational database source - cond::persistency::ConnectionPool connection; - //configure the connection - if (m_debug) { - connection.setMessageVerbosity(coral::Debug); - } else { - connection.setMessageVerbosity(coral::Error); - } - connection.setAuthenticationPath(m_authpath); - connection.configure(); - //create the sessions - cond::persistency::Session session = connection.createSession(m_connectionString, false); - cond::persistency::Session session2 = connection.createSession(m_ecalConnectionString, false); - // fetch last payload when available - if (!tagInfo().lastInterval.payloadId.empty()) { - cond::persistency::Session session3 = dbSession(); - session3.transaction().start(true); - m_prevPayload = session3.fetchPayload(tagInfo().lastInterval.payloadId); - session3.transaction().commit(); - } - - boost::posix_time::ptime executionTime = boost::posix_time::second_clock::local_time(); - cond::Time_t executionTimeIov = cond::time::from_boost(executionTime); - - cond::Time_t startTimestamp = m_startTime.is_not_a_date_time() ? 0 : cond::time::from_boost(m_startTime); - cond::Time_t nextFillSearchTimestamp = - std::max(startTimestamp, m_endFillMode ? lastSince : m_prevPayload->createTime()); - - edm::LogInfo(m_name) << "Starting sampling at " - << boost::posix_time::to_simple_string(cond::time::to_boost(nextFillSearchTimestamp)); - - while (true) { - if (nextFillSearchTimestamp >= executionTimeIov) { - edm::LogInfo(m_name) << "Sampling ended at the time " - << boost::posix_time::to_simple_string(cond::time::to_boost(executionTimeIov)); - break; - } - boost::posix_time::ptime nextFillSearchTime = cond::time::to_boost(nextFillSearchTimestamp); - boost::posix_time::ptime startSampleTime; - boost::posix_time::ptime endSampleTime; - - cond::OMSService oms; - oms.connect(m_omsBaseUrl); - auto query = oms.query("fills"); - - edm::LogInfo(m_name) << "Searching new fill after " << boost::posix_time::to_simple_string(nextFillSearchTime); - query->filterNotNull("start_stable_beam").filterNotNull("fill_number"); - if (nextFillSearchTime > cond::time::to_boost(m_prevPayload->createTime())) { - query->filterGE("start_time", nextFillSearchTime); - } else { - query->filterGT("start_time", nextFillSearchTime); - } - - query->filterLT("start_time", m_endTime); - if (m_endFillMode) - query->filterNotNull("end_time"); - bool foundFill = query->execute(); - if (foundFill) - foundFill = theLHCInfoPerFillImpl::makeFillPayload(m_fillPayload, query->result()); - if (!foundFill) { - edm::LogInfo(m_name) << "No fill found - END of job."; - break; - } - - startSampleTime = cond::time::to_boost(m_fillPayload->createTime()); - cond::Time_t startFillTime = m_fillPayload->createTime(); - cond::Time_t endFillTime = m_fillPayload->endTime(); - unsigned short lhcFill = m_fillPayload->fillNumber(); - bool ongoingFill = endFillTime == 0ULL; - if (ongoingFill) { - edm::LogInfo(m_name) << "Found ongoing fill " << lhcFill << " created at " - << cond::time::to_boost(startFillTime); - endSampleTime = executionTime; - nextFillSearchTimestamp = executionTimeIov; - } else { - edm::LogInfo(m_name) << "Found fill " << lhcFill << " created at " << cond::time::to_boost(startFillTime) - << " ending at " << cond::time::to_boost(endFillTime); - endSampleTime = cond::time::to_boost(endFillTime); - nextFillSearchTimestamp = endFillTime; - } - if (m_endFillMode || ongoingFill) { - getDipData(oms, startSampleTime, endSampleTime); - getLumiData(oms, lhcFill, startSampleTime, endSampleTime); - if (!m_tmpBuffer.empty()) { - boost::posix_time::ptime flumiStart = cond::time::to_boost(m_tmpBuffer.front().first); - boost::posix_time::ptime flumiStop = cond::time::to_boost(m_tmpBuffer.back().first); - edm::LogInfo(m_name) << "First lumi starts at " << flumiStart << " last lumi starts at " << flumiStop; - session.transaction().start(true); - getCTPPSData(session, startSampleTime, endSampleTime); - session.transaction().commit(); - session2.transaction().start(true); - getEcalData(session2, startSampleTime, endSampleTime); - session2.transaction().commit(); - } - } - - // In duringFill mode, convert the timestamp-type IOVs to lumiid-type IOVs - // before transferring the payloads from the buffer to the final collection - if (!m_endFillMode) { - convertBufferedIovsToLumiid(m_timestampToLumiid); - } - - size_t niovs = theLHCInfoPerFillImpl::transferPayloads(m_tmpBuffer, m_iovs, m_prevPayload); - edm::LogInfo(m_name) << "Added " << niovs << " iovs within the Fill time"; - m_tmpBuffer.clear(); - m_timestampToLumiid.clear(); - if (m_prevPayload->fillNumber() and !ongoingFill) { - if (m_endFillMode) { - addEmptyPayload(endFillTime); - } else { - addEmptyPayload(cond::lhcInfoHelper::getFillLastLumiIOV(oms, lhcFill)); - } - } - } - } - - std::string id() const override { return m_name; } - - static constexpr unsigned int kLumisectionsQueryLimit = 4000; - -private: - void addEmptyPayload(cond::Time_t iov) { - bool add = false; - if (m_iovs.empty()) { - if (!m_lastPayloadEmpty) - add = true; - } else { - auto lastAdded = m_iovs.rbegin()->second; - if (lastAdded->fillNumber() != 0) { - add = true; - } - } - if (add) { - auto newPayload = std::make_shared(); - m_iovs.insert(std::make_pair(iov, newPayload)); - m_prevPayload = newPayload; - edm::LogInfo(m_name) << "Added empty payload with IOV " << iov << " ( " - << boost::posix_time::to_iso_extended_string(cond::time::to_boost(iov)) << " )"; - } - } - - // Add payload to buffer and store corresponding lumiid IOV in m_timestampToLumiid map - void addPayloadToBuffer(cond::OMSServiceResultRef& row) { - auto startTime = row.get("start_time"); - auto delivLumi = row.get("delivered_lumi"); - auto recLumi = row.get("recorded_lumi"); - auto runNumber = std::stoul(row.get("run_number")); - auto lsNumber = std::stoul(row.get("lumisection_number")); - auto lumiid = cond::time::lumiTime(runNumber, lsNumber); - - LHCInfoPerFill* thisLumiSectionInfo = m_fillPayload->cloneFill(); - m_tmpBuffer.emplace_back(std::make_pair(cond::time::from_boost(startTime), thisLumiSectionInfo)); - if (!m_endFillMode) { - m_timestampToLumiid.insert(std::make_pair(cond::time::from_boost(startTime), lumiid)); - } - LHCInfoPerFill& payload = *thisLumiSectionInfo; - payload.setDelivLumi(delivLumi); - payload.setRecLumi(recLumi); - } - - void convertBufferedIovsToLumiid(std::map timestampToLumiid) { - for (auto& item : m_tmpBuffer) { - // Check if the lumiid IOV corresponding to the timestamp is present in the map - if (timestampToLumiid.find(item.first) == timestampToLumiid.end()) { - throw cms::Exception("LHCInfoPerFillPopConSourceHandler") - << "Can't find corresponding lumiid IOV for timestamp " << item.first << "\n"; - } - // Update the buffer with the lumiid-type IOV - item.first = timestampToLumiid.at(item.first); - } - } - - size_t getLumiData(const cond::OMSService& oms, - unsigned short fillId, - const boost::posix_time::ptime& beginFillTime, - const boost::posix_time::ptime& endFillTime) { - auto query = oms.query("lumisections"); - query->addOutputVars( - {"start_time", "delivered_lumi", "recorded_lumi", "beams_stable", "run_number", "lumisection_number"}); - query->filterEQ("fill_number", fillId); - query->filterGT("start_time", beginFillTime).filterLT("start_time", endFillTime); - query->filterEQ("beams_stable", "true"); - query->limit(kLumisectionsQueryLimit); - if (query->execute()) { - auto queryResult = query->result(); - edm::LogInfo(m_name) << "Found " << queryResult.size() << " lumisections with STABLE BEAM during the fill " - << fillId; - - if (!queryResult.empty()) { - if (m_endFillMode) { - auto firstRow = queryResult.front(); - addPayloadToBuffer(firstRow); - } - - auto lastRow = queryResult.back(); - addPayloadToBuffer(lastRow); - } - } - return 0; - } - - void getDipData(const cond::OMSService& oms, - const boost::posix_time::ptime& beginFillTime, - const boost::posix_time::ptime& endFillTime) { - // unsure how to handle this. - // the old implementation is not helping: apparently it is checking only the bunchconfiguration for the first diptime set of values... - auto query1 = oms.query("diplogger/dip/acc/LHC/RunControl/CirculatingBunchConfig/Beam1"); - query1->filterGT("dip_time", beginFillTime).filterLT("dip_time", endFillTime); - //This query is limited to 100 rows, but currently only one is used - //If all this data is needed and saved properly the limit has to be set: query1->limit(...) - if (query1->execute()) { - auto res = query1->result(); - if (!res.empty()) { - std::bitset bunchConfiguration1(0ULL); - auto row = *res.begin(); - auto vbunchConf1 = row.getArray("value"); - for (auto vb : vbunchConf1) { - if (vb != 0) { - unsigned short slot = (vb - 1) / 10 + 1; - bunchConfiguration1[slot] = true; - } - } - m_fillPayload->setBunchBitsetForBeam1(bunchConfiguration1); - } - } - auto query2 = oms.query("diplogger/dip/acc/LHC/RunControl/CirculatingBunchConfig/Beam2"); - query2->filterGT("dip_time", beginFillTime).filterLT("dip_time", endFillTime); - //This query is limited to 100 rows, but currently only one is used - if (query2->execute()) { - auto res = query2->result(); - if (!res.empty()) { - std::bitset bunchConfiguration2(0ULL); - auto row = *res.begin(); - auto vbunchConf2 = row.getArray("value"); - for (auto vb : vbunchConf2) { - if (vb != 0) { - unsigned short slot = (vb - 1) / 10 + 1; - bunchConfiguration2[slot] = true; - } - } - m_fillPayload->setBunchBitsetForBeam2(bunchConfiguration2); - } - } - - auto query3 = oms.query("diplogger/dip/CMS/LHC/LumiPerBunch"); - query3->filterGT("dip_time", beginFillTime).filterLT("dip_time", endFillTime); - //This query is limited to 100 rows, but currently only one is used - if (query3->execute()) { - auto res = query3->result(); - if (!res.empty()) { - std::vector lumiPerBX; - auto row = *res.begin(); - auto lumiBunchInst = row.getArray("lumi_bunch_inst"); - for (auto lb : lumiBunchInst) { - if (lb != 0.) { - lumiPerBX.push_back(lb); - } - } - m_fillPayload->setLumiPerBX(lumiPerBX); - } - } - } - - bool getCTPPSData(cond::persistency::Session& session, - const boost::posix_time::ptime& beginFillTime, - const boost::posix_time::ptime& endFillTime) { - //run the fifth query against the CTPPS schema - //Initializing the CMS_CTP_CTPPS_COND schema. - coral::ISchema& CTPPS = session.coralSession().schema("CMS_PPS_SPECT_COND"); - //execute query for CTPPS Data - std::unique_ptr CTPPSDataQuery(CTPPS.newQuery()); - //FROM clause - CTPPSDataQuery->addToTableList(std::string("PPS_LHC_MACHINE_PARAMS")); - //SELECT clause - CTPPSDataQuery->addToOutputList(std::string("DIP_UPDATE_TIME")); - CTPPSDataQuery->addToOutputList(std::string("LHC_STATE")); - CTPPSDataQuery->addToOutputList(std::string("LHC_COMMENT")); - if (m_debug) { - CTPPSDataQuery->addToOutputList(std::string("RUN_NUMBER")); - CTPPSDataQuery->addToOutputList(std::string("LUMI_SECTION")); - } - //WHERE CLAUSE - coral::AttributeList CTPPSDataBindVariables; - CTPPSDataBindVariables.extend(std::string("beginFillTime")); - CTPPSDataBindVariables.extend(std::string("endFillTime")); - CTPPSDataBindVariables[std::string("beginFillTime")].data() = coral::TimeStamp(beginFillTime); - CTPPSDataBindVariables[std::string("endFillTime")].data() = coral::TimeStamp(endFillTime); - std::string conditionStr = std::string("DIP_UPDATE_TIME>= :beginFillTime and DIP_UPDATE_TIME< :endFillTime"); - CTPPSDataQuery->setCondition(conditionStr, CTPPSDataBindVariables); - //ORDER BY clause - CTPPSDataQuery->addToOrderList(std::string("DIP_UPDATE_TIME")); - //define query output - coral::AttributeList CTPPSDataOutput; - CTPPSDataOutput.extend(std::string("DIP_UPDATE_TIME")); - CTPPSDataOutput.extend(std::string("LHC_STATE")); - CTPPSDataOutput.extend(std::string("LHC_COMMENT")); - if (m_debug) { - CTPPSDataOutput.extend(std::string("RUN_NUMBER")); - CTPPSDataOutput.extend(std::string("LUMI_SECTION")); - } - CTPPSDataQuery->defineOutput(CTPPSDataOutput); - //execute the query - coral::ICursor& CTPPSDataCursor = CTPPSDataQuery->execute(); - cond::Time_t dipTime = 0; - std::string lhcState = "", lhcComment = "", ctppsStatus = ""; - - //debug informations - unsigned int lumiSection = 0; - cond::Time_t runNumber = 0; - cond::Time_t savedDipTime = 0; - unsigned int savedLumiSection = 0; - cond::Time_t savedRunNumber = 0; - - bool ret = false; - LumiSectionFilter filter(m_tmpBuffer); - while (CTPPSDataCursor.next()) { - if (m_debug) { - std::ostringstream CTPPS; - CTPPSDataCursor.currentRow().toOutputStream(CTPPS); - } - coral::Attribute const& dipTimeAttribute = CTPPSDataCursor.currentRow()[std::string("DIP_UPDATE_TIME")]; - if (!dipTimeAttribute.isNull()) { - dipTime = cond::time::from_boost(dipTimeAttribute.data().time()); - if (filter.process(dipTime)) { - ret = true; - coral::Attribute const& lhcStateAttribute = CTPPSDataCursor.currentRow()[std::string("LHC_STATE")]; - if (!lhcStateAttribute.isNull()) { - lhcState = lhcStateAttribute.data(); - } - coral::Attribute const& lhcCommentAttribute = CTPPSDataCursor.currentRow()[std::string("LHC_COMMENT")]; - if (!lhcCommentAttribute.isNull()) { - lhcComment = lhcCommentAttribute.data(); - } - - if (m_debug) { - coral::Attribute const& runNumberAttribute = CTPPSDataCursor.currentRow()[std::string("RUN_NUMBER")]; - if (!runNumberAttribute.isNull()) { - runNumber = runNumberAttribute.data(); - } - coral::Attribute const& lumiSectionAttribute = CTPPSDataCursor.currentRow()[std::string("LUMI_SECTION")]; - if (!lumiSectionAttribute.isNull()) { - lumiSection = lumiSectionAttribute.data(); - } - } - - for (auto it = filter.current(); it != m_tmpBuffer.end(); it++) { - // set the current values to all of the payloads of the lumi section samples after the current since - LHCInfoPerFill& payload = *(it->second); - payload.setLhcState(lhcState); - payload.setLhcComment(lhcComment); - payload.setCtppsStatus(ctppsStatus); - - if (m_debug) { - savedDipTime = dipTime; - savedLumiSection = lumiSection; - savedRunNumber = runNumber; - } - } - } - } - } - if (m_debug) { - edm::LogInfo(m_name) << "Last assigned: " - << "DipTime: " << savedDipTime << " " - << "LumiSection: " << savedLumiSection << " " - << "RunNumber: " << savedRunNumber; - } - return ret; - } - - bool getEcalData(cond::persistency::Session& session, - const boost::posix_time::ptime& lowerTime, - const boost::posix_time::ptime& upperTime) { - //run the sixth query against the CMS_DCS_ENV_PVSS_COND schema - //Initializing the CMS_DCS_ENV_PVSS_COND schema. - coral::ISchema& ECAL = session.nominalSchema(); - //start the transaction against the fill logging schema - //execute query for ECAL Data - std::unique_ptr ECALDataQuery(ECAL.newQuery()); - //FROM clause - ECALDataQuery->addToTableList(std::string("BEAM_PHASE")); - //SELECT clause - ECALDataQuery->addToOutputList(std::string("CHANGE_DATE")); - ECALDataQuery->addToOutputList(std::string("DIP_value")); - ECALDataQuery->addToOutputList(std::string("element_nr")); - ECALDataQuery->addToOutputList(std::string("VALUE_NUMBER")); - //WHERE CLAUSE - coral::AttributeList ECALDataBindVariables; - ECALDataBindVariables.extend(std::string("lowerTime")); - ECALDataBindVariables.extend(std::string("upperTime")); - ECALDataBindVariables[std::string("lowerTime")].data() = coral::TimeStamp(lowerTime); - ECALDataBindVariables[std::string("upperTime")].data() = coral::TimeStamp(upperTime); - std::string conditionStr = std::string( - "(DIP_value LIKE '%beamPhaseMean%' OR DIP_value LIKE '%cavPhaseMean%') AND CHANGE_DATE >= :lowerTime AND " - "CHANGE_DATE < :upperTime"); - - ECALDataQuery->setCondition(conditionStr, ECALDataBindVariables); - //ORDER BY clause - ECALDataQuery->addToOrderList(std::string("CHANGE_DATE")); - ECALDataQuery->addToOrderList(std::string("DIP_value")); - ECALDataQuery->addToOrderList(std::string("element_nr")); - //define query output - coral::AttributeList ECALDataOutput; - ECALDataOutput.extend(std::string("CHANGE_DATE")); - ECALDataOutput.extend(std::string("DIP_value")); - ECALDataOutput.extend(std::string("element_nr")); - ECALDataOutput.extend(std::string("VALUE_NUMBER")); - //ECALDataQuery->limitReturnedRows( 14256 ); //3564 entries per vector. - ECALDataQuery->defineOutput(ECALDataOutput); - //execute the query - coral::ICursor& ECALDataCursor = ECALDataQuery->execute(); - cond::Time_t changeTime = 0; - cond::Time_t firstTime = 0; - std::string dipVal = ""; - unsigned int elementNr = 0; - float value = 0.; - std::set initializedVectors; - LumiSectionFilter filter(m_tmpBuffer); - bool ret = false; - if (m_prevPayload.get()) { - for (auto& lumiSlot : m_tmpBuffer) { - lumiSlot.second->setBeam1VC(m_prevPayload->beam1VC()); - lumiSlot.second->setBeam2VC(m_prevPayload->beam2VC()); - lumiSlot.second->setBeam1RF(m_prevPayload->beam1RF()); - lumiSlot.second->setBeam2RF(m_prevPayload->beam2RF()); - } - } - std::map iovMap; - if (m_tmpBuffer.empty()) { - return ret; - } - cond::Time_t lowerLumi = m_tmpBuffer.front().first; - while (ECALDataCursor.next()) { - if (m_debug) { - std::ostringstream ECAL; - ECALDataCursor.currentRow().toOutputStream(ECAL); - } - coral::Attribute const& changeDateAttribute = ECALDataCursor.currentRow()[std::string("CHANGE_DATE")]; - if (!changeDateAttribute.isNull()) { - ret = true; - boost::posix_time::ptime chTime = changeDateAttribute.data().time(); - // move the first IOV found to the start of the fill interval selected - if (changeTime == 0) { - firstTime = cond::time::from_boost(chTime); - } - changeTime = cond::time::from_boost(chTime); - cond::Time_t iovTime = changeTime; - if (changeTime == firstTime) - iovTime = lowerLumi; - coral::Attribute const& dipValAttribute = ECALDataCursor.currentRow()[std::string("DIP_value")]; - coral::Attribute const& valueNumberAttribute = ECALDataCursor.currentRow()[std::string("VALUE_NUMBER")]; - coral::Attribute const& elementNrAttribute = ECALDataCursor.currentRow()[std::string("element_nr")]; - if (!dipValAttribute.isNull() and !valueNumberAttribute.isNull()) { - dipVal = dipValAttribute.data(); - elementNr = elementNrAttribute.data(); - value = valueNumberAttribute.data(); - if (std::isnan(value)) - value = 0.; - if (filter.process(iovTime)) { - iovMap.insert(std::make_pair(changeTime, filter.current()->first)); - for (auto it = filter.current(); it != m_tmpBuffer.end(); it++) { - LHCInfoPerFill& payload = *(it->second); - theLHCInfoPerFillImpl::setElementData(it->first, dipVal, elementNr, value, payload, initializedVectors); - } - } - } - } - } - if (m_debug) { - for (auto& im : iovMap) { - edm::LogInfo(m_name) << "Found iov=" << im.first << " (" << cond::time::to_boost(im.first) << " ) moved to " - << im.second << " ( " << cond::time::to_boost(im.second) << " )"; - } - } - return ret; - } - -private: - bool m_debug; - // starting date for sampling - boost::posix_time::ptime m_startTime; - boost::posix_time::ptime m_endTime; - bool m_endFillMode = true; - std::string m_name; - //for reading from relational database source - std::string m_connectionString, m_ecalConnectionString; - std::string m_authpath; - std::string m_omsBaseUrl; - std::unique_ptr m_fillPayload; - std::shared_ptr m_prevPayload; - std::vector>> m_tmpBuffer; - bool m_lastPayloadEmpty = false; - // to hold correspondance between timestamp-type IOVs and lumiid-type IOVs - std::map m_timestampToLumiid; -}; diff --git a/CondTools/RunInfo/plugins/LHCInfoPerLSOnlinePopConAnalyzer.cc b/CondTools/RunInfo/plugins/LHCInfoPerLSOnlinePopConAnalyzer.cc new file mode 100644 index 0000000000000..709362592d160 --- /dev/null +++ b/CondTools/RunInfo/plugins/LHCInfoPerLSOnlinePopConAnalyzer.cc @@ -0,0 +1,7 @@ +#include "CondCore/PopCon/interface/OnlinePopConAnalyzer.h" +#include "CondTools/RunInfo/interface/LHCInfoPerLSPopConSourceHandler.h" +#include "FWCore/Framework/interface/MakerMacros.h" + +using LHCInfoPerLSOnlinePopConAnalyzer = popcon::OnlinePopConAnalyzer; + +DEFINE_FWK_MODULE(LHCInfoPerLSOnlinePopConAnalyzer); diff --git a/CondTools/RunInfo/plugins/LHCInfoPerLSPopConAnalyzer.cc b/CondTools/RunInfo/plugins/LHCInfoPerLSPopConAnalyzer.cc index dc5fe9b298c8e..9a1fcbaa4dfdd 100644 --- a/CondTools/RunInfo/plugins/LHCInfoPerLSPopConAnalyzer.cc +++ b/CondTools/RunInfo/plugins/LHCInfoPerLSPopConAnalyzer.cc @@ -1,589 +1,7 @@ -#include "CondCore/CondDB/interface/ConnectionPool.h" -#include "CondCore/CondDB/interface/Types.h" #include "CondCore/PopCon/interface/PopConAnalyzer.h" -#include "CondCore/PopCon/interface/PopConSourceHandler.h" -#include "CondFormats/Common/interface/TimeConversions.h" -#include "CondFormats/RunInfo/interface/LHCInfoPerLS.h" -#include "CondTools/RunInfo/interface/LHCInfoHelper.h" -#include "CondTools/RunInfo/interface/OMSAccess.h" -#include "CoralBase/Attribute.h" -#include "CoralBase/AttributeList.h" -#include "CoralBase/AttributeSpecification.h" -#include "CoralBase/TimeStamp.h" +#include "CondTools/RunInfo/interface/LHCInfoPerLSPopConSourceHandler.h" #include "FWCore/Framework/interface/MakerMacros.h" -#include "FWCore/MessageLogger/interface/MessageLogger.h" -#include "FWCore/ParameterSet/interface/ParameterSet.h" -#include "FWCore/ParameterSet/interface/ParameterSetfwd.h" -#include "RelationalAccess/ICursor.h" -#include "RelationalAccess/IQuery.h" -#include "RelationalAccess/ISchema.h" -#include "RelationalAccess/ISessionProxy.h" - -using std::make_pair; -using std::pair; - -class LHCInfoPerLSPopConSourceHandler; typedef popcon::PopConAnalyzer LHCInfoPerLSPopConAnalyzer; -//define this as a plug-in -DEFINE_FWK_MODULE(LHCInfoPerLSPopConAnalyzer); - -namespace theLHCInfoPerLSImpl { - bool comparePayloads(const LHCInfoPerLS& rhs, const LHCInfoPerLS& lhs) { - if (rhs.fillNumber() != lhs.fillNumber() || rhs.runNumber() != lhs.runNumber() || - rhs.crossingAngleX() != lhs.crossingAngleX() || rhs.crossingAngleY() != lhs.crossingAngleY() || - rhs.betaStarX() != lhs.betaStarX() || rhs.betaStarY() != lhs.betaStarY()) { - return false; - } - return true; - } - - size_t transferPayloads(const std::vector>>& buffer, - std::map>& iovsToTransfer, - std::shared_ptr& prevPayload, - const std::map, pair>& lsIdMap, - cond::Time_t startStableBeamTime, - cond::Time_t endStableBeamTime) { - int lsMissingInPPS = 0; - int xAngleBothZero = 0, xAngleBothNonZero = 0, xAngleNegative = 0; - int betaNegative = 0; - size_t niovs = 0; - std::stringstream condIovs; - std::stringstream missingLsList; - for (auto& iov : buffer) { - bool add = false; - auto payload = iov.second; - cond::Time_t since = iov.first; - if (iovsToTransfer.empty()) { - add = true; - } else { - LHCInfoPerLS& lastAdded = *iovsToTransfer.rbegin()->second; - if (!comparePayloads(lastAdded, *payload)) { - add = true; - } - } - auto id = make_pair(payload->runNumber(), payload->lumiSection()); - bool stableBeam = since >= startStableBeamTime && since <= endStableBeamTime; - bool isMissing = lsIdMap.find(id) != lsIdMap.end() && id != lsIdMap.at(id); - if (stableBeam && isMissing) { - missingLsList << id.first << "_" << id.second << " "; - lsMissingInPPS += isMissing; - } - if (add && !isMissing) { - niovs++; - if (stableBeam) { - if (payload->crossingAngleX() == 0 && payload->crossingAngleY() == 0) - xAngleBothZero++; - if (payload->crossingAngleX() != 0 && payload->crossingAngleY() != 0) - xAngleBothNonZero++; - if (payload->crossingAngleX() < 0 || payload->crossingAngleY() < 0) - xAngleNegative++; - if (payload->betaStarX() < 0 || payload->betaStarY() < 0) - betaNegative++; - } - - condIovs << since << " "; - iovsToTransfer.insert(make_pair(since, payload)); - prevPayload = iov.second; - } - } - unsigned short fillNumber = (!buffer.empty()) ? buffer.front().second->fillNumber() : 0; - if (lsMissingInPPS > 0) { - edm::LogWarning("transferPayloads") - << "Number of stable beam LS in OMS without corresponding record in PPS DB for fill " << fillNumber << ": " - << lsMissingInPPS; - edm::LogWarning("transferPayloads") - << "Stable beam LS in OMS without corresponding record in PPS DB (run_LS): " << missingLsList.str(); - } - if (xAngleBothZero > 0) { - edm::LogWarning("transferPayloads") - << "Number of payloads written with crossingAngle == 0 for both X and Y for fill " << fillNumber << ": " - << xAngleBothZero; - } - if (xAngleBothNonZero > 0) { - edm::LogWarning("transferPayloads") - << "Number of payloads written with crossingAngle != 0 for both X and Y for fill " << fillNumber << ": " - << xAngleBothNonZero; - } - if (xAngleNegative > 0) { - edm::LogWarning("transferPayloads") - << "Number of payloads written with negative crossingAngle for fill " << fillNumber << ": " << xAngleNegative; - } - if (betaNegative > 0) { - edm::LogWarning("transferPayloads") - << "Number of payloads written with negative betaSta for fill " << fillNumber << ": " << betaNegative; - } - - edm::LogInfo("transferPayloads") << "TRANSFERED COND IOVS: " << condIovs.str(); - return niovs; - } - -} // namespace theLHCInfoPerLSImpl - -class LHCInfoPerLSPopConSourceHandler : public popcon::PopConSourceHandler { -public: - LHCInfoPerLSPopConSourceHandler(edm::ParameterSet const& pset) - : m_debug(pset.getUntrackedParameter("debug", false)), - m_startTime(), - m_endTime(), - m_endFillMode(pset.getUntrackedParameter("endFill", true)), - m_name(pset.getUntrackedParameter("name", "LHCInfoPerLSPopConSourceHandler")), - m_connectionString(pset.getUntrackedParameter("connectionString", "")), - m_authpath(pset.getUntrackedParameter("authenticationPath", "")), - m_omsBaseUrl(pset.getUntrackedParameter("omsBaseUrl", "")), - m_fillPayload(), - m_prevPayload(), - m_tmpBuffer() { - if (!pset.getUntrackedParameter("startTime").empty()) { - m_startTime = boost::posix_time::time_from_string(pset.getUntrackedParameter("startTime")); - } - boost::posix_time::ptime now = boost::posix_time::second_clock::local_time(); - m_endTime = now; - if (!pset.getUntrackedParameter("endTime").empty()) { - m_endTime = boost::posix_time::time_from_string(pset.getUntrackedParameter("endTime")); - if (m_endTime > now) - m_endTime = now; - } - } - - ~LHCInfoPerLSPopConSourceHandler() override = default; - - void getNewObjects() override { - //if a new tag is created, transfer fake fill from 1 to the first fill for the first time - if (tagInfo().size == 0) { - edm::LogInfo(m_name) << "New tag " << tagInfo().name << "; from " << m_name << "::getNewObjects"; - } else { - //check what is already inside the database - edm::LogInfo(m_name) << "got info for tag " << tagInfo().name << ": size " << tagInfo().size - << ", last object valid since " << tagInfo().lastInterval.since << " ( " - << boost::posix_time::to_iso_extended_string( - cond::time::to_boost(tagInfo().lastInterval.since)) - << " ); from " << m_name << "::getNewObjects"; - } - - cond::Time_t lastSince = tagInfo().lastInterval.since; - if (tagInfo().isEmpty()) { - // for a new or empty tag, an empty payload should be added on top with since=1 - if (m_endFillMode) { - addEmptyPayload(1); - lastSince = 1; - } else { - addEmptyPayload(cond::time::lumiTime(1, 1)); - lastSince = cond::time::lumiTime(1, 1); - } - } else { - edm::LogInfo(m_name) << "The last Iov in tag " << tagInfo().name << " valid since " << lastSince << "from " - << m_name << "::getNewObjects"; - } - - //retrieve the data from the relational database source - cond::persistency::ConnectionPool connection; - //configure the connection - if (m_debug) { - connection.setMessageVerbosity(coral::Debug); - } else { - connection.setMessageVerbosity(coral::Error); - } - connection.setAuthenticationPath(m_authpath); - connection.configure(); - //create the sessions - cond::persistency::Session session = connection.createSession(m_connectionString, false); - // fetch last payload when available - if (!tagInfo().lastInterval.payloadId.empty()) { - cond::persistency::Session session3 = dbSession(); - session3.transaction().start(true); - m_prevPayload = session3.fetchPayload(tagInfo().lastInterval.payloadId); - session3.transaction().commit(); - - // find startFillTime and endFillTime of the most recent fill already saved in the tag - if (m_prevPayload->fillNumber() != 0) { - cond::OMSService oms; - oms.connect(m_omsBaseUrl); - auto query = oms.query("fills"); - query->addOutputVar("end_time"); - query->addOutputVar("start_time"); - query->filterEQ("fill_number", m_prevPayload->fillNumber()); - bool foundFill = query->execute(); - if (foundFill) { - auto result = query->result(); - - if (!result.empty()) { - std::string endTimeStr = (*result.begin()).get("end_time"); - m_prevEndFillTime = - (endTimeStr == "null") - ? 0 - : cond::time::from_boost((*result.begin()).get("end_time")); - auto startFillTime = (*result.begin()).get("start_time"); - m_prevStartFillTime = cond::time::from_boost(startFillTime); - } else { - foundFill = false; - } - } - if (!foundFill) { - edm::LogError(m_name) << "Could not find end time of fill #" << m_prevPayload->fillNumber(); - } - } else { - m_prevEndFillTime = 0; - m_prevStartFillTime = 0; - } - } - - boost::posix_time::ptime executionTime = boost::posix_time::second_clock::local_time(); - cond::Time_t executionTimeIov = cond::time::from_boost(executionTime); - - cond::Time_t startTimestamp = m_startTime.is_not_a_date_time() ? 0 : cond::time::from_boost(m_startTime); - cond::Time_t nextFillSearchTimestamp = std::max(startTimestamp, m_endFillMode ? lastSince : m_prevEndFillTime); - - edm::LogInfo(m_name) << "Starting sampling at " - << boost::posix_time::to_simple_string(cond::time::to_boost(nextFillSearchTimestamp)); - - while (true) { - if (nextFillSearchTimestamp >= executionTimeIov) { - edm::LogInfo(m_name) << "Sampling ended at the time " - << boost::posix_time::to_simple_string(cond::time::to_boost(executionTimeIov)); - break; - } - boost::posix_time::ptime nextFillSearchTime = cond::time::to_boost(nextFillSearchTimestamp); - boost::posix_time::ptime startSampleTime; - boost::posix_time::ptime endSampleTime; - - cond::OMSService oms; - oms.connect(m_omsBaseUrl); - auto query = oms.query("fills"); - - if (!m_endFillMode and m_prevPayload->fillNumber() and m_prevEndFillTime == 0ULL) { - // continue processing unfinished fill with some payloads already in the tag - edm::LogInfo(m_name) << "Searching started fill #" << m_prevPayload->fillNumber(); - query->filterEQ("fill_number", m_prevPayload->fillNumber()); - bool foundFill = query->execute(); - if (foundFill) - foundFill = makeFillPayload(m_fillPayload, query->result()); - if (!foundFill) { - edm::LogError(m_name) << "Could not find fill #" << m_prevPayload->fillNumber(); - break; - } - startSampleTime = cond::time::to_boost(lastSince); - } else { - edm::LogInfo(m_name) << "Searching new fill after " << boost::posix_time::to_simple_string(nextFillSearchTime); - query->filterNotNull("start_stable_beam").filterNotNull("fill_number"); - if (nextFillSearchTime > cond::time::to_boost(m_prevStartFillTime)) { - query->filterGE("start_time", nextFillSearchTime); - } else { - query->filterGT("start_time", nextFillSearchTime); - } - - query->filterLT("start_time", m_endTime); - if (m_endFillMode) - query->filterNotNull("end_time"); - bool foundFill = query->execute(); - if (foundFill) - foundFill = makeFillPayload(m_fillPayload, query->result()); - if (!foundFill) { - edm::LogInfo(m_name) << "No fill found - END of job."; - break; - } - startSampleTime = cond::time::to_boost(m_startFillTime); - } - - unsigned short lhcFill = m_fillPayload->fillNumber(); - bool ongoingFill = m_endFillTime == 0ULL; - if (ongoingFill) { - edm::LogInfo(m_name) << "Found ongoing fill " << lhcFill << " created at " - << cond::time::to_boost(m_startFillTime); - endSampleTime = executionTime; - nextFillSearchTimestamp = executionTimeIov; - } else { - edm::LogInfo(m_name) << "Found fill " << lhcFill << " created at " << cond::time::to_boost(m_startFillTime) - << " ending at " << cond::time::to_boost(m_endFillTime); - endSampleTime = cond::time::to_boost(m_endFillTime); - nextFillSearchTimestamp = m_endFillTime; - } - - if (m_endFillMode || ongoingFill) { - getLumiData(oms, lhcFill, startSampleTime, endSampleTime); - - if (!m_tmpBuffer.empty()) { - boost::posix_time::ptime flumiStart = cond::time::to_boost(m_tmpBuffer.front().first); - boost::posix_time::ptime flumiStop = cond::time::to_boost(m_tmpBuffer.back().first); - edm::LogInfo(m_name) << "First buffered lumi starts at " << flumiStart << " last lumi starts at " - << flumiStop; - session.transaction().start(true); - getCTPPSData(session, startSampleTime, endSampleTime); - session.transaction().commit(); - } - } - - size_t niovs = theLHCInfoPerLSImpl::transferPayloads( - m_tmpBuffer, m_iovs, m_prevPayload, m_lsIdMap, m_startStableBeamTime, m_endStableBeamTime); - edm::LogInfo(m_name) << "Added " << niovs << " iovs within the Fill time"; - if (niovs) { - m_prevEndFillTime = m_endFillTime; - m_prevStartFillTime = m_startFillTime; - } - m_tmpBuffer.clear(); - m_lsIdMap.clear(); - if (m_prevPayload->fillNumber() and !ongoingFill) { - if (m_endFillMode) { - addEmptyPayload(m_endFillTime); - } else { - addEmptyPayload(cond::lhcInfoHelper::getFillLastLumiIOV(oms, lhcFill)); - } - } - } - } - std::string id() const override { return m_name; } - - static constexpr unsigned int kLumisectionsQueryLimit = 4000; - -private: - void addEmptyPayload(cond::Time_t iov) { - bool add = false; - if (m_iovs.empty()) { - if (!m_lastPayloadEmpty) - add = true; - } else { - auto lastAdded = m_iovs.rbegin()->second; - if (lastAdded->fillNumber() != 0) { - add = true; - } - } - if (add) { - auto newPayload = std::make_shared(); - m_iovs.insert(make_pair(iov, newPayload)); - m_prevPayload = newPayload; - m_prevEndFillTime = 0; - m_prevStartFillTime = 0; - edm::LogInfo(m_name) << "Added empty payload with IOV" << iov << " ( " - << boost::posix_time::to_iso_extended_string(cond::time::to_boost(iov)) << " )"; - } - } - - bool makeFillPayload(std::unique_ptr& targetPayload, const cond::OMSServiceResult& queryResult) { - bool ret = false; - if (!queryResult.empty()) { - auto row = *queryResult.begin(); - auto currentFill = row.get("fill_number"); - m_startFillTime = cond::time::from_boost(row.get("start_time")); - std::string endTimeStr = row.get("end_time"); - m_endFillTime = - (endTimeStr == "null") ? 0 : cond::time::from_boost(row.get("end_time")); - m_startStableBeamTime = cond::time::from_boost(row.get("start_stable_beam")); - m_endStableBeamTime = cond::time::from_boost(row.get("end_stable_beam")); - targetPayload = std::make_unique(); - targetPayload->setFillNumber(currentFill); - ret = true; - } - return ret; - } - - void addPayloadToBuffer(cond::OMSServiceResultRef& row) { - auto lumiTime = row.get("start_time"); - LHCInfoPerLS* thisLumiSectionInfo = new LHCInfoPerLS(*m_fillPayload); - thisLumiSectionInfo->setLumiSection(std::stoul(row.get("lumisection_number"))); - thisLumiSectionInfo->setRunNumber(std::stoul(row.get("run_number"))); - m_lsIdMap[make_pair(thisLumiSectionInfo->runNumber(), thisLumiSectionInfo->lumiSection())] = make_pair(-1, -1); - if (m_endFillMode) { - m_tmpBuffer.emplace_back(make_pair(cond::time::from_boost(lumiTime), thisLumiSectionInfo)); - } else { - m_tmpBuffer.emplace_back( - make_pair(cond::time::lumiTime(thisLumiSectionInfo->runNumber(), thisLumiSectionInfo->lumiSection()), - thisLumiSectionInfo)); - } - } - - size_t bufferAllLS(const cond::OMSServiceResult& queryResult) { - for (auto r : queryResult) { - addPayloadToBuffer(r); - } - return queryResult.size(); - } - - size_t bufferFirstStableBeamLS(const cond::OMSServiceResult& queryResult) { - for (auto r : queryResult) { - if (r.get("beams_stable") == "true") { - addPayloadToBuffer(r); - edm::LogInfo(m_name) << "Buffered first lumisection of stable beam: LS: " - << r.get("lumisection_number") - << " run: " << r.get("run_number"); - return 1; - } - } - return 0; - } - - size_t getLumiData(const cond::OMSService& oms, - unsigned short fillId, - const boost::posix_time::ptime& beginFillTime, - const boost::posix_time::ptime& endFillTime) { - auto query = oms.query("lumisections"); - query->addOutputVars({"start_time", "run_number", "beams_stable", "lumisection_number"}); - query->filterEQ("fill_number", fillId); - query->filterGT("start_time", beginFillTime).filterLT("start_time", endFillTime); - query->limit(kLumisectionsQueryLimit); - size_t nlumi = 0; - if (query->execute()) { - auto queryResult = query->result(); - if (m_endFillMode) { - nlumi = bufferAllLS(queryResult); - } else if (!queryResult.empty()) { - auto newestPayload = queryResult.back(); - if (newestPayload.get("beams_stable") == "true") { - addPayloadToBuffer(newestPayload); - nlumi = 1; - edm::LogInfo(m_name) << "Buffered most recent lumisection:" - << " LS: " << newestPayload.get("lumisection_number") - << " run: " << newestPayload.get("run_number"); - } - } - edm::LogInfo(m_name) << "Found " << queryResult.size() << " lumisections during the fill " << fillId; - } else { - edm::LogInfo(m_name) << "OMS query for lumisections of fill " << fillId << "failed, status:" << query->status(); - } - return nlumi; - } - - bool getCTPPSData(cond::persistency::Session& session, - const boost::posix_time::ptime& beginFillTime, - const boost::posix_time::ptime& endFillTime) { - //run the fifth query against the CTPPS schema - //Initializing the CMS_CTP_CTPPS_COND schema. - coral::ISchema& CTPPS = session.coralSession().schema("CMS_PPS_SPECT_COND"); - //execute query for CTPPS Data - std::unique_ptr CTPPSDataQuery(CTPPS.newQuery()); - //FROM clause - CTPPSDataQuery->addToTableList(std::string("PPS_LHC_MACHINE_PARAMS")); - //SELECT clause - CTPPSDataQuery->addToOutputList(std::string("DIP_UPDATE_TIME")); - CTPPSDataQuery->addToOutputList(std::string("LUMI_SECTION")); - CTPPSDataQuery->addToOutputList(std::string("RUN_NUMBER")); - CTPPSDataQuery->addToOutputList(std::string("FILL_NUMBER")); - CTPPSDataQuery->addToOutputList(std::string("XING_ANGLE_P5_X_URAD")); - CTPPSDataQuery->addToOutputList(std::string("XING_ANGLE_P5_Y_URAD")); - CTPPSDataQuery->addToOutputList(std::string("BETA_STAR_P5_X_M")); - CTPPSDataQuery->addToOutputList(std::string("BETA_STAR_P5_Y_M")); - //WHERE CLAUSE - coral::AttributeList CTPPSDataBindVariables; - CTPPSDataBindVariables.extend(std::string("beginFillTime")); - CTPPSDataBindVariables.extend(std::string("endFillTime")); - CTPPSDataBindVariables[std::string("beginFillTime")].data() = coral::TimeStamp(beginFillTime); - CTPPSDataBindVariables[std::string("endFillTime")].data() = coral::TimeStamp(endFillTime); - std::string conditionStr = std::string("DIP_UPDATE_TIME>= :beginFillTime and DIP_UPDATE_TIME< :endFillTime"); - CTPPSDataQuery->setCondition(conditionStr, CTPPSDataBindVariables); - //ORDER BY clause - CTPPSDataQuery->addToOrderList(std::string("DIP_UPDATE_TIME")); - //define query output - coral::AttributeList CTPPSDataOutput; - CTPPSDataOutput.extend(std::string("DIP_UPDATE_TIME")); - CTPPSDataOutput.extend(std::string("LUMI_SECTION")); - CTPPSDataOutput.extend(std::string("RUN_NUMBER")); - CTPPSDataOutput.extend(std::string("FILL_NUMBER")); - CTPPSDataOutput.extend(std::string("XING_ANGLE_P5_X_URAD")); - CTPPSDataOutput.extend(std::string("XING_ANGLE_P5_Y_URAD")); - CTPPSDataOutput.extend(std::string("BETA_STAR_P5_X_M")); - CTPPSDataOutput.extend(std::string("BETA_STAR_P5_Y_M")); - CTPPSDataQuery->defineOutput(CTPPSDataOutput); - //execute the query - coral::ICursor& CTPPSDataCursor = CTPPSDataQuery->execute(); - unsigned int lumiSection = 0; - cond::Time_t runNumber = 0; - int fillNumber = 0; - float crossingAngleX = 0., betaStarX = 0.; - float crossingAngleY = 0., betaStarY = 0.; - - bool ret = false; - int wrongFillNumbers = 0; - std::stringstream wrongFills; - std::vector>>::iterator current = m_tmpBuffer.begin(); - while (CTPPSDataCursor.next()) { - if (m_debug) { - std::ostringstream CTPPS; - CTPPSDataCursor.currentRow().toOutputStream(CTPPS); - } - coral::Attribute const& dipTimeAttribute = CTPPSDataCursor.currentRow()[std::string("DIP_UPDATE_TIME")]; - if (!dipTimeAttribute.isNull()) { - ret = true; - coral::Attribute const& lumiSectionAttribute = CTPPSDataCursor.currentRow()[std::string("LUMI_SECTION")]; - if (!lumiSectionAttribute.isNull()) { - lumiSection = lumiSectionAttribute.data(); - } - coral::Attribute const& runNumberAttribute = CTPPSDataCursor.currentRow()[std::string("RUN_NUMBER")]; - if (!runNumberAttribute.isNull()) { - runNumber = runNumberAttribute.data(); - } - coral::Attribute const& fillNumberAttribute = CTPPSDataCursor.currentRow()[std::string("FILL_NUMBER")]; - if (!fillNumberAttribute.isNull()) { - fillNumber = fillNumberAttribute.data(); - } - coral::Attribute const& crossingAngleXAttribute = - CTPPSDataCursor.currentRow()[std::string("XING_ANGLE_P5_X_URAD")]; - if (!crossingAngleXAttribute.isNull()) { - crossingAngleX = crossingAngleXAttribute.data(); - } - coral::Attribute const& crossingAngleYAttribute = - CTPPSDataCursor.currentRow()[std::string("XING_ANGLE_P5_Y_URAD")]; - if (!crossingAngleYAttribute.isNull()) { - crossingAngleY = crossingAngleYAttribute.data(); - } - coral::Attribute const& betaStarXAttribute = CTPPSDataCursor.currentRow()[std::string("BETA_STAR_P5_X_M")]; - if (!betaStarXAttribute.isNull()) { - betaStarX = betaStarXAttribute.data(); - } - coral::Attribute const& betaStarYAttribute = CTPPSDataCursor.currentRow()[std::string("BETA_STAR_P5_Y_M")]; - if (!betaStarYAttribute.isNull()) { - betaStarY = betaStarYAttribute.data(); - } - if (current != m_tmpBuffer.end() && current->second->fillNumber() != fillNumber) { - wrongFills << "( " << runNumber << "_" << lumiSection << " fill: OMS: " << current->second->fillNumber() - << " PPSdb: " << fillNumber << " ) "; - wrongFillNumbers++; - } - for (; - current != m_tmpBuffer.end() && make_pair(current->second->runNumber(), current->second->lumiSection()) <= - make_pair(runNumber, lumiSection); - current++) { - LHCInfoPerLS& payload = *(current->second); - payload.setCrossingAngleX(crossingAngleX); - payload.setCrossingAngleY(crossingAngleY); - payload.setBetaStarX(betaStarX); - payload.setBetaStarY(betaStarY); - payload.setLumiSection(lumiSection); - payload.setRunNumber(runNumber); - if (m_lsIdMap.find(make_pair(payload.runNumber(), payload.lumiSection())) != m_lsIdMap.end()) { - m_lsIdMap[make_pair(payload.runNumber(), payload.lumiSection())] = make_pair(runNumber, lumiSection); - } - } - } - } - if (wrongFillNumbers) { - edm::LogWarning("getCTPPSData") << "Number of records from PPS DB with fillNumber different from OMS: " - << wrongFillNumbers; - edm::LogWarning("getCTPPSData") << "Records from PPS DB with fillNumber different from OMS: " << wrongFills.str(); - } - return ret; - } -private: - bool m_debug; - // starting date for sampling - boost::posix_time::ptime m_startTime; - boost::posix_time::ptime m_endTime; - bool m_endFillMode = true; - std::string m_name; - //for reading from relational database source - std::string m_connectionString; - std::string m_authpath; - std::string m_omsBaseUrl; - std::unique_ptr m_fillPayload; - std::shared_ptr m_prevPayload; - cond::Time_t m_startFillTime; - cond::Time_t m_endFillTime; - cond::Time_t m_prevEndFillTime; - cond::Time_t m_prevStartFillTime; - cond::Time_t m_startStableBeamTime; - cond::Time_t m_endStableBeamTime; - std::vector>> m_tmpBuffer; - bool m_lastPayloadEmpty = false; - //mapping of lumisections IDs (pairs of runnumber an LS number) found in OMS to the IDs they've been assignd from PPS DB - //value pair(-1, -1) means lumisection corresponding to the key exists in OMS but no lumisection was matched from PPS - std::map, pair> m_lsIdMap; -}; +DEFINE_FWK_MODULE(LHCInfoPerLSPopConAnalyzer); \ No newline at end of file diff --git a/CondTools/RunInfo/python/LHCInfoPerFillPopConAnalyzer_cfg.py b/CondTools/RunInfo/python/LHCInfoPerFillPopConAnalyzer_cfg.py index d71362ecebfa8..689cf418a245a 100644 --- a/CondTools/RunInfo/python/LHCInfoPerFillPopConAnalyzer_cfg.py +++ b/CondTools/RunInfo/python/LHCInfoPerFillPopConAnalyzer_cfg.py @@ -3,10 +3,6 @@ import FWCore.ParameterSet.VarParsing as VarParsing process = cms.Process("LHCInfoPerFillPopulator") from CondCore.CondDB.CondDB_cfi import * -#process.load("CondCore.DBCommon.CondDBCommon_cfi") -#process.CondDBCommon.connect = 'sqlite_file:lhcinfoperls_pop_test.db' -#process.CondDBCommon.DBParameters.authenticationPath = '.' -#process.CondDBCommon.DBParameters.messageLevel=cms.untracked.int32(1) sourceConnection = 'oracle://cms_omds_adg/CMS_RUNINFO_R' if socket.getfqdn().find('.cms') != -1: @@ -60,6 +56,32 @@ processes only fills starting before endTime; default to empty string which sets no restriction""" ) + +#duringFill mode specific: +options.register( 'lastLumiFile' + , '' + , VarParsing.VarParsing.multiplicity.singleton + , VarParsing.VarParsing.varType.string + , """duringFill only: path to file with lumiid to override the last lumisection processed by HLT. + Used for testing. Leave empty for production behaviour (getting this info from OMS)""" + ) +options.register( 'frontierKey' + , '' + , VarParsing.VarParsing.multiplicity.singleton + , VarParsing.VarParsing.varType.string + , """duringFill only: run-unique key for writing with OnlinePopCon + (used for confirming proper upload)""" + ) + + +# so far there was no need to use option, added just in case +options.register( 'authenticationPath' + , "" + , VarParsing.VarParsing.multiplicity.singleton + , VarParsing.VarParsing.varType.string + , """for now this option was always left empty""" + ) + options.parseArguments() if options.mode is None: raise ValueError("mode argument not provided. Supported modes are: duringFill endFill") @@ -68,6 +90,7 @@ CondDBConnection = CondDB.clone( connect = cms.string( options.destinationConnection ) ) CondDBConnection.DBParameters.messageLevel = cms.untracked.int32( options.messageLevel ) +CondDBConnection.DBParameters.authenticationPath = cms.untracked.string(options.authenticationPath) process.MessageLogger = cms.Service("MessageLogger", cout = cms.untracked.PSet(threshold = cms.untracked.string('INFO')), @@ -87,30 +110,52 @@ else: timetype = 'lumiid' -process.PoolDBOutputService = cms.Service("PoolDBOutputService", - CondDBConnection, - timetype = cms.untracked.string(timetype), - toPut = cms.VPSet(cms.PSet(record = cms.string('LHCInfoPerFillRcd'), - tag = cms.string( options.tag ) - ) - ) - ) +if options.mode == "endFill": + process.PoolDBOutputService = cms.Service("PoolDBOutputService", + CondDBConnection, + timetype = cms.untracked.string(timetype), + toPut = cms.VPSet(cms.PSet(record = cms.string('LHCInfoPerFillRcd'), + tag = cms.string( options.tag ) + ) + ) + ) +else: + process.OnlineDBOutputService = cms.Service("OnlineDBOutputService", + CondDBConnection, + preLoadConnectionString = cms.untracked.string('frontier://FrontierProd/CMS_CONDITIONS' + if not options.destinationConnection.startswith('sqlite') + else options.destinationConnection ), + lastLumiFile = cms.untracked.string(options.lastLumiFile), + omsServiceUrl = cms.untracked.string('http://cmsoms-eventing.cms:9949/urn:xdaq-application:lid=100/getRunAndLumiSection' + if not options.lastLumiFile else "" ), + # runNumber = cms.untracked.uint64(384468), #not used in production, the last LS processed is set as the 1st LS of this + #run if the omsServiceUrl is empty and file specified in lastLumiFile is empty + latency = cms.untracked.uint32(2), + timetype = cms.untracked.string(timetype), + toPut = cms.VPSet(cms.PSet( + record = cms.string('LHCInfoPerFillRcd'), + tag = cms.string( options.tag ), + onlyAppendUpdatePolicy = cms.untracked.bool(True) + )), + frontierKey = cms.untracked.string('wrong-key') +) + -process.Test1 = cms.EDAnalyzer("LHCInfoPerFillPopConAnalyzer", +process.Test1 = cms.EDAnalyzer("LHCInfoPerFillPopConAnalyzer" if options.mode == "endFill" else "LHCInfoPerFillOnlinePopConAnalyzer", SinceAppendMode = cms.bool(True), record = cms.string('LHCInfoPerFillRcd'), name = cms.untracked.string('LHCInfo'), - Source = cms.PSet(fill = cms.untracked.uint32(6417), + Source = cms.PSet( startTime = cms.untracked.string(options.startTime), endTime = cms.untracked.string(options.endTime), - endFill = cms.untracked.bool(True if options.mode == "endFill" else False), + endFill = cms.untracked.bool(options.mode == "endFill"), name = cms.untracked.string("LHCInfoPerFillPopConSourceHandler"), connectionString = cms.untracked.string("oracle://cms_orcon_adg/CMS_RUNTIME_LOGGER"), ecalConnectionString = cms.untracked.string("oracle://cms_orcon_adg/CMS_DCS_ENV_PVSS_COND"), omsBaseUrl = cms.untracked.string("http://vocms0184.cern.ch/agg/api/v1"), - authenticationPath = cms.untracked.string(""), + authenticationPath = cms.untracked.string(options.authenticationPath), debug=cms.untracked.bool(False) - ), + ), loggingOn = cms.untracked.bool(True), IsDestDbCheckedInQueryLog = cms.untracked.bool(False) ) diff --git a/CondTools/RunInfo/python/LHCInfoPerLSPopConAnalyzer_cfg.py b/CondTools/RunInfo/python/LHCInfoPerLSPopConAnalyzer_cfg.py index 603c7f514d4ee..290d7479b1052 100644 --- a/CondTools/RunInfo/python/LHCInfoPerLSPopConAnalyzer_cfg.py +++ b/CondTools/RunInfo/python/LHCInfoPerLSPopConAnalyzer_cfg.py @@ -3,10 +3,6 @@ import FWCore.ParameterSet.VarParsing as VarParsing process = cms.Process("LHCInfoPerLSPopulator") from CondCore.CondDB.CondDB_cfi import * -#process.load("CondCore.DBCommon.CondDBCommon_cfi") -#process.CondDBCommon.connect = 'sqlite_file:lhcinfoperls_pop_test.db' -#process.CondDBCommon.DBParameters.authenticationPath = '.' -#process.CondDBCommon.DBParameters.messageLevel=cms.untracked.int32(1) sourceConnection = 'oracle://cms_omds_adg/CMS_RUNINFO_R' if socket.getfqdn().find('.cms') != -1: @@ -60,6 +56,95 @@ processes only fills starting before endTime; default to empty string which sets no restriction""" ) + +#duringFill mode specific: +options.register( 'lastLumiFile' + , '' + , VarParsing.VarParsing.multiplicity.singleton + , VarParsing.VarParsing.varType.string + , """duringFill only: path to file with lumiid to override the last lumisection processed by HLT. + Used for testing. Leave empty for production behaviour (getting this info from OMS)""" + ) +options.register( 'frontierKey' + , '' + , VarParsing.VarParsing.multiplicity.singleton + , VarParsing.VarParsing.varType.string + , """duringFill only: run-unique key for writing with OnlinePopCon + (used for confirming proper upload)""" + ) +options.register('offsetLS' + , 2 + , VarParsing.VarParsing.multiplicity.singleton + , VarParsing.VarParsing.varType.int + , """duringFill only: offset between lastLumi (last LS processed by HLT or overriden by lastLumiFile) + and the IOV of the payload to be uploaded""" + ) +options.register( 'debugLogic' + , False + , VarParsing.VarParsing.multiplicity.singleton + , VarParsing.VarParsing.varType.bool + , """duringFill only: Enables debug logic, meant to be used only for tests""" + ) +options.register( 'defaultXangleX' + , 160.0 # urad + , VarParsing.VarParsing.multiplicity.singleton + , VarParsing.VarParsing.varType.float + , """duringFill only: crossingAngleX value (in urad) for the default payload. + The default payload is inserted after the last processed fill has ended + and there's no ongoing stable beam yet. """ + ) +options.register( 'defaultXangleY' + , 0.0 # urad + , VarParsing.VarParsing.multiplicity.singleton + , VarParsing.VarParsing.varType.float + , """duringFill only: crossingAngleY value (in urad) for the default payload. + The default payload is inserted after the last processed fill has ended + and there's no ongoing stable beam yet. """ + ) +options.register( 'defaultBetaX' + , 0.3 # meters + , VarParsing.VarParsing.multiplicity.singleton + , VarParsing.VarParsing.varType.float + , """duringFill only: betaStarX value (in meters) for the default payload. + The default payload is inserted after the last processed fill has ended + and there's no ongoing stable beam yet. """ + ) +options.register( 'defaultBetaY' + , 0.3 # meters + , VarParsing.VarParsing.multiplicity.singleton + , VarParsing.VarParsing.varType.float + , """duringFill only: betaStarY value (in meters) for the default payload. + The default payload is inserted after the last processed fill has ended + and there's no ongoing stable beam yet. """ + ) + + +# it's unlikely to ever use values different from the defaults, added as a parameter just in case +options.register('minBetaStar', 0.1 + , VarParsing.VarParsing.multiplicity.singleton, VarParsing.VarParsing.varType.float + , """duringFill only: [meters] min value of the range of valid values. + If the value is outside of this range the payload is not uploaded""") +options.register('maxBetaStar', 100. + , VarParsing.VarParsing.multiplicity.singleton, VarParsing.VarParsing.varType.float + , """duringFill only: [meters] min value of the range of valid values. + If the value is outside of this range the payload is not uploaded""") +options.register('minCrossingAngle', 10. + , VarParsing.VarParsing.multiplicity.singleton, VarParsing.VarParsing.varType.float + , """duringFill only: [urad] min value of the range of valid values. + If the value is outside of this range the payload is not uploaded""") +options.register('maxCrossingAngle', 500. + , VarParsing.VarParsing.multiplicity.singleton, VarParsing.VarParsing.varType.float + , """duringFill only: [urad] min value of the range of valid values. + If the value is outside of this range the payload is not uploaded""") + +# as the previous options, so far there was no need to use option, added just in case +options.register( 'authenticationPath' + , "" + , VarParsing.VarParsing.multiplicity.singleton + , VarParsing.VarParsing.varType.string + , """for now this option was always left empty""" + ) + options.parseArguments() if options.mode is None: raise ValueError("mode argument not provided. Supported modes are: duringFill endFill") @@ -68,6 +153,7 @@ CondDBConnection = CondDB.clone( connect = cms.string( options.destinationConnection ) ) CondDBConnection.DBParameters.messageLevel = cms.untracked.int32( options.messageLevel ) +CondDBConnection.DBParameters.authenticationPath = cms.untracked.string(options.authenticationPath) process.MessageLogger = cms.Service("MessageLogger", cout = cms.untracked.PSet(threshold = cms.untracked.string('INFO')), @@ -87,31 +173,63 @@ else: timetype = 'lumiid' -process.PoolDBOutputService = cms.Service("PoolDBOutputService", - CondDBConnection, - timetype = cms.untracked.string(timetype), - toPut = cms.VPSet(cms.PSet(record = cms.string('LHCInfoPerLSRcd'), - tag = cms.string( options.tag ) - ) - ) - ) +if options.mode == "endFill": + process.PoolDBOutputService = cms.Service("PoolDBOutputService", + CondDBConnection, + timetype = cms.untracked.string(timetype), + toPut = cms.VPSet(cms.PSet(record = cms.string('LHCInfoPerLSRcd'), + tag = cms.string( options.tag ) + ) + ) + ) +else: + process.OnlineDBOutputService = cms.Service("OnlineDBOutputService", + CondDBConnection, + preLoadConnectionString = cms.untracked.string('frontier://FrontierProd/CMS_CONDITIONS' + if not options.destinationConnection.startswith('sqlite') + else options.destinationConnection ), + lastLumiFile = cms.untracked.string(options.lastLumiFile), + omsServiceUrl = cms.untracked.string('http://cmsoms-eventing.cms:9949/urn:xdaq-application:lid=100/getRunAndLumiSection' + if not options.lastLumiFile else "" ), + # runNumber = cms.untracked.uint64(384468), #not used in production, the last LS processed is set as the 1st LS of this + #run if the omsServiceUrl is empty and file specified in lastLumiFile is empty + latency = cms.untracked.uint32(options.offsetLS), + timetype = cms.untracked.string(timetype), + toPut = cms.VPSet(cms.PSet( + record = cms.string('LHCInfoPerLSRcd'), + tag = cms.string( options.tag ), + onlyAppendUpdatePolicy = cms.untracked.bool(True) + )), + frontierKey = cms.untracked.string(options.frontierKey) +) + -process.Test1 = cms.EDAnalyzer("LHCInfoPerLSPopConAnalyzer", +process.Test1 = cms.EDAnalyzer(("LHCInfoPerLSPopConAnalyzer" if options.mode == "endFill" + else "LHCInfoPerLSOnlinePopConAnalyzer"), SinceAppendMode = cms.bool(True), record = cms.string('LHCInfoPerLSRcd'), name = cms.untracked.string('LHCInfo'), - Source = cms.PSet(fill = cms.untracked.uint32(6417), + Source = cms.PSet( startTime = cms.untracked.string(options.startTime), endTime = cms.untracked.string(options.endTime), - endFill = cms.untracked.bool(True if options.mode == "endFill" else False), + endFill = cms.untracked.bool(options.mode == "endFill"), name = cms.untracked.string("LHCInfoPerLSPopConSourceHandler"), connectionString = cms.untracked.string("oracle://cms_orcon_adg/CMS_RUNTIME_LOGGER"), omsBaseUrl = cms.untracked.string("http://vocms0184.cern.ch/agg/api/v1"), - authenticationPath = cms.untracked.string(""), - debug=cms.untracked.bool(False) - ), + authenticationPath = cms.untracked.string(options.authenticationPath), + debug=cms.untracked.bool(False), # Additional logs + debugLogic=cms.untracked.bool(options.debugLogic), + defaultCrossingAngleX = cms.untracked.double(options.defaultXangleX), + defaultCrossingAngleY = cms.untracked.double(options.defaultXangleY), + defaultBetaStarX = cms.untracked.double(options.defaultBetaX), + defaultBetaStarY = cms.untracked.double(options.defaultBetaY), + minBetaStar = cms.untracked.double(options.minBetaStar), + maxBetaStar = cms.untracked.double(options.maxBetaStar), + minCrossingAngle = cms.untracked.double(options.minCrossingAngle), + maxCrossingAngle = cms.untracked.double(options.maxCrossingAngle), + ), loggingOn = cms.untracked.bool(True), IsDestDbCheckedInQueryLog = cms.untracked.bool(False) ) -process.p = cms.Path(process.Test1) +process.p = cms.Path(process.Test1) \ No newline at end of file diff --git a/CondTools/RunInfo/src/LHCInfoHelper.cc b/CondTools/RunInfo/src/LHCInfoHelper.cc index 54503068b8924..e23f4cc199e58 100644 --- a/CondTools/RunInfo/src/LHCInfoHelper.cc +++ b/CondTools/RunInfo/src/LHCInfoHelper.cc @@ -5,7 +5,9 @@ // Returns lumi-type IOV (packed using cond::time::lumiTime) from // last LS of last Run of the specified Fill //***************************************************************** -cond::Time_t cond::lhcInfoHelper::getFillLastLumiIOV(const cond::OMSService& oms, unsigned short fillId) { + +std::pair cond::lhcInfoHelper::getFillLastRunAndLS(const cond::OMSService& oms, + unsigned short fillId) { // Define query auto query = oms.query("lumisections"); query->addOutputVars({"lumisection_number", "run_number"}); @@ -27,5 +29,10 @@ cond::Time_t cond::lhcInfoHelper::getFillLastLumiIOV(const cond::OMSService& oms // Return the final IOV auto lastRun = queryResult.back().get("run_number"); auto lastLumi = queryResult.back().get("lumisection_number"); - return cond::time::lumiTime(lastRun, lastLumi); + return std::make_pair(lastRun, lastLumi); } + +cond::Time_t cond::lhcInfoHelper::getFillLastLumiIOV(const cond::OMSService& oms, unsigned short fillId) { + auto [lastRun, lastLumi] = getFillLastRunAndLS(oms, fillId); + return cond::time::lumiTime(lastRun, lastLumi); +} \ No newline at end of file diff --git a/CondTools/RunInfo/src/LHCInfoPerFillPopConSourceHandler.cc b/CondTools/RunInfo/src/LHCInfoPerFillPopConSourceHandler.cc new file mode 100644 index 0000000000000..553a0b6d9e2d6 --- /dev/null +++ b/CondTools/RunInfo/src/LHCInfoPerFillPopConSourceHandler.cc @@ -0,0 +1,744 @@ +#include "CondTools/RunInfo/interface/LHCInfoPerFillPopConSourceHandler.h" +#include "CondCore/CondDB/interface/ConnectionPool.h" +#include "CondFormats/Common/interface/TimeConversions.h" +#include "CondFormats/RunInfo/interface/LHCInfoPerFill.h" +#include "CondTools/RunInfo/interface/LumiSectionFilter.h" +#include "CondTools/RunInfo/interface/LHCInfoHelper.h" +#include "CondTools/RunInfo/interface/OMSAccess.h" +#include "CoralBase/Attribute.h" +#include "CoralBase/AttributeList.h" +#include "CoralBase/AttributeSpecification.h" +#include "CoralBase/TimeStamp.h" +#include "FWCore/MessageLogger/interface/MessageLogger.h" +#include "FWCore/ParameterSet/interface/ParameterSet.h" +#include "RelationalAccess/ICursor.h" +#include "RelationalAccess/IQuery.h" +#include "RelationalAccess/ISchema.h" +#include "RelationalAccess/ISessionProxy.h" + +using std::make_pair; +using std::pair; + +namespace cond { + namespace theLHCInfoPerFillPopConImpl { + + static const pair s_fillTypeMap[] = { + make_pair("PROTONS", LHCInfoPerFill::PROTONS), + make_pair("IONS", LHCInfoPerFill::IONS), + make_pair("COSMICS", LHCInfoPerFill::COSMICS), + make_pair("GAP", LHCInfoPerFill::GAP)}; + + static const pair s_particleTypeMap[] = { + make_pair("PROTON", LHCInfoPerFill::PROTON), + make_pair("PB82", LHCInfoPerFill::PB82), + make_pair("AR18", LHCInfoPerFill::AR18), + make_pair("D", LHCInfoPerFill::D), + make_pair("XE54", LHCInfoPerFill::XE54)}; + + LHCInfoPerFill::FillType fillTypeFromString(const std::string& s_fill_type) { + for (auto const& i : s_fillTypeMap) + if (s_fill_type == i.first) + return i.second; + return LHCInfoPerFill::UNKNOWN; + } + + LHCInfoPerFill::ParticleType particleTypeFromString(const std::string& s_particle_type) { + for (auto const& i : s_particleTypeMap) + if (s_particle_type == i.first) + return i.second; + return LHCInfoPerFill::NONE; + } + } // namespace theLHCInfoPerFillPopConImpl + + namespace impl { + + template <> + LHCInfoPerFill::FillType from_string(const std::string& attributeValue) { + return from_string_impl( + attributeValue, LHCInfoPerFill::UNKNOWN); + } + + template <> + LHCInfoPerFill::ParticleType from_string(const std::string& attributeValue) { + return from_string_impl( + attributeValue, LHCInfoPerFill::NONE); + } + + } // namespace impl +} // namespace cond + +namespace theLHCInfoPerFillImpl { + + bool makeFillPayload(std::unique_ptr& targetPayload, const cond::OMSServiceResult& queryResult) { + bool ret = false; + if (!queryResult.empty()) { + auto row = *queryResult.begin(); + auto currentFill = row.get("fill_number"); + auto bunches1 = row.get("bunches_beam1"); + auto bunches2 = row.get("bunches_beam2"); + auto collidingBunches = row.get("bunches_colliding"); + auto targetBunches = row.get("bunches_target"); + auto fillType = row.get("fill_type_runtime"); + auto particleType1 = row.get("fill_type_party1"); + auto particleType2 = row.get("fill_type_party2"); + auto intensityBeam1 = row.get("intensity_beam1"); + auto intensityBeam2 = row.get("intensity_beam2"); + auto energy = row.get("energy"); + auto creationTime = row.get("start_time"); + auto stableBeamStartTime = row.get("start_stable_beam"); + std::string endTimeStr = row.get("end_time"); + auto beamDumpTime = + (endTimeStr == "null") ? 0 : cond::time::from_boost(row.get("end_time")); + auto injectionScheme = row.get("injection_scheme"); + targetPayload = std::make_unique(); + targetPayload->setFillNumber(currentFill); + targetPayload->setBunchesInBeam1(bunches1); + targetPayload->setBunchesInBeam2(bunches2); + targetPayload->setCollidingBunches(collidingBunches); + targetPayload->setTargetBunches(targetBunches); + targetPayload->setFillType(fillType); + targetPayload->setParticleTypeForBeam1(particleType1); + targetPayload->setParticleTypeForBeam2(particleType2); + targetPayload->setIntensityForBeam1(intensityBeam1); + targetPayload->setIntensityForBeam2(intensityBeam2); + targetPayload->setEnergy(energy); + targetPayload->setCreationTime(cond::time::from_boost(creationTime)); + targetPayload->setBeginTime(cond::time::from_boost(stableBeamStartTime)); + targetPayload->setEndTime(beamDumpTime); + targetPayload->setInjectionScheme(injectionScheme); + ret = true; + } + return ret; + } +} // namespace theLHCInfoPerFillImpl + +namespace theLHCInfoPerFillImpl { + static const std::map vecMap = { + {"Beam1/beamPhaseMean", 1}, {"Beam2/beamPhaseMean", 2}, {"Beam1/cavPhaseMean", 3}, {"Beam2/cavPhaseMean", 4}}; + void setElementData(cond::Time_t since, + const std::string& dipVal, + unsigned int elementNr, + float value, + LHCInfoPerFill& payload, + std::set& initList) { + if (initList.find(since) == initList.end()) { + payload.beam1VC().resize(LHCInfoPerFill::bunchSlots, 0.); + payload.beam2VC().resize(LHCInfoPerFill::bunchSlots, 0.); + payload.beam1RF().resize(LHCInfoPerFill::bunchSlots, 0.); + payload.beam2RF().resize(LHCInfoPerFill::bunchSlots, 0.); + initList.insert(since); + } + // set the current values to all of the payloads of the lumi section samples after the current since + if (elementNr < LHCInfoPerFill::bunchSlots) { + switch (vecMap.at(dipVal)) { + case 1: + payload.beam1VC()[elementNr] = value; + break; + case 2: + payload.beam2VC()[elementNr] = value; + break; + case 3: + payload.beam1RF()[elementNr] = value; + break; + case 4: + payload.beam2RF()[elementNr] = value; + break; + default: + break; + } + } + } +} // namespace theLHCInfoPerFillImpl + +namespace theLHCInfoPerFillImpl { + bool comparePayloads(const LHCInfoPerFill& rhs, const LHCInfoPerFill& lhs) { + if (rhs.fillNumber() != lhs.fillNumber() || rhs.delivLumi() != lhs.delivLumi() || rhs.recLumi() != lhs.recLumi() || + rhs.instLumi() != lhs.instLumi() || rhs.instLumiError() != lhs.instLumiError() || + rhs.lhcState() != lhs.lhcState() || rhs.lhcComment() != lhs.lhcComment() || + rhs.ctppsStatus() != lhs.ctppsStatus()) { + return false; + } + return true; + } + + size_t transferPayloads(const std::vector>>& buffer, + std::map>& iovsToTransfer, + std::shared_ptr& prevPayload) { + size_t niovs = 0; + std::stringstream condIovs; + std::stringstream formattedIovs; + for (auto& iov : buffer) { + bool add = false; + auto payload = iov.second; + cond::Time_t since = iov.first; + if (iovsToTransfer.empty()) { + add = true; + } else { + LHCInfoPerFill& lastAdded = *iovsToTransfer.rbegin()->second; + if (!comparePayloads(lastAdded, *payload)) { + add = true; + } + } + if (add) { + niovs++; + condIovs << since << " "; + formattedIovs << boost::posix_time::to_iso_extended_string(cond::time::to_boost(since)) << " "; + iovsToTransfer.insert(make_pair(since, payload)); + prevPayload = iov.second; + } + } + edm::LogInfo("transferPayloads") << "TRANSFERED IOVS: " << condIovs.str(); + edm::LogInfo("transferPayloads") << "FORMATTED TRANSFERED IOVS: " << formattedIovs.str(); + return niovs; + } + +} // namespace theLHCInfoPerFillImpl + +LHCInfoPerFillPopConSourceHandler::LHCInfoPerFillPopConSourceHandler(edm::ParameterSet const& pset) + : m_debug(pset.getUntrackedParameter("debug", false)), + m_startTime(), + m_endTime(), + m_endFillMode(pset.getUntrackedParameter("endFill", true)), + m_name(pset.getUntrackedParameter("name", "LHCInfoPerFillPopConSourceHandler")), + m_connectionString(pset.getUntrackedParameter("connectionString", "")), + m_ecalConnectionString(pset.getUntrackedParameter("ecalConnectionString", "")), + m_authpath(pset.getUntrackedParameter("authenticationPath", "")), + m_omsBaseUrl(pset.getUntrackedParameter("omsBaseUrl", "")), + m_fillPayload(), + m_prevPayload(), + m_tmpBuffer() { + if (!pset.getUntrackedParameter("startTime").empty()) { + m_startTime = boost::posix_time::time_from_string(pset.getUntrackedParameter("startTime")); + } + boost::posix_time::ptime now = boost::posix_time::second_clock::local_time(); + m_endTime = now; + if (!pset.getUntrackedParameter("endTime").empty()) { + m_endTime = boost::posix_time::time_from_string(pset.getUntrackedParameter("endTime")); + if (m_endTime > now) + m_endTime = now; + } +} + +void LHCInfoPerFillPopConSourceHandler::getNewObjects() { + //if a new tag is created, transfer fake fill from 1 to the first fill for the first time + if (tagInfo().size == 0) { + edm::LogInfo(m_name) << "New tag " << tagInfo().name << "; from " << m_name << "::getNewObjects"; + } else { + //check what is already inside the database + edm::LogInfo(m_name) << "got info for tag " << tagInfo().name << ": size " << tagInfo().size + << ", last object valid since " << tagInfo().lastInterval.since << " ( " + << boost::posix_time::to_iso_extended_string( + cond::time::to_boost(tagInfo().lastInterval.since)) + << " ); from " << m_name << "::getNewObjects"; + } + + cond::Time_t lastSince = tagInfo().lastInterval.since; + if (tagInfo().isEmpty()) { + // for a new or empty tag in endFill mode, an empty payload should be added on top with since=1 + addEmptyPayload(1); + lastSince = 1; + if (!m_endFillMode) { + edm::LogInfo(m_name) << "Empty or new tag: uploading a default payload and ending the job"; + return; + } + } else { + edm::LogInfo(m_name) << "The last Iov in tag " << tagInfo().name << " valid since " << lastSince << "from " + << m_name << "::getNewObjects"; + } + + //retrieve the data from the relational database source + cond::persistency::ConnectionPool connection; + //configure the connection + if (m_debug) { + connection.setMessageVerbosity(coral::Debug); + } else { + connection.setMessageVerbosity(coral::Error); + } + connection.setAuthenticationPath(m_authpath); + connection.configure(); + //create the sessions + cond::persistency::Session session = connection.createSession(m_connectionString, false); + cond::persistency::Session session2 = connection.createSession(m_ecalConnectionString, false); + // fetch last payload when available + if (!tagInfo().lastInterval.payloadId.empty()) { + cond::persistency::Session session3 = dbSession(); + session3.transaction().start(true); + m_prevPayload = session3.fetchPayload(tagInfo().lastInterval.payloadId); + session3.transaction().commit(); + } + + boost::posix_time::ptime executionTime = boost::posix_time::second_clock::local_time(); + cond::Time_t executionTimeIov = cond::time::from_boost(executionTime); + + cond::Time_t startTimestamp = m_startTime.is_not_a_date_time() ? 0 : cond::time::from_boost(m_startTime); + cond::Time_t nextFillSearchTimestamp = + std::max(startTimestamp, m_endFillMode ? lastSince : m_prevPayload->createTime()); + + edm::LogInfo(m_name) << "Starting sampling at " + << boost::posix_time::to_simple_string(cond::time::to_boost(nextFillSearchTimestamp)); + + while (true) { + if (nextFillSearchTimestamp >= executionTimeIov) { + edm::LogInfo(m_name) << "Sampling ended at the time " + << boost::posix_time::to_simple_string(cond::time::to_boost(executionTimeIov)); + break; + } + boost::posix_time::ptime nextFillSearchTime = cond::time::to_boost(nextFillSearchTimestamp); + boost::posix_time::ptime startSampleTime; + boost::posix_time::ptime endSampleTime; + + cond::OMSService oms; + oms.connect(m_omsBaseUrl); + auto query = oms.query("fills"); + + edm::LogInfo(m_name) << "Searching new fill after " << boost::posix_time::to_simple_string(nextFillSearchTime); + query->filterNotNull("start_stable_beam").filterNotNull("fill_number"); + if (nextFillSearchTime > cond::time::to_boost(m_prevPayload->createTime())) { + query->filterGE("start_time", nextFillSearchTime); + } else { + query->filterGT("start_time", nextFillSearchTime); + } + + query->filterLT("start_time", m_endTime); + if (m_endFillMode) + query->filterNotNull("end_time"); + else + query->filterEQ("end_time", cond::OMSServiceQuery::SNULL); + + bool foundFill = query->execute(); + if (foundFill) + foundFill = theLHCInfoPerFillImpl::makeFillPayload(m_fillPayload, query->result()); + if (!foundFill) { + edm::LogInfo(m_name) << "No fill found - END of job."; + break; + } + + startSampleTime = cond::time::to_boost(m_fillPayload->createTime()); + cond::Time_t startFillTime = m_fillPayload->createTime(); + cond::Time_t endFillTime = m_fillPayload->endTime(); + unsigned short lhcFill = m_fillPayload->fillNumber(); + bool ongoingFill = endFillTime == 0ULL; + if (ongoingFill) { + edm::LogInfo(m_name) << "Found ongoing fill " << lhcFill << " created at " << cond::time::to_boost(startFillTime); + endSampleTime = executionTime; + nextFillSearchTimestamp = executionTimeIov; + } else { + edm::LogInfo(m_name) << "Found fill " << lhcFill << " created at " << cond::time::to_boost(startFillTime) + << " ending at " << cond::time::to_boost(endFillTime); + endSampleTime = cond::time::to_boost(endFillTime); + nextFillSearchTimestamp = endFillTime; + } + if (m_endFillMode || ongoingFill) { + getDipData(oms, startSampleTime, endSampleTime); + getLumiData(oms, lhcFill, startSampleTime, endSampleTime); + if (!m_tmpBuffer.empty()) { + boost::posix_time::ptime flumiStart = cond::time::to_boost(m_tmpBuffer.front().first); + boost::posix_time::ptime flumiStop = cond::time::to_boost(m_tmpBuffer.back().first); + edm::LogInfo(m_name) << "First lumi starts at " << flumiStart << " last lumi starts at " << flumiStop; + session.transaction().start(true); + getCTPPSData(session, startSampleTime, endSampleTime); + session.transaction().commit(); + session2.transaction().start(true); + getEcalData(session2, startSampleTime, endSampleTime); + session2.transaction().commit(); + } + } + + if (!m_endFillMode) { + if (m_tmpBuffer.size() > 1) { + throw cms::Exception("LHCInfoPerFillPopConSourceHandler") + << "More than 1 payload buffered for writing in duringFill mode.\ + In this mode only up to 1 payload can be written"; + } else if (m_tmpBuffer.size() == 1) { + if (theLHCInfoPerFillImpl::comparePayloads(*(m_tmpBuffer.begin()->second), *m_prevPayload)) { + m_tmpBuffer.clear(); + edm::LogInfo(m_name) + << "The buffered payload has the same data as the previous payload in the tag. It will not be written."; + } + } else if (m_tmpBuffer.empty()) { + addEmptyPayload( + cond::lhcInfoHelper::getFillLastLumiIOV(oms, lhcFill)); //the IOV doesn't matter when using OnlinePopCon + } + // In duringFill mode, convert the timestamp-type IOVs to lumiid-type IOVs + // before transferring the payloads from the buffer to the final collection + convertBufferedIovsToLumiid(m_timestampToLumiid); + } + + size_t niovs = theLHCInfoPerFillImpl::transferPayloads(m_tmpBuffer, m_iovs, m_prevPayload); + edm::LogInfo(m_name) << "Added " << niovs << " iovs within the Fill time"; + m_tmpBuffer.clear(); + m_timestampToLumiid.clear(); + + if (!m_endFillMode) { + return; + } + + // endFill mode only: + if (m_prevPayload->fillNumber() and !ongoingFill) { + if (m_endFillMode) { + addEmptyPayload(endFillTime); + } else { + addEmptyPayload(cond::lhcInfoHelper::getFillLastLumiIOV(oms, lhcFill)); + } + } + } +} + +std::string LHCInfoPerFillPopConSourceHandler::id() const { return m_name; } + +void LHCInfoPerFillPopConSourceHandler::addEmptyPayload(cond::Time_t iov) { + bool add = false; + if (m_iovs.empty()) { + if (!m_lastPayloadEmpty) + add = true; + } else { + auto lastAdded = m_iovs.rbegin()->second; + if (lastAdded->fillNumber() != 0) { + add = true; + } + } + if (add) { + auto newPayload = std::make_shared(); + m_iovs.insert(make_pair(iov, newPayload)); + m_prevPayload = newPayload; + edm::LogInfo(m_name) << "Added empty payload with IOV " << iov << " ( " + << boost::posix_time::to_iso_extended_string(cond::time::to_boost(iov)) << " )"; + } +} + +// Add payload to buffer and store corresponding lumiid IOV in m_timestampToLumiid map +void LHCInfoPerFillPopConSourceHandler::addPayloadToBuffer(cond::OMSServiceResultRef& row) { + auto startTime = row.get("start_time"); + auto delivLumi = row.get("delivered_lumi"); + auto recLumi = row.get("recorded_lumi"); + auto runNumber = std::stoul(row.get("run_number")); + auto lsNumber = std::stoul(row.get("lumisection_number")); + auto lumiid = cond::time::lumiTime(runNumber, lsNumber); + + LHCInfoPerFill* thisLumiSectionInfo = m_fillPayload->cloneFill(); + m_tmpBuffer.emplace_back(make_pair(cond::time::from_boost(startTime), thisLumiSectionInfo)); + if (!m_endFillMode) { + m_timestampToLumiid.insert(make_pair(cond::time::from_boost(startTime), lumiid)); + } + LHCInfoPerFill& payload = *thisLumiSectionInfo; + payload.setDelivLumi(delivLumi); + payload.setRecLumi(recLumi); +} + +void LHCInfoPerFillPopConSourceHandler::convertBufferedIovsToLumiid( + std::map timestampToLumiid) { + for (auto& item : m_tmpBuffer) { + // Check if the lumiid IOV corresponding to the timestamp is present in the map + if (timestampToLumiid.find(item.first) == timestampToLumiid.end()) { + throw cms::Exception("LHCInfoPerFillPopConSourceHandler") + << "Can't find corresponding lumiid IOV for timestamp " << item.first << "\n"; + } + // Update the buffer with the lumiid-type IOV + item.first = timestampToLumiid.at(item.first); + } +} + +size_t LHCInfoPerFillPopConSourceHandler::getLumiData(const cond::OMSService& oms, + unsigned short fillId, + const boost::posix_time::ptime& beginFillTime, + const boost::posix_time::ptime& endFillTime) { + auto query = oms.query("lumisections"); + query->addOutputVars( + {"start_time", "delivered_lumi", "recorded_lumi", "beams_stable", "run_number", "lumisection_number"}); + query->filterEQ("fill_number", fillId); + query->filterGT("start_time", beginFillTime).filterLT("start_time", endFillTime); + query->filterEQ("beams_stable", "true"); + query->limit(cond::lhcInfoHelper::kLumisectionsQueryLimit); + if (query->execute()) { + auto queryResult = query->result(); + edm::LogInfo(m_name) << "Found " << queryResult.size() << " lumisections with STABLE BEAM during the fill " + << fillId; + + if (!queryResult.empty()) { + if (m_endFillMode) { + auto firstRow = queryResult.front(); + addPayloadToBuffer(firstRow); + } + + auto lastRow = queryResult.back(); + addPayloadToBuffer(lastRow); + } + } + return 0; +} + +void LHCInfoPerFillPopConSourceHandler::getDipData(const cond::OMSService& oms, + const boost::posix_time::ptime& beginFillTime, + const boost::posix_time::ptime& endFillTime) { + // unsure how to handle this. + // the old implementation is not helping: apparently it is checking only the bunchconfiguration for the first diptime set of values... + auto query1 = oms.query("diplogger/dip/acc/LHC/RunControl/CirculatingBunchConfig/Beam1"); + query1->filterGT("dip_time", beginFillTime).filterLT("dip_time", endFillTime); + //This query is limited to 100 rows, but currently only one is used + //If all this data is needed and saved properly the limit has to be set: query1->limit(...) + if (query1->execute()) { + auto res = query1->result(); + if (!res.empty()) { + std::bitset bunchConfiguration1(0ULL); + auto row = *res.begin(); + auto vbunchConf1 = row.getArray("value"); + for (auto vb : vbunchConf1) { + if (vb != 0) { + unsigned short slot = (vb - 1) / 10 + 1; + bunchConfiguration1[slot] = true; + } + } + m_fillPayload->setBunchBitsetForBeam1(bunchConfiguration1); + } + } + auto query2 = oms.query("diplogger/dip/acc/LHC/RunControl/CirculatingBunchConfig/Beam2"); + query2->filterGT("dip_time", beginFillTime).filterLT("dip_time", endFillTime); + //This query is limited to 100 rows, but currently only one is used + if (query2->execute()) { + auto res = query2->result(); + if (!res.empty()) { + std::bitset bunchConfiguration2(0ULL); + auto row = *res.begin(); + auto vbunchConf2 = row.getArray("value"); + for (auto vb : vbunchConf2) { + if (vb != 0) { + unsigned short slot = (vb - 1) / 10 + 1; + bunchConfiguration2[slot] = true; + } + } + m_fillPayload->setBunchBitsetForBeam2(bunchConfiguration2); + } + } + + auto query3 = oms.query("diplogger/dip/CMS/LHC/LumiPerBunch"); + query3->filterGT("dip_time", beginFillTime).filterLT("dip_time", endFillTime); + //This query is limited to 100 rows, but currently only one is used + if (query3->execute()) { + auto res = query3->result(); + if (!res.empty()) { + std::vector lumiPerBX; + auto row = *res.begin(); + auto lumiBunchInst = row.getArray("lumi_bunch_inst"); + for (auto lb : lumiBunchInst) { + if (lb != 0.) { + lumiPerBX.push_back(lb); + } + } + m_fillPayload->setLumiPerBX(lumiPerBX); + } + } +} + +bool LHCInfoPerFillPopConSourceHandler::getCTPPSData(cond::persistency::Session& session, + const boost::posix_time::ptime& beginFillTime, + const boost::posix_time::ptime& endFillTime) { + //run the fifth query against the CTPPS schema + //Initializing the CMS_CTP_CTPPS_COND schema. + coral::ISchema& CTPPS = session.coralSession().schema("CMS_PPS_SPECT_COND"); + //execute query for CTPPS Data + std::unique_ptr CTPPSDataQuery(CTPPS.newQuery()); + //FROM clause + CTPPSDataQuery->addToTableList(std::string("PPS_LHC_MACHINE_PARAMS")); + //SELECT clause + CTPPSDataQuery->addToOutputList(std::string("DIP_UPDATE_TIME")); + CTPPSDataQuery->addToOutputList(std::string("LHC_STATE")); + CTPPSDataQuery->addToOutputList(std::string("LHC_COMMENT")); + if (m_debug) { + CTPPSDataQuery->addToOutputList(std::string("RUN_NUMBER")); + CTPPSDataQuery->addToOutputList(std::string("LUMI_SECTION")); + } + //WHERE CLAUSE + coral::AttributeList CTPPSDataBindVariables; + CTPPSDataBindVariables.extend(std::string("beginFillTime")); + CTPPSDataBindVariables.extend(std::string("endFillTime")); + CTPPSDataBindVariables[std::string("beginFillTime")].data() = coral::TimeStamp(beginFillTime); + CTPPSDataBindVariables[std::string("endFillTime")].data() = coral::TimeStamp(endFillTime); + std::string conditionStr = std::string("DIP_UPDATE_TIME>= :beginFillTime and DIP_UPDATE_TIME< :endFillTime"); + CTPPSDataQuery->setCondition(conditionStr, CTPPSDataBindVariables); + //ORDER BY clause + CTPPSDataQuery->addToOrderList(std::string("DIP_UPDATE_TIME")); + //define query output + coral::AttributeList CTPPSDataOutput; + CTPPSDataOutput.extend(std::string("DIP_UPDATE_TIME")); + CTPPSDataOutput.extend(std::string("LHC_STATE")); + CTPPSDataOutput.extend(std::string("LHC_COMMENT")); + if (m_debug) { + CTPPSDataOutput.extend(std::string("RUN_NUMBER")); + CTPPSDataOutput.extend(std::string("LUMI_SECTION")); + } + CTPPSDataQuery->defineOutput(CTPPSDataOutput); + //execute the query + coral::ICursor& CTPPSDataCursor = CTPPSDataQuery->execute(); + cond::Time_t dipTime = 0; + std::string lhcState = "", lhcComment = "", ctppsStatus = ""; + + //debug informations + unsigned int lumiSection = 0; + cond::Time_t runNumber = 0; + cond::Time_t savedDipTime = 0; + unsigned int savedLumiSection = 0; + cond::Time_t savedRunNumber = 0; + + bool ret = false; + LumiSectionFilter filter(m_tmpBuffer); + while (CTPPSDataCursor.next()) { + if (m_debug) { + std::ostringstream CTPPS; + CTPPSDataCursor.currentRow().toOutputStream(CTPPS); + } + coral::Attribute const& dipTimeAttribute = CTPPSDataCursor.currentRow()[std::string("DIP_UPDATE_TIME")]; + if (!dipTimeAttribute.isNull()) { + dipTime = cond::time::from_boost(dipTimeAttribute.data().time()); + if (filter.process(dipTime)) { + ret = true; + coral::Attribute const& lhcStateAttribute = CTPPSDataCursor.currentRow()[std::string("LHC_STATE")]; + if (!lhcStateAttribute.isNull()) { + lhcState = lhcStateAttribute.data(); + } + coral::Attribute const& lhcCommentAttribute = CTPPSDataCursor.currentRow()[std::string("LHC_COMMENT")]; + if (!lhcCommentAttribute.isNull()) { + lhcComment = lhcCommentAttribute.data(); + } + + if (m_debug) { + coral::Attribute const& runNumberAttribute = CTPPSDataCursor.currentRow()[std::string("RUN_NUMBER")]; + if (!runNumberAttribute.isNull()) { + runNumber = runNumberAttribute.data(); + } + coral::Attribute const& lumiSectionAttribute = CTPPSDataCursor.currentRow()[std::string("LUMI_SECTION")]; + if (!lumiSectionAttribute.isNull()) { + lumiSection = lumiSectionAttribute.data(); + } + } + + for (auto it = filter.current(); it != m_tmpBuffer.end(); it++) { + // set the current values to all of the payloads of the lumi section samples after the current since + LHCInfoPerFill& payload = *(it->second); + payload.setLhcState(lhcState); + payload.setLhcComment(lhcComment); + payload.setCtppsStatus(ctppsStatus); + + if (m_debug) { + savedDipTime = dipTime; + savedLumiSection = lumiSection; + savedRunNumber = runNumber; + } + } + } + } + } + if (m_debug) { + edm::LogInfo(m_name) << "Last assigned: " + << "DipTime: " << savedDipTime << " " + << "LumiSection: " << savedLumiSection << " " + << "RunNumber: " << savedRunNumber; + } + return ret; +} + +bool LHCInfoPerFillPopConSourceHandler::getEcalData(cond::persistency::Session& session, + const boost::posix_time::ptime& lowerTime, + const boost::posix_time::ptime& upperTime) { + //run the sixth query against the CMS_DCS_ENV_PVSS_COND schema + //Initializing the CMS_DCS_ENV_PVSS_COND schema. + coral::ISchema& ECAL = session.nominalSchema(); + //start the transaction against the fill logging schema + //execute query for ECAL Data + std::unique_ptr ECALDataQuery(ECAL.newQuery()); + //FROM clause + ECALDataQuery->addToTableList(std::string("BEAM_PHASE")); + //SELECT clause + ECALDataQuery->addToOutputList(std::string("CHANGE_DATE")); + ECALDataQuery->addToOutputList(std::string("DIP_value")); + ECALDataQuery->addToOutputList(std::string("element_nr")); + ECALDataQuery->addToOutputList(std::string("VALUE_NUMBER")); + //WHERE CLAUSE + coral::AttributeList ECALDataBindVariables; + ECALDataBindVariables.extend(std::string("lowerTime")); + ECALDataBindVariables.extend(std::string("upperTime")); + ECALDataBindVariables[std::string("lowerTime")].data() = coral::TimeStamp(lowerTime); + ECALDataBindVariables[std::string("upperTime")].data() = coral::TimeStamp(upperTime); + std::string conditionStr = std::string( + "(DIP_value LIKE '%beamPhaseMean%' OR DIP_value LIKE '%cavPhaseMean%') AND CHANGE_DATE >= :lowerTime AND " + "CHANGE_DATE < :upperTime"); + + ECALDataQuery->setCondition(conditionStr, ECALDataBindVariables); + //ORDER BY clause + ECALDataQuery->addToOrderList(std::string("CHANGE_DATE")); + ECALDataQuery->addToOrderList(std::string("DIP_value")); + ECALDataQuery->addToOrderList(std::string("element_nr")); + //define query output + coral::AttributeList ECALDataOutput; + ECALDataOutput.extend(std::string("CHANGE_DATE")); + ECALDataOutput.extend(std::string("DIP_value")); + ECALDataOutput.extend(std::string("element_nr")); + ECALDataOutput.extend(std::string("VALUE_NUMBER")); + //ECALDataQuery->limitReturnedRows( 14256 ); //3564 entries per vector. + ECALDataQuery->defineOutput(ECALDataOutput); + //execute the query + coral::ICursor& ECALDataCursor = ECALDataQuery->execute(); + cond::Time_t changeTime = 0; + cond::Time_t firstTime = 0; + std::string dipVal = ""; + unsigned int elementNr = 0; + float value = 0.; + std::set initializedVectors; + LumiSectionFilter filter(m_tmpBuffer); + bool ret = false; + if (m_prevPayload.get()) { + for (auto& lumiSlot : m_tmpBuffer) { + lumiSlot.second->setBeam1VC(m_prevPayload->beam1VC()); + lumiSlot.second->setBeam2VC(m_prevPayload->beam2VC()); + lumiSlot.second->setBeam1RF(m_prevPayload->beam1RF()); + lumiSlot.second->setBeam2RF(m_prevPayload->beam2RF()); + } + } + std::map iovMap; + if (m_tmpBuffer.empty()) { + return ret; + } + cond::Time_t lowerLumi = m_tmpBuffer.front().first; + while (ECALDataCursor.next()) { + if (m_debug) { + std::ostringstream ECAL; + ECALDataCursor.currentRow().toOutputStream(ECAL); + } + coral::Attribute const& changeDateAttribute = ECALDataCursor.currentRow()[std::string("CHANGE_DATE")]; + if (!changeDateAttribute.isNull()) { + ret = true; + boost::posix_time::ptime chTime = changeDateAttribute.data().time(); + // move the first IOV found to the start of the fill interval selected + if (changeTime == 0) { + firstTime = cond::time::from_boost(chTime); + } + changeTime = cond::time::from_boost(chTime); + cond::Time_t iovTime = changeTime; + if (changeTime == firstTime) + iovTime = lowerLumi; + coral::Attribute const& dipValAttribute = ECALDataCursor.currentRow()[std::string("DIP_value")]; + coral::Attribute const& valueNumberAttribute = ECALDataCursor.currentRow()[std::string("VALUE_NUMBER")]; + coral::Attribute const& elementNrAttribute = ECALDataCursor.currentRow()[std::string("element_nr")]; + if (!dipValAttribute.isNull() and !valueNumberAttribute.isNull()) { + dipVal = dipValAttribute.data(); + elementNr = elementNrAttribute.data(); + value = valueNumberAttribute.data(); + if (std::isnan(value)) + value = 0.; + if (filter.process(iovTime)) { + iovMap.insert(make_pair(changeTime, filter.current()->first)); + for (auto it = filter.current(); it != m_tmpBuffer.end(); it++) { + LHCInfoPerFill& payload = *(it->second); + theLHCInfoPerFillImpl::setElementData(it->first, dipVal, elementNr, value, payload, initializedVectors); + } + } + } + } + } + if (m_debug) { + for (auto& im : iovMap) { + edm::LogInfo(m_name) << "Found iov=" << im.first << " (" << cond::time::to_boost(im.first) << " ) moved to " + << im.second << " ( " << cond::time::to_boost(im.second) << " )"; + } + } + return ret; +} diff --git a/CondTools/RunInfo/src/LHCInfoPerLSPopConSourceHandler.cc b/CondTools/RunInfo/src/LHCInfoPerLSPopConSourceHandler.cc new file mode 100644 index 0000000000000..8cb01ac930cf8 --- /dev/null +++ b/CondTools/RunInfo/src/LHCInfoPerLSPopConSourceHandler.cc @@ -0,0 +1,659 @@ +#include "CondTools/RunInfo/interface/LHCInfoPerLSPopConSourceHandler.h" +#include "CondCore/CondDB/interface/ConnectionPool.h" +#include "CondCore/CondDB/interface/Types.h" +#include "CondFormats/Common/interface/TimeConversions.h" +#include "CondTools/RunInfo/interface/LHCInfoHelper.h" +#include "CondTools/RunInfo/interface/OMSAccess.h" +#include "CoralBase/Attribute.h" +#include "CoralBase/AttributeList.h" +#include "CoralBase/AttributeSpecification.h" +#include "CoralBase/TimeStamp.h" +#include "FWCore/MessageLogger/interface/MessageLogger.h" +#include "FWCore/ParameterSet/interface/ParameterSet.h" +#include "RelationalAccess/ICursor.h" +#include "RelationalAccess/IQuery.h" +#include "RelationalAccess/ISchema.h" +#include "RelationalAccess/ISessionProxy.h" +#include +#include + +using std::make_pair; +using std::pair; + +namespace theLHCInfoPerLSImpl { + bool comparePayloads(const LHCInfoPerLS& rhs, const LHCInfoPerLS& lhs) { + if (rhs.fillNumber() != lhs.fillNumber() || rhs.runNumber() != lhs.runNumber() || + rhs.crossingAngleX() != lhs.crossingAngleX() || rhs.crossingAngleY() != lhs.crossingAngleY() || + rhs.betaStarX() != lhs.betaStarX() || rhs.betaStarY() != lhs.betaStarY()) { + return false; + } + return true; + } + + size_t transferPayloads(const std::vector>>& buffer, + std::map>& iovsToTransfer, + std::shared_ptr& prevPayload, + const std::map, pair>& lsIdMap, + cond::Time_t startStableBeamTime, + cond::Time_t endStableBeamTime) { + int lsMissingInPPS = 0; + int xAngleBothZero = 0, xAngleBothNonZero = 0, xAngleNegative = 0; + int betaNegative = 0; + size_t niovs = 0; + std::stringstream condIovs; + std::stringstream missingLsList; + for (auto& iov : buffer) { + bool add = false; + auto payload = iov.second; + cond::Time_t since = iov.first; + if (iovsToTransfer.empty()) { + add = true; + } else { + LHCInfoPerLS& lastAdded = *iovsToTransfer.rbegin()->second; + if (!comparePayloads(lastAdded, *payload)) { + add = true; + } + } + auto id = make_pair(payload->runNumber(), payload->lumiSection()); + bool stableBeam = since >= startStableBeamTime && since <= endStableBeamTime; + bool isMissing = lsIdMap.find(id) != lsIdMap.end() && id != lsIdMap.at(id); + if (stableBeam && isMissing) { + missingLsList << id.first << "_" << id.second << " "; + lsMissingInPPS += isMissing; + } + if (add && !isMissing) { + niovs++; + if (stableBeam) { + if (payload->crossingAngleX() == 0 && payload->crossingAngleY() == 0) + xAngleBothZero++; + if (payload->crossingAngleX() != 0 && payload->crossingAngleY() != 0) + xAngleBothNonZero++; + if (payload->crossingAngleX() < 0 || payload->crossingAngleY() < 0) + xAngleNegative++; + if (payload->betaStarX() < 0 || payload->betaStarY() < 0) + betaNegative++; + } + + condIovs << since << " "; + iovsToTransfer.insert(make_pair(since, payload)); + prevPayload = iov.second; + } + } + unsigned short fillNumber = (!buffer.empty()) ? buffer.front().second->fillNumber() : 0; + if (lsMissingInPPS > 0) { + edm::LogWarning("transferPayloads") + << "Number of stable beam LS in OMS without corresponding record in PPS DB for fill " << fillNumber << ": " + << lsMissingInPPS; + edm::LogWarning("transferPayloads") + << "Stable beam LS in OMS without corresponding record in PPS DB (run_LS): " << missingLsList.str(); + } + if (xAngleBothZero > 0) { + edm::LogWarning("transferPayloads") + << "Number of payloads written with crossingAngle == 0 for both X and Y for fill " << fillNumber << ": " + << xAngleBothZero; + } + if (xAngleBothNonZero > 0) { + edm::LogWarning("transferPayloads") + << "Number of payloads written with crossingAngle != 0 for both X and Y for fill " << fillNumber << ": " + << xAngleBothNonZero; + } + if (xAngleNegative > 0) { + edm::LogWarning("transferPayloads") + << "Number of payloads written with negative crossingAngle for fill " << fillNumber << ": " << xAngleNegative; + } + if (betaNegative > 0) { + edm::LogWarning("transferPayloads") + << "Number of payloads written with negative betaSta for fill " << fillNumber << ": " << betaNegative; + } + + edm::LogInfo("transferPayloads") << "TRANSFERED COND IOVS: " << condIovs.str(); + return niovs; + } + +} // namespace theLHCInfoPerLSImpl + +LHCInfoPerLSPopConSourceHandler::LHCInfoPerLSPopConSourceHandler(edm::ParameterSet const& pset) + : m_debug(pset.getUntrackedParameter("debug", false)), + m_startTime(), + m_endTime(), + m_endFillMode(pset.getUntrackedParameter("endFill", true)), + m_name(pset.getUntrackedParameter("name", "LHCInfoPerLSPopConSourceHandler")), + m_connectionString(pset.getUntrackedParameter("connectionString", "")), + m_authpath(pset.getUntrackedParameter("authenticationPath", "")), + m_omsBaseUrl(pset.getUntrackedParameter("omsBaseUrl", "")), + m_debugLogic(pset.getUntrackedParameter("debugLogic", false)), + m_defaultCrossingAngleX(pset.getUntrackedParameter("defaultCrossingAngleX", 0)), + m_defaultCrossingAngleY(pset.getUntrackedParameter("defaultCrossingAngleY", 0)), + m_defaultBetaStarX(pset.getUntrackedParameter("defaultBetaStarX", 0)), + m_defaultBetaStarY(pset.getUntrackedParameter("defaultBetaStarY", 0)), + m_minBetaStar(pset.getUntrackedParameter("minBetaStar", 0.1)), + m_maxBetaStar(pset.getUntrackedParameter("maxBetaStar", 100.)), + m_minCrossingAngle(pset.getUntrackedParameter("minCrossingAngle", 10.)), + m_maxCrossingAngle(pset.getUntrackedParameter("maxCrossingAngle", 500.)), + m_fillPayload(), + m_prevPayload(), + m_tmpBuffer() { + if (!pset.getUntrackedParameter("startTime").empty()) { + m_startTime = boost::posix_time::time_from_string(pset.getUntrackedParameter("startTime")); + } + boost::posix_time::ptime now = boost::posix_time::second_clock::local_time(); + m_endTime = now; + if (!pset.getUntrackedParameter("endTime").empty()) { + m_endTime = boost::posix_time::time_from_string(pset.getUntrackedParameter("endTime")); + if (m_endTime > now) + m_endTime = now; + } + if (m_debugLogic && m_endFillMode) { + throw cms::Exception("invalid argument") << "debugLogic == true not supported for endFillMode == true"; + } +} + +LHCInfoPerLSPopConSourceHandler::~LHCInfoPerLSPopConSourceHandler() = default; + +void LHCInfoPerLSPopConSourceHandler::getNewObjects() { + populateIovs(); + if (!m_endFillMode) { // duringFill mode + filterInvalidPayloads(); + } +} + +void LHCInfoPerLSPopConSourceHandler::filterInvalidPayloads() { + // note: at the moment used only in duringFill mode so the m_iovs is quaranteed to have size() <= 1 + // but iterating through the whole map is implemented just in case the way it's used changes + auto it = m_iovs.begin(); + while (it != m_iovs.end()) { + std::stringstream payloadData; + payloadData << "LS = " << it->second->lumiSection() << ", run = " << it->second->runNumber() << ", " + << "xAngleX = " << it->second->crossingAngleX() << " urad, " + << "xAngleY = " << it->second->crossingAngleY() << " urad, " + << "beta*X = " << it->second->betaStarX() << " m, " + << "beta*Y = " << it->second->betaStarY() << " m"; + if (!isPayloadValid(*(it->second))) { + edm::LogWarning(m_name) << "Skipping upload of payload with invalid values: " << payloadData.str(); + m_iovs.erase(it++); // note: post-increment necessary to avoid using invalidated iterators + } else { + edm::LogInfo(m_name) << "Payload to be uploaded: " << payloadData.str(); + ++it; + } + } +} + +bool LHCInfoPerLSPopConSourceHandler::isPayloadValid(const LHCInfoPerLS& payload) const { + if ((payload.crossingAngleX() == 0. && payload.crossingAngleY() == 0.) || + (payload.crossingAngleX() != 0. && payload.crossingAngleY() != 0.)) + return false; + auto non0CrossingAngle = payload.crossingAngleX() != 0. ? payload.crossingAngleX() : payload.crossingAngleY(); + if ((non0CrossingAngle < m_minCrossingAngle || m_maxCrossingAngle < non0CrossingAngle) || + (payload.betaStarX() < m_minBetaStar || m_maxBetaStar < payload.betaStarX()) || + (payload.betaStarX() < m_minBetaStar || m_maxBetaStar < payload.betaStarX())) + return false; + return true; +} + +void LHCInfoPerLSPopConSourceHandler::populateIovs() { + //if a new tag is created, transfer fake fill from 1 to the first fill for the first time + if (tagInfo().size == 0) { + edm::LogInfo(m_name) << "New tag " << tagInfo().name << "; from " << m_name << "::getNewObjects"; + } else { + //check what is already inside the database + edm::LogInfo(m_name) << "got info for tag " << tagInfo().name << ": size " << tagInfo().size + << ", last object valid since " << tagInfo().lastInterval.since << " ( " + << boost::posix_time::to_iso_extended_string( + cond::time::to_boost(tagInfo().lastInterval.since)) + << " ); from " << m_name << "::getNewObjects"; + } + + cond::Time_t lastSince = tagInfo().lastInterval.since; + if (tagInfo().isEmpty()) { + // for a new or empty tag in endFill mode, an empty payload should be added on top with since=1 + if (m_endFillMode) { + addEmptyPayload(1); + lastSince = 1; + } else { //duringFill mode + edm::LogInfo(m_name) << "Empty or new tag: uploading a default payload and ending the job"; + cond::OMSService oms; + oms.connect(m_omsBaseUrl); + addDefaultPayload(1, 1, 1, 1); + return; + } + } else { + edm::LogInfo(m_name) << "The last Iov in tag " << tagInfo().name << " valid since " << lastSince << "from " + << m_name << "::getNewObjects"; + } + + //retrieve the data from the relational database source + cond::persistency::ConnectionPool connection; + //configure the connection + if (m_debug) { + connection.setMessageVerbosity(coral::Debug); + } else { + connection.setMessageVerbosity(coral::Error); + } + connection.setAuthenticationPath(m_authpath); + connection.configure(); + //create the sessions + cond::persistency::Session session = connection.createSession(m_connectionString, false); + // fetch last payload when available + if (!tagInfo().lastInterval.payloadId.empty()) { + cond::persistency::Session session3 = dbSession(); + session3.transaction().start(true); + m_prevPayload = session3.fetchPayload(tagInfo().lastInterval.payloadId); + session3.transaction().commit(); + + // find startFillTime and endFillTime of the most recent fill already saved in the tag + if (m_prevPayload->fillNumber() != 0) { + cond::OMSService oms; + oms.connect(m_omsBaseUrl); + auto query = oms.query("fills"); + query->addOutputVar("end_time"); + query->addOutputVar("start_time"); + query->filterEQ("fill_number", m_prevPayload->fillNumber()); + bool foundFill = query->execute(); + if (foundFill) { + auto result = query->result(); + + if (!result.empty()) { + std::string endTimeStr = (*result.begin()).get("end_time"); + m_prevEndFillTime = (endTimeStr == "null") + ? 0 + : cond::time::from_boost((*result.begin()).get("end_time")); + auto startFillTime = (*result.begin()).get("start_time"); + m_prevStartFillTime = cond::time::from_boost(startFillTime); + } else { + foundFill = false; + } + } + if (!foundFill) { + edm::LogError(m_name) << "Could not find end time of fill #" << m_prevPayload->fillNumber(); + } + } else { + m_prevEndFillTime = 0; + m_prevStartFillTime = 0; + } + } + + boost::posix_time::ptime executionTime = boost::posix_time::second_clock::local_time(); + cond::Time_t executionTimeIov = cond::time::from_boost(executionTime); + + cond::Time_t startTimestamp = m_startTime.is_not_a_date_time() ? 0 : cond::time::from_boost(m_startTime); + cond::Time_t nextFillSearchTimestamp = std::max(startTimestamp, m_endFillMode ? lastSince : m_prevEndFillTime); + + edm::LogInfo(m_name) << "Starting sampling at " + << boost::posix_time::to_simple_string(cond::time::to_boost(nextFillSearchTimestamp)); + + while (true) { + if (nextFillSearchTimestamp >= executionTimeIov) { + edm::LogInfo(m_name) << "Sampling ended at the time " + << boost::posix_time::to_simple_string(cond::time::to_boost(executionTimeIov)); + break; + } + boost::posix_time::ptime nextFillSearchTime = cond::time::to_boost(nextFillSearchTimestamp); + boost::posix_time::ptime startSampleTime; + boost::posix_time::ptime endSampleTime; + + cond::OMSService oms; + oms.connect(m_omsBaseUrl); + auto query = oms.query("fills"); + + if (m_debugLogic) + m_prevEndFillTime = 0ULL; + + if (!m_endFillMode and m_prevPayload->fillNumber() and m_prevEndFillTime == 0ULL) { + // continue processing unfinished fill with some payloads already in the tag + edm::LogInfo(m_name) << "Searching started fill #" << m_prevPayload->fillNumber(); + query->filterEQ("fill_number", m_prevPayload->fillNumber()); + bool foundFill = query->execute(); + if (foundFill) + foundFill = makeFillPayload(m_fillPayload, query->result()); + if (!foundFill) { + edm::LogError(m_name) << "Could not find fill #" << m_prevPayload->fillNumber(); + break; + } + } else { + edm::LogInfo(m_name) << "Searching new fill after " << boost::posix_time::to_simple_string(nextFillSearchTime); + query->filterNotNull("start_stable_beam").filterNotNull("fill_number"); + if (nextFillSearchTime > cond::time::to_boost(m_prevStartFillTime)) { + query->filterGE("start_time", nextFillSearchTime); + } else { + query->filterGT("start_time", nextFillSearchTime); + } + + query->filterLT("start_time", m_endTime); + if (m_endFillMode) + query->filterNotNull("end_time"); + else + query->filterEQ("end_time", cond::OMSServiceQuery::SNULL); + + bool querySuccess = query->execute(); + if (!querySuccess) { + edm::LogError(m_name) << "OMS fill query failed (http status not 200 nor 201). Request URL:\n" << query->url(); + } + bool foundFill = querySuccess ? makeFillPayload(m_fillPayload, query->result()) : false; + + if (!foundFill) { + if (m_endFillMode) { + edm::LogInfo(m_name) << "No fill found - END of job."; + } else { //duringFill mode + edm::LogInfo(m_name) << "No ongoing fill found."; + addDefaultPayload(1, m_prevPayload->fillNumber(), oms); //IOV doesn't matter here in duringFill mode + } + break; + } + } + startSampleTime = cond::time::to_boost(m_startFillTime); + + unsigned short lhcFill = m_fillPayload->fillNumber(); + bool ongoingFill = m_endFillTime == 0ULL; + if (ongoingFill) { + edm::LogInfo(m_name) << "Found ongoing fill " << lhcFill << " created at " + << cond::time::to_boost(m_startFillTime); + endSampleTime = executionTime; + nextFillSearchTimestamp = executionTimeIov; + } else { + edm::LogInfo(m_name) << "Found fill " << lhcFill << " created at " << cond::time::to_boost(m_startFillTime) + << " ending at " << cond::time::to_boost(m_endFillTime); + endSampleTime = cond::time::to_boost(m_endFillTime); + nextFillSearchTimestamp = m_endFillTime; + } + + if (m_endFillMode || ongoingFill) { + getLumiData(oms, lhcFill, startSampleTime, endSampleTime); + + if (!m_tmpBuffer.empty()) { + boost::posix_time::ptime flumiStart = cond::time::to_boost(m_tmpBuffer.front().first); + boost::posix_time::ptime flumiStop = cond::time::to_boost(m_tmpBuffer.back().first); + edm::LogInfo(m_name) << "First buffered lumi starts at " << flumiStart << " last lumi starts at " << flumiStop; + session.transaction().start(true); + getCTPPSData(session, startSampleTime, endSampleTime); + session.transaction().commit(); + } + } + + if (!m_endFillMode) { + if (m_tmpBuffer.size() > 1) { + throw cms::Exception("LHCInfoPerFillPopConSourceHandler") + << "More than 1 payload buffered for writing in duringFill mode.\ + In this mode only up to 1 payload can be written"; + } else if (m_tmpBuffer.size() == 1) { + if (theLHCInfoPerLSImpl::comparePayloads(*(m_tmpBuffer.begin()->second), *m_prevPayload)) { + m_tmpBuffer.clear(); + edm::LogInfo(m_name) + << "The buffered payload has the same data as the previous payload in the tag. It will not be written."; + } + } else if (m_tmpBuffer.empty()) { + // note: the IOV doesn't matter when using OnlinePopCon: + addDefaultPayload(1, lhcFill, oms); + } + } + + size_t niovs = theLHCInfoPerLSImpl::transferPayloads( + m_tmpBuffer, m_iovs, m_prevPayload, m_lsIdMap, m_startStableBeamTime, m_endStableBeamTime); + edm::LogInfo(m_name) << "Added " << niovs << " iovs within the Fill time"; + m_tmpBuffer.clear(); + m_lsIdMap.clear(); + + if (!m_endFillMode) { + return; + } + + // endFill mode only: + if (niovs) { + m_prevEndFillTime = m_endFillTime; + m_prevStartFillTime = m_startFillTime; + } + if (m_prevPayload->fillNumber() and !ongoingFill) { + if (m_endFillMode) { + addEmptyPayload(m_endFillTime); + } + } + } +} + +std::string LHCInfoPerLSPopConSourceHandler::id() const { return m_name; } + +void LHCInfoPerLSPopConSourceHandler::addEmptyPayload(cond::Time_t iov) { + bool add = false; + if (m_iovs.empty()) { + if (!m_lastPayloadEmpty) + add = true; + } else { + auto lastAdded = m_iovs.rbegin()->second; + if (lastAdded->fillNumber() != 0) { + add = true; + } + } + if (add) { + auto newPayload = std::make_shared(); + m_iovs.insert(make_pair(iov, newPayload)); + m_prevPayload = newPayload; + m_prevEndFillTime = 0; + m_prevStartFillTime = 0; + edm::LogInfo(m_name) << "Added empty payload with IOV" << iov << " ( " + << boost::posix_time::to_iso_extended_string(cond::time::to_boost(iov)) << " )"; + } +} + +void LHCInfoPerLSPopConSourceHandler::addDefaultPayload(cond::Time_t iov, + unsigned short fill, + const cond::OMSService& oms) { + auto defaultPayload = std::make_shared(); + defaultPayload->setFillNumber(fill); + auto [lastRun, lastLumi] = cond::lhcInfoHelper::getFillLastRunAndLS(oms, fill); + addDefaultPayload(iov, fill, lastRun, lastLumi); +} + +void LHCInfoPerLSPopConSourceHandler::addDefaultPayload(cond::Time_t iov, + unsigned short fill, + int run, + unsigned short lumi) { + auto defaultPayload = std::make_shared(); + defaultPayload->setFillNumber(fill); + defaultPayload->setRunNumber(run); + defaultPayload->setLumiSection(lumi); + defaultPayload->setCrossingAngleX(m_defaultCrossingAngleX); + defaultPayload->setCrossingAngleY(m_defaultCrossingAngleY); + defaultPayload->setBetaStarX(m_defaultBetaStarX); + defaultPayload->setBetaStarY(m_defaultBetaStarY); + + if (m_prevPayload && theLHCInfoPerLSImpl::comparePayloads(*defaultPayload, *m_prevPayload)) { + edm::LogInfo(m_name) + << "The default payload has the same data as the previous payload in the tag. It will not be written."; + } else { + m_iovs.insert(make_pair(iov, defaultPayload)); + edm::LogInfo(m_name) << "Uploading the default payload."; + } +} + +bool LHCInfoPerLSPopConSourceHandler::makeFillPayload(std::unique_ptr& targetPayload, + const cond::OMSServiceResult& queryResult) { + bool ret = false; + if (!queryResult.empty()) { + auto row = *queryResult.begin(); + auto currentFill = row.get("fill_number"); + m_startFillTime = cond::time::from_boost(row.get("start_time")); + std::string endTimeStr = row.get("end_time"); + if (m_debugLogic) { + m_endFillTime = 0; + } else { + m_endFillTime = + (endTimeStr == "null") ? 0 : cond::time::from_boost(row.get("end_time")); + } + m_startStableBeamTime = cond::time::from_boost(row.get("start_stable_beam")); + m_endStableBeamTime = cond::time::from_boost(row.get("end_stable_beam")); + targetPayload = std::make_unique(); + targetPayload->setFillNumber(currentFill); + ret = true; + } + return ret; +} + +void LHCInfoPerLSPopConSourceHandler::addPayloadToBuffer(cond::OMSServiceResultRef& row) { + auto lumiTime = row.get("start_time"); + LHCInfoPerLS* thisLumiSectionInfo = new LHCInfoPerLS(*m_fillPayload); + thisLumiSectionInfo->setLumiSection(std::stoul(row.get("lumisection_number"))); + thisLumiSectionInfo->setRunNumber(std::stoul(row.get("run_number"))); + m_lsIdMap[make_pair(thisLumiSectionInfo->runNumber(), thisLumiSectionInfo->lumiSection())] = make_pair(-1, -1); + if (m_endFillMode) { + m_tmpBuffer.emplace_back(make_pair(cond::time::from_boost(lumiTime), thisLumiSectionInfo)); + } else { + m_tmpBuffer.emplace_back( + make_pair(cond::time::lumiTime(thisLumiSectionInfo->runNumber(), thisLumiSectionInfo->lumiSection()), + thisLumiSectionInfo)); + } +} + +size_t LHCInfoPerLSPopConSourceHandler::bufferAllLS(const cond::OMSServiceResult& queryResult) { + for (auto r : queryResult) { + addPayloadToBuffer(r); + } + return queryResult.size(); +} + +size_t LHCInfoPerLSPopConSourceHandler::getLumiData(const cond::OMSService& oms, + unsigned short fillId, + const boost::posix_time::ptime& beginFillTime, + const boost::posix_time::ptime& endFillTime) { + auto query = oms.query("lumisections"); + query->addOutputVars({"start_time", "run_number", "beams_stable", "lumisection_number"}); + query->filterEQ("fill_number", fillId); + query->filterGT("start_time", beginFillTime).filterLT("start_time", endFillTime); + query->limit(cond::lhcInfoHelper::kLumisectionsQueryLimit); + size_t nlumi = 0; + if (query->execute()) { + auto queryResult = query->result(); + if (m_endFillMode) { + nlumi = bufferAllLS(queryResult); + } else if (!queryResult.empty()) { + auto newestPayload = queryResult.back(); + if (newestPayload.get("beams_stable") == "true" || m_debugLogic) { + addPayloadToBuffer(newestPayload); + nlumi = 1; + edm::LogInfo(m_name) << "Buffered most recent lumisection:" + << " LS: " << newestPayload.get("lumisection_number") + << " run: " << newestPayload.get("run_number"); + } + } + edm::LogInfo(m_name) << "Found " << queryResult.size() << " lumisections during the fill " << fillId; + } else { + edm::LogInfo(m_name) << "OMS query for lumisections of fill " << fillId << "failed, status:" << query->status(); + } + return nlumi; +} +bool LHCInfoPerLSPopConSourceHandler::getCTPPSData(cond::persistency::Session& session, + const boost::posix_time::ptime& beginFillTime, + const boost::posix_time::ptime& endFillTime) { + //run the fifth query against the CTPPS schema + //Initializing the CMS_CTP_CTPPS_COND schema. + coral::ISchema& CTPPS = session.coralSession().schema("CMS_PPS_SPECT_COND"); + //execute query for CTPPS Data + std::unique_ptr CTPPSDataQuery(CTPPS.newQuery()); + //FROM clause + CTPPSDataQuery->addToTableList(std::string("PPS_LHC_MACHINE_PARAMS")); + //SELECT clause + CTPPSDataQuery->addToOutputList(std::string("DIP_UPDATE_TIME")); + CTPPSDataQuery->addToOutputList(std::string("LUMI_SECTION")); + CTPPSDataQuery->addToOutputList(std::string("RUN_NUMBER")); + CTPPSDataQuery->addToOutputList(std::string("FILL_NUMBER")); + CTPPSDataQuery->addToOutputList(std::string("XING_ANGLE_P5_X_URAD")); + CTPPSDataQuery->addToOutputList(std::string("XING_ANGLE_P5_Y_URAD")); + CTPPSDataQuery->addToOutputList(std::string("BETA_STAR_P5_X_M")); + CTPPSDataQuery->addToOutputList(std::string("BETA_STAR_P5_Y_M")); + //WHERE CLAUSE + coral::AttributeList CTPPSDataBindVariables; + CTPPSDataBindVariables.extend(std::string("beginFillTime")); + CTPPSDataBindVariables.extend(std::string("endFillTime")); + CTPPSDataBindVariables[std::string("beginFillTime")].data() = coral::TimeStamp(beginFillTime); + CTPPSDataBindVariables[std::string("endFillTime")].data() = coral::TimeStamp(endFillTime); + std::string conditionStr = std::string("DIP_UPDATE_TIME>= :beginFillTime and DIP_UPDATE_TIME< :endFillTime"); + CTPPSDataQuery->setCondition(conditionStr, CTPPSDataBindVariables); + //ORDER BY clause + CTPPSDataQuery->addToOrderList(std::string("DIP_UPDATE_TIME")); + //define query output + coral::AttributeList CTPPSDataOutput; + CTPPSDataOutput.extend(std::string("DIP_UPDATE_TIME")); + CTPPSDataOutput.extend(std::string("LUMI_SECTION")); + CTPPSDataOutput.extend(std::string("RUN_NUMBER")); + CTPPSDataOutput.extend(std::string("FILL_NUMBER")); + CTPPSDataOutput.extend(std::string("XING_ANGLE_P5_X_URAD")); + CTPPSDataOutput.extend(std::string("XING_ANGLE_P5_Y_URAD")); + CTPPSDataOutput.extend(std::string("BETA_STAR_P5_X_M")); + CTPPSDataOutput.extend(std::string("BETA_STAR_P5_Y_M")); + CTPPSDataQuery->defineOutput(CTPPSDataOutput); + //execute the query + coral::ICursor& CTPPSDataCursor = CTPPSDataQuery->execute(); + unsigned int lumiSection = 0; + cond::Time_t runNumber = 0; + int fillNumber = 0; + float crossingAngleX = 0., betaStarX = 0.; + float crossingAngleY = 0., betaStarY = 0.; + + bool ret = false; + int wrongFillNumbers = 0; + std::stringstream wrongFills; + std::vector>>::iterator current = m_tmpBuffer.begin(); + while (CTPPSDataCursor.next()) { + if (m_debug) { + std::ostringstream CTPPS; + CTPPSDataCursor.currentRow().toOutputStream(CTPPS); + } + coral::Attribute const& dipTimeAttribute = CTPPSDataCursor.currentRow()[std::string("DIP_UPDATE_TIME")]; + if (!dipTimeAttribute.isNull()) { + ret = true; + coral::Attribute const& lumiSectionAttribute = CTPPSDataCursor.currentRow()[std::string("LUMI_SECTION")]; + if (!lumiSectionAttribute.isNull()) { + lumiSection = lumiSectionAttribute.data(); + } + coral::Attribute const& runNumberAttribute = CTPPSDataCursor.currentRow()[std::string("RUN_NUMBER")]; + if (!runNumberAttribute.isNull()) { + runNumber = runNumberAttribute.data(); + } + coral::Attribute const& fillNumberAttribute = CTPPSDataCursor.currentRow()[std::string("FILL_NUMBER")]; + if (!fillNumberAttribute.isNull()) { + fillNumber = fillNumberAttribute.data(); + } + coral::Attribute const& crossingAngleXAttribute = + CTPPSDataCursor.currentRow()[std::string("XING_ANGLE_P5_X_URAD")]; + if (!crossingAngleXAttribute.isNull()) { + crossingAngleX = crossingAngleXAttribute.data(); + } + coral::Attribute const& crossingAngleYAttribute = + CTPPSDataCursor.currentRow()[std::string("XING_ANGLE_P5_Y_URAD")]; + if (!crossingAngleYAttribute.isNull()) { + crossingAngleY = crossingAngleYAttribute.data(); + } + coral::Attribute const& betaStarXAttribute = CTPPSDataCursor.currentRow()[std::string("BETA_STAR_P5_X_M")]; + if (!betaStarXAttribute.isNull()) { + betaStarX = betaStarXAttribute.data(); + } + coral::Attribute const& betaStarYAttribute = CTPPSDataCursor.currentRow()[std::string("BETA_STAR_P5_Y_M")]; + if (!betaStarYAttribute.isNull()) { + betaStarY = betaStarYAttribute.data(); + } + if (current != m_tmpBuffer.end() && current->second->fillNumber() != fillNumber) { + wrongFills << "( " << runNumber << "_" << lumiSection << " fill: OMS: " << current->second->fillNumber() + << " PPSdb: " << fillNumber << " ) "; + wrongFillNumbers++; + } + for (; current != m_tmpBuffer.end() && make_pair(current->second->runNumber(), current->second->lumiSection()) <= + make_pair(runNumber, lumiSection); + current++) { + LHCInfoPerLS& payload = *(current->second); + payload.setCrossingAngleX(crossingAngleX); + payload.setCrossingAngleY(crossingAngleY); + payload.setBetaStarX(betaStarX); + payload.setBetaStarY(betaStarY); + payload.setLumiSection(lumiSection); + payload.setRunNumber(runNumber); + if (m_lsIdMap.find(make_pair(payload.runNumber(), payload.lumiSection())) != m_lsIdMap.end()) { + m_lsIdMap[make_pair(payload.runNumber(), payload.lumiSection())] = make_pair(runNumber, lumiSection); + } + } + } + } + if (wrongFillNumbers) { + edm::LogWarning("getCTPPSData") << "Number of records from PPS DB with fillNumber different from OMS: " + << wrongFillNumbers; + edm::LogWarning("getCTPPSData") << "Records from PPS DB with fillNumber different from OMS: " << wrongFills.str(); + } + return ret; +} diff --git a/CondTools/RunInfo/src/LHCInfoPopConSourceHandler.cc b/CondTools/RunInfo/src/LHCInfoPopConSourceHandler.cc index 0a344bd479711..e32604bbd20b2 100644 --- a/CondTools/RunInfo/src/LHCInfoPopConSourceHandler.cc +++ b/CondTools/RunInfo/src/LHCInfoPopConSourceHandler.cc @@ -1,5 +1,6 @@ #include "CondCore/CondDB/interface/ConnectionPool.h" #include "CondFormats/Common/interface/TimeConversions.h" +#include "CondTools/RunInfo/interface/LHCInfoHelper.h" #include "CondTools/RunInfo/interface/LHCInfoPopConSourceHandler.h" #include "CondTools/RunInfo/interface/LumiSectionFilter.h" #include "CondTools/RunInfo/interface/OMSAccess.h" @@ -159,7 +160,7 @@ size_t LHCInfoPopConSourceHandler::getLumiData(const cond::OMSService& oms, query->addOutputVars({"start_time", "delivered_lumi", "recorded_lumi"}); query->filterEQ("fill_number", fillId); query->filterGT("start_time", beginFillTime).filterLT("start_time", endFillTime); - query->limit(kLumisectionsQueryLimit); + query->limit(cond::lhcInfoHelper::kLumisectionsQueryLimit); size_t nlumi = 0; if (query->execute()) { auto res = query->result(); diff --git a/CondTools/RunInfo/test/last_lumi.txt b/CondTools/RunInfo/test/last_lumi.txt new file mode 100644 index 0000000000000..c4ed12a727c79 --- /dev/null +++ b/CondTools/RunInfo/test/last_lumi.txt @@ -0,0 +1 @@ +1663505258250241 diff --git a/CondTools/RunInfo/test/test_lhcInfoNewPopCon.sh b/CondTools/RunInfo/test/test_lhcInfoNewPopCon.sh index 750e1501f07a4..8ef3751949388 100755 --- a/CondTools/RunInfo/test/test_lhcInfoNewPopCon.sh +++ b/CondTools/RunInfo/test/test_lhcInfoNewPopCon.sh @@ -2,15 +2,24 @@ SCRIPTS_DIR=${CMSSW_BASE}/src/CondTools/RunInfo/python -function die { echo Failure $1: status $2 ; exit $2 ; } +function die { + log_file="$3" + if [ -f "$log_file" ]; then + echo "Log output:" + cat "$log_file" + fi + echo "Failure $1: status $2" + exit $2 +} assert_equal() { expected="$1" actual="$2" message="$3" + log_file="$4" if [ "$expected" != "$actual" ]; then - die "$message: Expected $expected, but got $actual" 1 + die "$message: Expected $expected, but got $actual" 1 "$log_file" fi } @@ -20,7 +29,7 @@ function assert_found_fills { shift 2 for fill_nr in "$@"; do if ! grep -q "Found fill $fill_nr" "$log_file"; then - die "$script_name didn't find fill $fill_nr" 1 # TODO FIX + die "$script_name didn't find fill $fill_nr" 1 "$log_file" fi done } @@ -31,43 +40,45 @@ echo "testing LHCInfoPerFillPopConAnalyzer in endFill mode for startTime=\"2022- cmsRun ${SCRIPTS_DIR}/LHCInfoPerFillPopConAnalyzer_cfg.py mode=endFill \ destinationConnection="sqlite_file:lhcinfo_pop_unit_test.db" \ startTime="2022-10-24 01:00:00.000" endTime="2022-10-24 20:00:00.000" \ - tag=fill_end_test > fill_end_test.log || die "cmsRun LHCInfoPerFillPopConAnalyzer_cfg.py mode=endFill" $? + tag=fill_end_test > fill_end_test.log || die "cmsRun LHCInfoPerFillPopConAnalyzer_cfg.py mode=endFill" $? "fill_end_test.log" assert_equal 7 `cat fill_end_test.log | grep -E '^Since ' | \ - wc -l` "LHCInfoPerFillPopConAnalyzer in EndFill mode written wrong number of payloads" + wc -l` "LHCInfoPerFillPopConAnalyzer in EndFill mode written wrong number of payloads" "fill_end_test.log" assert_found_fills fill_end_test.log "LHCInfoPerFillPopConAnalyzer in EndFill mode" 8307 8309 echo "testing LHCInfoPerLSPopConAnalyzer in endFill mode for startTime=\"2022-10-24 01:00:00.000\" endTime=\"2022-10-24 20:00:00.000\"" cmsRun ${SCRIPTS_DIR}/LHCInfoPerLSPopConAnalyzer_cfg.py mode=endFill \ destinationConnection="sqlite_file:lhcinfo_pop_unit_test.db" \ startTime="2022-10-24 01:00:00.000" endTime="2022-10-24 20:00:00.000" \ - tag=ls_end_test > ls_end_test.log || die "cmsRun LHCInfoPerLSPopConAnalyzer_cfg.py mode=endFill" $? + tag=ls_end_test > ls_end_test.log || die "cmsRun LHCInfoPerLSPopConAnalyzer_cfg.py mode=endFill" $? "ls_end_test.log" assert_equal 169 `cat ls_end_test.log | grep -E '^Since ' | \ - wc -l` "LHCInfoPerLSPopConAnalyzer in endFill mode written wrong number of payloads" + wc -l` "LHCInfoPerLSPopConAnalyzer in endFill mode written wrong number of payloads" "ls_end_test.log" assert_found_fills ls_end_test.log "LHCInfoPerLSPopConAnalyzer in endFill mode" 8307 8309 echo "testing LHCInfoPerLSPopConAnalyzer in endFill mode for startTime=\"2022-07-11 22:00:00.000\" endTime=\"2022-07-12 18:10:10.000\"" cmsRun ${SCRIPTS_DIR}/LHCInfoPerLSPopConAnalyzer_cfg.py mode=endFill \ destinationConnection="sqlite_file:lhcinfo_pop_unit_test.db" \ startTime="2022-07-11 22:00:00.000" endTime="2022-07-12 18:10:10.000" \ - tag=ls_end_test2 > ls_end_test2.log || die "cmsRun LHCInfoPerLSPopConAnalyzer_cfg.py mode=endFill" $? + tag=ls_end_test2 > ls_end_test2.log || die "cmsRun LHCInfoPerLSPopConAnalyzer_cfg.py mode=endFill" $? "ls_end_test2.log" assert_equal 70 `cat ls_end_test2.log | grep -E '^Since ' | \ - wc -l` "LHCInfoPerLSPopConAnalyzer in endFill mode written wrong number of payloads" + wc -l` "LHCInfoPerLSPopConAnalyzer in endFill mode written wrong number of payloads" "ls_end_test2.log" assert_found_fills ls_end_test2.log "LHCInfoPerLSPopConAnalyzer in endFill mode" 7967 +echo "1663505258250241" > last_lumi.txt + echo "testing LHCInfoPerFillPopConAnalyzer in duringFill mode for startTime=\"2022-10-24 01:00:00.000\" endTime=\"2022-10-24 20:00:00.000\"" cmsRun ${SCRIPTS_DIR}/LHCInfoPerFillPopConAnalyzer_cfg.py mode=duringFill \ destinationConnection="sqlite_file:lhcinfo_pop_unit_test.db" \ startTime="2022-10-24 01:00:00.000" endTime="2022-10-24 20:00:00.000" \ - tag=fill_during_test > fill_during_test.log || die "cmsRun LHCInfoPerFillPopConAnalyzer_cfg.py" $? -assert_equal 1 `cat fill_during_test.log | grep -E '^Since ' | \ - wc -l` "LHCInfoPerFillPopConAnalyzer in DuringFill written wrong number of payloads" -assert_found_fills fill_during_test.log "LHCInfoPerFillPopConAnalyzer in DuringFill" 8307 8309 + lastLumiFile=last_lumi.txt \ + tag=fill_during_test > fill_during_test.log || die "cmsRun LHCInfoPerFillPopConAnalyzer_cfg.py" $? "fill_during_test.log" +assert_equal 1 `cat fill_during_test.log | grep -E 'uploaded with since' | \ + wc -l` "LHCInfoPerFillPopConAnalyzer in DuringFill written wrong number of payloads" "fill_during_test.log" echo "testing LHCInfoPerLSPopConAnalyzer in duringFill mode for startTime=\"2022-10-24 01:00:00.000\" endTime=\"2022-10-24 20:00:00.000\"" cmsRun ${SCRIPTS_DIR}/LHCInfoPerLSPopConAnalyzer_cfg.py mode=duringFill \ destinationConnection="sqlite_file:lhcinfo_pop_unit_test.db" \ + lastLumiFile=last_lumi.txt \ startTime="2022-10-24 01:00:00.000" endTime="2022-10-24 20:00:00.000" \ - tag=ls_during_test > ls_during_test.log || die "cmsRun LHCInfoPerLSPopConAnalyzer_cfg.py mode=duringFill" $? -assert_equal 1 `cat ls_during_test.log | grep -E '^Since ' | \ - wc -l` "LHCInfoPerLSPopConAnalyzer in duringFill mode written wrong number of payloads" -assert_found_fills ls_during_test.log "LHCInfoPerLSPopConAnalyzer in duringFill mode" 8307 8309 + tag=ls_during_test > ls_during_test.log || die "cmsRun LHCInfoPerLSPopConAnalyzer_cfg.py mode=duringFill" $? "ls_during_test.log" +assert_equal 1 `cat ls_during_test.log | grep -E 'uploaded with since' | \ + wc -l` "LHCInfoPerLSPopConAnalyzer in duringFill mode written wrong number of payloads" "ls_during_test.log"