HLT-compatible LHCInfoPer* PopCon (duringFill mode)

adjust LHCInfoPer*PopConSourceHandler duringFill mode for HLT-compatibility

fix LHCInfoPerLS PopCon duringFill startSampleTime error

fix filtering in  duringFill mode (prevPayload error)

Remove adding empty payload to empty tag

Extract SourceHandler from LHCInfoPer*PopConAnalyzer

Make perLS and perFill duringFill use OnlinePopCon

Attempt to configure OnlineDBOutputService in LHCInfoPer*PopConAnalyzer_cfg.py

finish LHCInfoPer*PopConAnalyzer_cfg files for updated duringFill

For perLS duringfill implement adding defaultPayload and skipping invalid payloads

implement adding defaultPayload in duringFill mode

duringFill perLS: skip upload if invalid values

Adjust duringFill unit tests

remove duplicated kLumisectionsQueryLimit

fix unit tests, print logs of failed lhcInfoNewPopCon test

CondTools/RunInfo/interface/LHCInfoHelper.h

@@ -11,6 +11,9 @@ namespace cond {
     // Large number of LS for the OMS query, covering around 25 hours
     static constexpr unsigned int kLumisectionsQueryLimit = 4000;
 
+    // last Run number and LS number of the specified Fill
+    std::pair<int, unsigned short> getFillLastRunAndLS(const cond::OMSService& oms, unsigned short fillId);
+
     // Returns lumi-type IOV from last LS of last Run of the specified Fill
     cond::Time_t getFillLastLumiIOV(const cond::OMSService& oms, unsigned short fillId);
 

CondTools/RunInfo/interface/LHCInfoPerFillPopConSourceHandler.h

@@ -0,0 +1,55 @@
+#include "CondCore/PopCon/interface/PopConSourceHandler.h"
+#include "CondFormats/RunInfo/interface/LHCInfoPerFill.h"
+#include "CondTools/RunInfo/interface/OMSAccess.h"
+#include "FWCore/ParameterSet/interface/ParameterSet.h"
+
+class LHCInfoPerFillPopConSourceHandler : public popcon::PopConSourceHandler<LHCInfoPerFill> {
+public:
+  LHCInfoPerFillPopConSourceHandler(edm::ParameterSet const& pset);
+  ~LHCInfoPerFillPopConSourceHandler() override = default;
+
+  void getNewObjects() override;
+  std::string id() const override;
+
+private:
+  void addEmptyPayload(cond::Time_t iov);
+
+  // Add payload to buffer and store corresponding lumiid IOV in m_timestampToLumiid map
+  void addPayloadToBuffer(cond::OMSServiceResultRef& row);
+  void convertBufferedIovsToLumiid(std::map<cond::Time_t, cond::Time_t> timestampToLumiid);
+
+  size_t getLumiData(const cond::OMSService& oms,
+                     unsigned short fillId,
+                     const boost::posix_time::ptime& beginFillTime,
+                     const boost::posix_time::ptime& endFillTime);
+
+  void getDipData(const cond::OMSService& oms,
+                  const boost::posix_time::ptime& beginFillTime,
+                  const boost::posix_time::ptime& endFillTime);
+
+  bool getCTPPSData(cond::persistency::Session& session,
+                    const boost::posix_time::ptime& beginFillTime,
+                    const boost::posix_time::ptime& endFillTime);
+
+  bool getEcalData(cond::persistency::Session& session,
+                   const boost::posix_time::ptime& lowerTime,
+                   const boost::posix_time::ptime& upperTime);
+
+private:
+  bool m_debug;
+  // starting date for sampling
+  boost::posix_time::ptime m_startTime;
+  boost::posix_time::ptime m_endTime;
+  bool m_endFillMode = true;
+  std::string m_name;
+  //for reading from relational database source
+  std::string m_connectionString, m_ecalConnectionString;
+  std::string m_authpath;
+  std::string m_omsBaseUrl;
+  std::unique_ptr<LHCInfoPerFill> m_fillPayload;
+  std::shared_ptr<LHCInfoPerFill> m_prevPayload;
+  std::vector<std::pair<cond::Time_t, std::shared_ptr<LHCInfoPerFill>>> m_tmpBuffer;
+  bool m_lastPayloadEmpty = false;
+  // to hold correspondance between timestamp-type IOVs and lumiid-type IOVs
+  std::map<cond::Time_t, cond::Time_t> m_timestampToLumiid;
+};

CondTools/RunInfo/interface/LHCInfoPerLSPopConSourceHandler.h

@@ -0,0 +1,82 @@
+#ifndef LHCInfoPerLSPopConSourceHandler_h
+#define LHCInfoPerLSPopConSourceHandler_h
+
+#include "CondCore/PopCon/interface/PopConSourceHandler.h"
+#include "CondFormats/RunInfo/interface/LHCInfoPerLS.h"
+#include "FWCore/ParameterSet/interface/ParameterSet.h"
+#include "CondTools/RunInfo/interface/OMSAccess.h"
+#include <map>
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+class LHCInfoPerLSPopConSourceHandler : public popcon::PopConSourceHandler<LHCInfoPerLS> {
+public:
+  LHCInfoPerLSPopConSourceHandler(edm::ParameterSet const& pset);
+  ~LHCInfoPerLSPopConSourceHandler() override;
+  void getNewObjects() override;
+  std::string id() const override;
+
+private:
+  void populateIovs();
+  void filterInvalidPayloads();
+  bool isPayloadValid(const LHCInfoPerLS& payload) const;
+  void addEmptyPayload(cond::Time_t iov);
+  void addDefaultPayload(cond::Time_t iov, unsigned short fill, const cond::OMSService& oms);
+  void addDefaultPayload(cond::Time_t iov, unsigned short fill, int run, unsigned short lumi);
+  bool makeFillPayload(std::unique_ptr<LHCInfoPerLS>& targetPayload, const cond::OMSServiceResult& queryResult);
+  void addPayloadToBuffer(cond::OMSServiceResultRef& row);
+  size_t bufferAllLS(const cond::OMSServiceResult& queryResult);
+  size_t getLumiData(const cond::OMSService& oms,
+                     unsigned short fillId,
+                     const boost::posix_time::ptime& beginFillTime,
+                     const boost::posix_time::ptime& endFillTime);
+  bool getCTPPSData(cond::persistency::Session& session,
+                    const boost::posix_time::ptime& beginFillTime,
+                    const boost::posix_time::ptime& endFillTime);
+
+  bool m_debug;
+  // starting date for sampling
+  boost::posix_time::ptime m_startTime;
+  boost::posix_time::ptime m_endTime;
+  bool m_endFillMode = true;
+  std::string m_name;
+  // for reading from relational database source
+  std::string m_connectionString;
+  std::string m_authpath;
+  std::string m_omsBaseUrl;
+  // Allows for basic test of durigFill mode when there is no Stable Beams in LHC
+  // makes duringFill interpret fills as ongoing fill and writing their last LS
+  // (disabling the check if the last LS is in stable beams,
+  // although still only fills with stable beams are being processed
+  // also, still only up to one payload will be written)
+  const bool m_debugLogic;
+  // values for the default payload which is inserted after the last processed fill
+  // has ended and there's no ongoing stable beam yet:
+  float m_defaultCrossingAngleX;
+  float m_defaultCrossingAngleY;
+  float m_defaultBetaStarX;
+  float m_defaultBetaStarY;
+  float m_minBetaStar;       // meters
+  float m_maxBetaStar;       // meters
+  float m_minCrossingAngle;  // urad
+  float m_maxCrossingAngle;  // urad
+
+  std::unique_ptr<LHCInfoPerLS> m_fillPayload;
+  std::shared_ptr<LHCInfoPerLS> m_prevPayload;
+  cond::Time_t m_startFillTime;
+  cond::Time_t m_endFillTime;
+  cond::Time_t m_prevEndFillTime = 0;
+  cond::Time_t m_prevStartFillTime;
+  cond::Time_t m_startStableBeamTime;
+  cond::Time_t m_endStableBeamTime;
+  std::vector<std::pair<cond::Time_t, std::shared_ptr<LHCInfoPerLS>>> m_tmpBuffer;
+  bool m_lastPayloadEmpty = false;
+  // mapping of lumisections IDs (pairs of runnumber an LS number) found in OMS to
+  // the IDs they've been assignd from PPS DB value pair(-1, -1) means lumisection
+  // corresponding to the key exists in OMS but no lumisection was matched from PPS
+  std::map<std::pair<cond::Time_t, unsigned int>, std::pair<cond::Time_t, unsigned int>> m_lsIdMap;
+};
+
+#endif

CondTools/RunInfo/interface/LHCInfoPopConSourceHandler.h

@@ -18,8 +18,6 @@ class LHCInfoPopConSourceHandler : public popcon::PopConSourceHandler<LHCInfo> {
   void getNewObjects() override;
   std::string id() const override;
 
-  static constexpr unsigned int kLumisectionsQueryLimit = 4000;  // enough for fills not exceeding 25h
-
 private:
   void addEmptyPayload(cond::Time_t iov);
 

CondTools/RunInfo/plugins/BuildFile.xml

@@ -59,6 +59,10 @@
   <flags EDM_PLUGIN="1"/>
 </library>
 
+<library file="LHCInfoPerLSWriter.cc" name="LHCInfoPerLSWriter">
+  <flags EDM_PLUGIN="1"/>
+</library>
+
 <library file="LHCInfoPerFillAnalyzer.cc" name="LHCInfoPerFillAnalyzer">
   <flags EDM_PLUGIN="1"/>
 </library>
@@ -67,10 +71,14 @@
   <flags EDM_PLUGIN="1"/>
 </library>
 
+<library file="LHCInfoPerLSOnlinePopConAnalyzer.cc" name="CondToolsLHCInfoPerLSOnlinePopConAnalyzer">
+  <flags EDM_PLUGIN="1"/>
+</library>
+
 <library file="LHCInfoPerFillPopConAnalyzer.cc" name="CondToolsLHCInfoPerFillPopConAnalyzer">
   <flags EDM_PLUGIN="1"/>
 </library>
 
-<library file="LHCInfoPerLSWriter.cc" name="LHCInfoPerLSWriter">
+<library file="LHCInfoPerFillOnlinePopConAnalyzer.cc" name="CondToolsLHCInfoPerFillOnlinePopConAnalyzer">
   <flags EDM_PLUGIN="1"/>
 </library>

CondTools/RunInfo/plugins/LHCInfoPerFillOnlinePopConAnalyzer.cc

@@ -0,0 +1,7 @@
+#include "CondCore/PopCon/interface/OnlinePopConAnalyzer.h"
+#include "CondTools/RunInfo/interface/LHCInfoPerFillPopConSourceHandler.h"
+#include "FWCore/Framework/interface/MakerMacros.h"
+
+typedef popcon::OnlinePopConAnalyzer<LHCInfoPerFillPopConSourceHandler> LHCInfoPerFillOnlinePopConAnalyzer;
+
+DEFINE_FWK_MODULE(LHCInfoPerFillOnlinePopConAnalyzer);