Merge pull request #3034 from ControlSystemStudio/archive-datasource

Archive datasource

app/databrowser/pom.xml

@@ -56,24 +56,19 @@
     </dependency>
     <dependency>
       <groupId>org.phoebus</groupId>
-      <artifactId>app-rtplot</artifactId>
+      <artifactId>app-trends-archive-reader</artifactId>
       <version>4.7.4-SNAPSHOT</version>
     </dependency>
     <dependency>
-      <groupId>com.google.protobuf</groupId>
-      <artifactId>protobuf-java</artifactId>
-      <version>3.21.9</version>
+      <groupId>org.phoebus</groupId>
+      <artifactId>app-rtplot</artifactId>
+      <version>4.7.4-SNAPSHOT</version>
     </dependency>
     <dependency>
       <groupId>org.epics</groupId>
       <artifactId>epics-util</artifactId>
       <version>${epics.util.version}</version>
     </dependency>
-    <dependency>
-      <groupId>org.epics</groupId>
-      <artifactId>pbrawclient</artifactId>
-      <version>0.0.10</version>
-    </dependency>
     <!-- https://mvnrepository.com/artifact/org.apache.commons/commons-math3 -->
     <dependency>
       <groupId>org.apache.commons</groupId>

app/databrowser/src/main/java/org/csstudio/trends/databrowser3/imports/ImportArchiveReaderFactory.java

@@ -7,12 +7,12 @@
  ******************************************************************************/
 package org.csstudio.trends.databrowser3.imports;
 
+import org.phoebus.archive.reader.ArchiveReader;
+import org.phoebus.archive.reader.spi.ArchiveReaderFactory;
+
 import java.util.Collection;
 import java.util.concurrent.ConcurrentHashMap;
-
 import org.csstudio.trends.databrowser3.model.ArchiveDataSource;
-import org.phoebus.archive.reader.ArchiveReader;
-import org.phoebus.archive.reader.spi.ArchiveReaderFactory;
 
 /** Factory for {@link ArchiveReader} that imports data from file
  *  @author Kay Kasemir

app/databrowser/src/main/java/org/csstudio/trends/databrowser3/preferences/Preferences.java

@@ -88,6 +88,7 @@ public static class TimePreset
     /** Setting */
     @Preference public static boolean drop_failed_archives;
     /** Setting */
+    @Deprecated
     @Preference public static String[]  equivalent_pv_prefixes;
     /** Setting */
     @Preference public static boolean use_trace_names;

app/databrowser/src/main/resources/META-INF/services/org.phoebus.archive.reader.spi.ArchiveReaderFactory

@@ -1,5 +1 @@
-org.phoebus.archive.reader.appliance.ApplianceArchiveReaderFactory
-org.phoebus.archive.reader.rdb.RDBArchiveReaderFactory
-org.phoebus.archive.reader.channelarchiver.XMLRPCArchiveReaderFactory
-org.phoebus.archive.reader.channelarchiver.file.ArchiveFileReaderFactory
 org.csstudio.trends.databrowser3.imports.ImportArchiveReaderFactory

app/trends/archive-datasource/build.xml

@@ -0,0 +1,19 @@
+<project default="app-trends-rich-adapters">
+  <import file="../../../dependencies/ant_settings.xml"/>
+
+  <target name="app-trends-archive-datasource">
+    <mkdir dir="${classes}"/>
+    <javac destdir="${classes}" debug="${debug}">
+      <src path="${src}"/>
+      <classpath>
+        <path refid="app-classpath"/>
+      </classpath>
+    </javac>
+
+    <jar destfile="${build}/app-trends-archive-datasource-${version}.jar">
+      <fileset dir="${classes}"/>
+      <fileset dir="${resources}"/>
+    </jar>
+  </target>
+
+</project>

app/trends/archive-datasource/doc/index.rst

@@ -0,0 +1,29 @@
+Archive Datasources
+===================
+
+Overview
+--------
+The archive datasources allow accessing historical data as a PV. There are two types of archive datasources:
+
+1. `archive`: Retrieves a single archived value at a particular instant of time.
+2. `replay`: Creates a PV that recreates changes in values based on data from the archive.
+
+Archive PV Syntax
+-----------------
+
+The prefix for the datasource is ``archive://``, which can be omitted if configured as the default datasource.
+The archiver PVs are read-only and constant.
+
+- `archive://pv_name`: Retrieves the latest value in the archiver.
+- `archive://pv_name(time)`: Retrieves the last value at or before the specified "time".
+
+Replay PV Syntax
+----------------
+
+The prefix for this datasource is ``replay://``.
+The replay PVs are read-only and constant.
+
+- `replay://pv_name`: Retrieves the last 5 minutes of data for this PV from the archiver and replays them at 10Hz.
+- `replay://pv_name(start, end)`: Recreates the PV value changes using the data from the archiver between the specific start and end times.
+- `replay://pv_name(start, end, update_rate)`: Recreates the PV value changes using the data from the archiver between the specified start and end times. Updates occur at the rate specified by `update_rate` (a value defined in seconds).
+

app/trends/archive-datasource/pom.xml

@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>app-trends</artifactId>
+        <groupId>org.phoebus</groupId>
+        <version>4.7.4-SNAPSHOT</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+    <artifactId>app-trends-archive-datasource</artifactId>
+
+    <properties>
+        <maven.compiler.source>17</maven.compiler.source>
+        <maven.compiler.target>17</maven.compiler.target>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.phoebus</groupId>
+            <artifactId>app-trends-archive-reader</artifactId>
+            <version>4.7.4-SNAPSHOT</version>
+        </dependency>
+    </dependencies>
+</project>

app/trends/archive-datasource/src/main/java/org/phoebus/pv/archive/ArchiveReaderService.java

@@ -0,0 +1,32 @@
+package org.phoebus.pv.archive;
+
+import org.phoebus.archive.reader.ArchiveReader;
+import org.phoebus.archive.reader.appliance.ApplianceArchiveReader;
+
+public class ArchiveReaderService {
+
+    /**
+     * Singleton
+     */
+    private static final ArchiveReaderService INSTANCE = new ArchiveReaderService();
+
+    private final ArchiveReader reader;
+
+    public static ArchiveReaderService getService() {
+        return INSTANCE;
+    }
+
+    private ArchiveReaderService() {
+        // Might have to add support for multiple AA URL's
+        reader = createReader(Preferences.archive_url);
+    }
+
+    private ArchiveReader createReader(final String url) {
+        final ApplianceArchiveReader reader = new ApplianceArchiveReader(url, false, true);
+        return reader;
+    }
+
+    public ArchiveReader getReader() {
+        return reader;
+    }
+}

app/trends/archive-datasource/src/main/java/org/phoebus/pv/archive/ArchiveReaderUtil.java

@@ -0,0 +1,40 @@
+package org.phoebus.pv.archive;
+
+import java.time.DateTimeException;
+import java.time.Instant;
+
+import static org.phoebus.util.time.TimestampFormats.DATETIME_FORMAT;
+import static org.phoebus.util.time.TimestampFormats.FULL_FORMAT;
+import static org.phoebus.util.time.TimestampFormats.MILLI_FORMAT;
+import static org.phoebus.util.time.TimestampFormats.SECONDS_FORMAT;
+
+public class ArchiveReaderUtil {
+
+    /**
+     * A utility method to parse a subset of supported time formats used by the archive datasources
+     * Support formats include
+     *     FULL_PATTERN = "yyyy-MM-dd HH:mm:ss.nnnnnnnnn";
+     *     MILLI_PATTERN = "yyyy-MM-dd HH:mm:ss.SSS";
+     *     SECONDS_PATTERN = "yyyy-MM-dd HH:mm:ss";
+     *     DATETIME_PATTERN = "yyyy-MM-dd HH:mm";
+     *
+     * @param timeValue
+     * @return
+     */
+    public static Instant parseSupportedTimeFormat(String timeValue) throws Exception {
+        Instant time;
+        try {
+            switch (timeValue.length()) {
+                case 16 -> time = Instant.from(DATETIME_FORMAT.parse(timeValue));
+                case 19 -> time = Instant.from(SECONDS_FORMAT.parse(timeValue));
+                case 23 -> time = Instant.from(MILLI_FORMAT.parse(timeValue));
+                case 29 -> time = Instant.from(FULL_FORMAT.parse(timeValue));
+                default -> throw new Exception("Time value defined in a unknown format, '" + timeValue + "'");
+            }
+        } catch (
+                DateTimeException e) {
+            throw new Exception("Time value defined in a unknown format, '" + timeValue + "'");
+        }
+        return time;
+    }
+}

app/trends/archive-datasource/src/main/java/org/phoebus/pv/archive/Preferences.java

@@ -0,0 +1,23 @@
+package org.phoebus.pv.archive;
+
+import org.phoebus.framework.preferences.AnnotatedPreferences;
+import org.phoebus.framework.preferences.Preference;
+import org.phoebus.framework.preferences.PreferencesReader;
+
+/** Helper for reading preference settings
+ *
+ *  @author Kunal Shroff
+ */
+@SuppressWarnings("nls")
+public class Preferences
+{
+
+    /** Setting */
+    @Preference public static String archive_url;
+
+    static
+    {
+        final PreferencesReader prefs = AnnotatedPreferences.initialize(Preferences.class, "/appliance_datasource_preferences.properties");
+    }
+
+}

app/trends/archive-datasource/src/main/java/org/phoebus/pv/archive/replay/ReplayPV.java

@@ -0,0 +1,107 @@
+package org.phoebus.pv.archive.replay;
+
+import org.phoebus.archive.reader.ValueIterator;
+import org.phoebus.pv.PV;
+import org.phoebus.pv.archive.ArchiveReaderService;
+
+import java.time.Instant;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.logging.Level;
+
+/** Base for replay PVs
+ *
+ *  @author Kunal Shroff, Kay Kasemir, based on similar code in org.csstudio.utility.pv and diirt
+ */
+@SuppressWarnings("nls")
+public class ReplayPV extends PV
+{
+    ArchiveReaderService service = ArchiveReaderService.getService();
+
+    /** Timer for replaying updates */
+    private final static ScheduledExecutorService executor = Executors.newScheduledThreadPool(1, target ->
+    {
+        final Thread thread = new Thread(target, "ReplayPV");
+        thread.setDaemon(true);
+        return thread;
+    });
+
+    /** Task that was submitted for periodic updates */
+    private ScheduledFuture<?> task;
+
+    private ValueIterator i;
+
+    /**
+     *
+     * @param completeName
+     * @param name
+     * @param start
+     * @param end
+     */
+    public ReplayPV(String completeName, final String name, Instant start, Instant end)
+    {
+        this(completeName, name, start, end, .1);
+    }
+
+    /**
+     *
+     * @param completeName
+     * @param name
+     * @param start
+     * @param end
+     * @param update
+     */
+    public ReplayPV(String completeName, final String name, Instant start, Instant end, double update)
+    {
+        super(completeName);
+
+        // ReplayPV PVs are read-only
+        notifyListenersOfPermissions(true);
+
+        try {
+            i = service.getReader().getRawValues(name, start, end);
+
+            if (i.hasNext()) {
+                notifyListenersOfValue(i.next());
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+        start(update);
+    }
+
+    /** Start periodic updates
+     *  @param update_seconds Update period in seconds
+     */
+    protected void start(final double update_seconds)
+    {
+        // Limit rate to 100 Hz
+        final long milli = Math.round(Math.max(update_seconds, 0.01) * 1000);
+        task = executor.scheduleAtFixedRate(this::update, milli, milli, TimeUnit.MILLISECONDS);
+    }
+
+    /** Called by periodic timer */
+    protected void update()
+    {
+        try {
+            if (i.hasNext()) {
+                notifyListenersOfValue(i.next());
+            } else {
+                close();
+            }
+        } catch (Exception e) {
+            logger.log(Level.WARNING, "failed to update pv: " + getName() , e);
+            close();
+        }
+    }
+
+    @Override
+    protected void close()
+    {
+        if (!task.isDone() && !task.cancel(false))
+            logger.log(Level.WARNING, "Cannot cancel updates for " + getName());
+        super.close();
+    }
+}