From b1ceeb1bee4f52a1efff1da91a29cf2aaf4526eb Mon Sep 17 00:00:00 2001 From: Adam Carpenter Date: Wed, 17 May 2023 13:22:43 -0400 Subject: [PATCH] Cli multi channels (#8) * Initial commit * Support multi-channel myStats query * Fixed bug in multi-channel support * Switched param from l to c for back compat * Switched param from l to c for back compat * Updated test for multi-channel support * Factored out generateMetadataStream * Methods for writing enum labels and metadata * Factored out generateMetadataStream * Update for jmyapi v7.0.0 * Update version and release date * Fixed unclosed stream bug * Fix JSON output error * Update to support multi-channel queries * Update to match new multiechannel output * Only write the stats for one channel, not all * Update output format to match mySampler. * Update channels label * Update release date --- build.gradle | 6 +- .../org/jlab/myquery/MySamplerQueryTest.java | 1080 ++++++++++++++++- .../org/jlab/myquery/MyStatsQueryTest.java | 287 +++-- .../org/jlab/myquery/ChannelController.java | 22 +- .../org/jlab/myquery/IntervalWebService.java | 2 +- .../org/jlab/myquery/MySamplerController.java | 250 ++-- .../org/jlab/myquery/MySamplerWebService.java | 22 +- .../org/jlab/myquery/MyStatsController.java | 111 +- .../java/org/jlab/myquery/MyStatsResults.java | 51 + .../org/jlab/myquery/QueryController.java | 187 ++- src/main/webapp/mysampler-form.html | 2 +- src/main/webapp/mystats-form.html | 2 +- 12 files changed, 1702 insertions(+), 320 deletions(-) create mode 100644 src/main/java/org/jlab/myquery/MyStatsResults.java diff --git a/build.gradle b/build.gradle index 4746012..5faec78 100644 --- a/build.gradle +++ b/build.gradle @@ -4,10 +4,10 @@ plugins { } description = "MYA web-based query tool" -version = '3.0.0' +version = '4.0.0' ext { - releaseDate = 'Jan 24 2023' + releaseDate = 'May 16, 2023' } tasks.withType(JavaCompile) { @@ -35,7 +35,7 @@ configurations { } dependencies { - implementation 'org.jlab:jmyapi:6.3.0', + implementation 'org.jlab:jmyapi:7.0.0', 'org.eclipse.parsson:parsson:1.1.1' providedCompile 'jakarta.servlet:jakarta.servlet-api:6.0.0' testImplementation 'junit:junit:4.13.2' diff --git a/src/integration/java/org/jlab/myquery/MySamplerQueryTest.java b/src/integration/java/org/jlab/myquery/MySamplerQueryTest.java index f57b175..3e9c502 100644 --- a/src/integration/java/org/jlab/myquery/MySamplerQueryTest.java +++ b/src/integration/java/org/jlab/myquery/MySamplerQueryTest.java @@ -24,36 +24,1056 @@ public void basicUsageTest() throws IOException, InterruptedException { assertEquals(200, response.statusCode()); String jsonString = """ - { - "datatype": "DBR_DOUBLE", - "datasize": 1, - "datahost": "mya", - "ioc": null, - "active": true, - "data": [ - { - "d": "2019-08-12 23:59:00.000000", - "v": 94.550102 - }, - { - "d": "2019-08-12 23:59:15.000000", - "v": 94.987701 - }, - { - "d": "2019-08-12 23:59:30.000000", - "v": 94.651604 - }, - { - "d": "2019-08-12 23:59:45.000000", - "v": 94.292702 - }, - { - "d": "2019-08-13 00:00:00.000000", - "v": 95.179703 - } - ], - "returnCount": 5 - }"""; + { + "channels": { + "channel1": { + "metadata": { + "name": "channel1", + "datatype": "DBR_DOUBLE", + "datasize": 1, + "datahost": "mya", + "ioc": null, + "active": true + }, + "data": [ + { + "d": "2019-08-12 23:59:00.000000", + "v": 94.550102 + }, + { + "d": "2019-08-12 23:59:15.000000", + "v": 94.987701 + }, + { + "d": "2019-08-12 23:59:30.000000", + "v": 94.651604 + }, + { + "d": "2019-08-12 23:59:45.000000", + "v": 94.292702 + }, + { + "d": "2019-08-13 00:00:00.000000", + "v": 95.179703 + } + ], + "returnCount": 5 + } + } + }"""; + String exp; + try(JsonReader r = Json.createReader(new StringReader(jsonString))) { + exp = r.readObject().toString(); + } + + try(JsonReader reader = Json.createReader(new StringReader(response.body()))) { + JsonObject json = reader.readObject(); + assertEquals(exp, json.toString()); + } + } + + @Test + public void multiChannelTest() throws IOException, InterruptedException { + HttpClient client = HttpClient.newHttpClient(); + HttpRequest request = HttpRequest.newBuilder().uri(URI.create("http://localhost:8080/myquery/mysampler?c=channel1,channel2,channel3,channel4&b=2019-08-12+23%3A59%3A00&n=5&s=15000&m=docker&f=6&v=")).build(); + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + + assertEquals(200, response.statusCode()); + + String jsonString = """ + { + "channels": { + "channel1": { + "metadata": { + "name": "channel1", + "datatype": "DBR_DOUBLE", + "datasize": 1, + "datahost": "mya", + "ioc": null, + "active": true + }, + "data": [ + { + "d": "2019-08-12 23:59:00.000000", + "v": 94.550102 + }, + { + "d": "2019-08-12 23:59:15.000000", + "v": 94.987701 + }, + { + "d": "2019-08-12 23:59:30.000000", + "v": 94.651604 + }, + { + "d": "2019-08-12 23:59:45.000000", + "v": 94.292702 + }, + { + "d": "2019-08-13 00:00:00.000000", + "v": 95.179703 + } + ], + "returnCount": 5 + }, + "channel2": { + "metadata": { + "name": "channel2", + "datatype": "DBR_ENUM", + "datasize": 1, + "datahost": "mya", + "ioc": null, + "active": true + }, + "labels": [ + { + "d": "2016-08-12 13:00:49.000000", + "value": [ + "BEAM SYNC ONLY", + "PULSE MODE VL", + "TUNE MODE", + "CW MODE (DC)", + "USER MODE" + ] + } + ], + "data": [ + { + "d": "2019-08-12 23:59:00.000000", + "v": 3 + }, + { + "d": "2019-08-12 23:59:15.000000", + "v": 3 + }, + { + "d": "2019-08-12 23:59:30.000000", + "v": 3 + }, + { + "d": "2019-08-12 23:59:45.000000", + "v": 3 + }, + { + "d": "2019-08-13 00:00:00.000000", + "v": 3 + } + ], + "returnCount": 5 + }, + "channel3": { + "metadata": { + "name": "channel3", + "datatype": "DBR_DOUBLE", + "datasize": 168, + "datahost": "mya", + "ioc": null, + "active": true + }, + "data": [ + { + "d": "2019-08-12 23:59:00.000000", + "v": [ + "1565670000", + "1565660000", + "1565660000", + "1565650000", + "1565650000", + "1565650000", + "1565640000", + "1565640000", + "1565640000", + "1565630000", + "1565630000", + "1565630000", + "1565620000", + "1565620000", + "1565610000", + "1565610000", + "1565610000", + "1565600000", + "1565600000", + "1565600000", + "1565590000", + "1565590000", + "1565590000", + "1565580000", + "1565580000", + "1565580000", + "1565570000", + "1565570000", + "1565560000", + "1565560000", + "1565560000", + "1565550000", + "1565550000", + "1565550000", + "1565540000", + "1565540000", + "1565540000", + "1565530000", + "1565530000", + "1565520000", + "1565520000", + "1565520000", + "1565510000", + "1565510000", + "1565510000", + "1565500000", + "1565500000", + "1565500000", + "1565490000", + "1565490000", + "1565490000", + "1565480000", + "1565480000", + "1565470000", + "1565470000", + "1565470000", + "1565460000", + "1565460000", + "1565460000", + "1565450000", + "1565450000", + "1565450000", + "1565440000", + "1565440000", + "1565430000", + "1565430000", + "1565430000", + "1565420000", + "1565420000", + "1565420000", + "1565410000", + "1565410000", + "1565410000", + "1565400000", + "1565400000", + "1565400000", + "1565390000", + "1565390000", + "1565380000", + "1565380000", + "1565380000", + "1565370000", + "1565370000", + "1565370000", + "1565360000", + "1565360000", + "1565360000", + "1565350000", + "1565350000", + "1565340000", + "1565340000", + "1565340000", + "1565330000", + "1565330000", + "1565330000", + "1565320000", + "1565320000", + "1565320000", + "1565310000", + "1565310000", + "1565310000", + "1565300000", + "1565300000", + "1565290000", + "1565290000", + "1565290000", + "1565280000", + "1565280000", + "1565280000", + "1565270000", + "1565270000", + "1565270000", + "1565260000", + "1565260000", + "1565250000", + "1565250000", + "1565250000", + "1565240000", + "1565240000", + "1565240000", + "1565230000", + "1565230000", + "1565230000", + "1565220000", + "1565220000", + "1565220000", + "1565210000", + "1565210000", + "1565200000", + "1565200000", + "1565200000", + "1565190000", + "1565190000", + "1565190000", + "1565180000", + "1565180000", + "1565180000", + "1565170000", + "1565170000", + "1565160000", + "1565160000", + "1565160000", + "1565150000", + "1565150000", + "1565150000", + "1565140000", + "1565140000", + "1565140000", + "1565130000", + "1565130000", + "1565130000", + "1565120000", + "1565120000", + "1565110000", + "1565110000", + "1565110000", + "1565100000", + "1565100000", + "1565100000", + "1565090000", + "1565090000", + "1565090000", + "1565080000", + "1565080000", + "1565070000", + "1565070000", + "1565070000", + "1565060000" + ] + }, + { + "d": "2019-08-12 23:59:15.000000", + "v": [ + "1565670000", + "1565660000", + "1565660000", + "1565650000", + "1565650000", + "1565650000", + "1565640000", + "1565640000", + "1565640000", + "1565630000", + "1565630000", + "1565630000", + "1565620000", + "1565620000", + "1565610000", + "1565610000", + "1565610000", + "1565600000", + "1565600000", + "1565600000", + "1565590000", + "1565590000", + "1565590000", + "1565580000", + "1565580000", + "1565580000", + "1565570000", + "1565570000", + "1565560000", + "1565560000", + "1565560000", + "1565550000", + "1565550000", + "1565550000", + "1565540000", + "1565540000", + "1565540000", + "1565530000", + "1565530000", + "1565520000", + "1565520000", + "1565520000", + "1565510000", + "1565510000", + "1565510000", + "1565500000", + "1565500000", + "1565500000", + "1565490000", + "1565490000", + "1565490000", + "1565480000", + "1565480000", + "1565470000", + "1565470000", + "1565470000", + "1565460000", + "1565460000", + "1565460000", + "1565450000", + "1565450000", + "1565450000", + "1565440000", + "1565440000", + "1565430000", + "1565430000", + "1565430000", + "1565420000", + "1565420000", + "1565420000", + "1565410000", + "1565410000", + "1565410000", + "1565400000", + "1565400000", + "1565400000", + "1565390000", + "1565390000", + "1565380000", + "1565380000", + "1565380000", + "1565370000", + "1565370000", + "1565370000", + "1565360000", + "1565360000", + "1565360000", + "1565350000", + "1565350000", + "1565340000", + "1565340000", + "1565340000", + "1565330000", + "1565330000", + "1565330000", + "1565320000", + "1565320000", + "1565320000", + "1565310000", + "1565310000", + "1565310000", + "1565300000", + "1565300000", + "1565290000", + "1565290000", + "1565290000", + "1565280000", + "1565280000", + "1565280000", + "1565270000", + "1565270000", + "1565270000", + "1565260000", + "1565260000", + "1565250000", + "1565250000", + "1565250000", + "1565240000", + "1565240000", + "1565240000", + "1565230000", + "1565230000", + "1565230000", + "1565220000", + "1565220000", + "1565220000", + "1565210000", + "1565210000", + "1565200000", + "1565200000", + "1565200000", + "1565190000", + "1565190000", + "1565190000", + "1565180000", + "1565180000", + "1565180000", + "1565170000", + "1565170000", + "1565160000", + "1565160000", + "1565160000", + "1565150000", + "1565150000", + "1565150000", + "1565140000", + "1565140000", + "1565140000", + "1565130000", + "1565130000", + "1565130000", + "1565120000", + "1565120000", + "1565110000", + "1565110000", + "1565110000", + "1565100000", + "1565100000", + "1565100000", + "1565090000", + "1565090000", + "1565090000", + "1565080000", + "1565080000", + "1565070000", + "1565070000", + "1565070000", + "1565060000" + ] + }, + { + "d": "2019-08-12 23:59:30.000000", + "v": [ + "1565670000", + "1565660000", + "1565660000", + "1565650000", + "1565650000", + "1565650000", + "1565640000", + "1565640000", + "1565640000", + "1565630000", + "1565630000", + "1565630000", + "1565620000", + "1565620000", + "1565610000", + "1565610000", + "1565610000", + "1565600000", + "1565600000", + "1565600000", + "1565590000", + "1565590000", + "1565590000", + "1565580000", + "1565580000", + "1565580000", + "1565570000", + "1565570000", + "1565560000", + "1565560000", + "1565560000", + "1565550000", + "1565550000", + "1565550000", + "1565540000", + "1565540000", + "1565540000", + "1565530000", + "1565530000", + "1565520000", + "1565520000", + "1565520000", + "1565510000", + "1565510000", + "1565510000", + "1565500000", + "1565500000", + "1565500000", + "1565490000", + "1565490000", + "1565490000", + "1565480000", + "1565480000", + "1565470000", + "1565470000", + "1565470000", + "1565460000", + "1565460000", + "1565460000", + "1565450000", + "1565450000", + "1565450000", + "1565440000", + "1565440000", + "1565430000", + "1565430000", + "1565430000", + "1565420000", + "1565420000", + "1565420000", + "1565410000", + "1565410000", + "1565410000", + "1565400000", + "1565400000", + "1565400000", + "1565390000", + "1565390000", + "1565380000", + "1565380000", + "1565380000", + "1565370000", + "1565370000", + "1565370000", + "1565360000", + "1565360000", + "1565360000", + "1565350000", + "1565350000", + "1565340000", + "1565340000", + "1565340000", + "1565330000", + "1565330000", + "1565330000", + "1565320000", + "1565320000", + "1565320000", + "1565310000", + "1565310000", + "1565310000", + "1565300000", + "1565300000", + "1565290000", + "1565290000", + "1565290000", + "1565280000", + "1565280000", + "1565280000", + "1565270000", + "1565270000", + "1565270000", + "1565260000", + "1565260000", + "1565250000", + "1565250000", + "1565250000", + "1565240000", + "1565240000", + "1565240000", + "1565230000", + "1565230000", + "1565230000", + "1565220000", + "1565220000", + "1565220000", + "1565210000", + "1565210000", + "1565200000", + "1565200000", + "1565200000", + "1565190000", + "1565190000", + "1565190000", + "1565180000", + "1565180000", + "1565180000", + "1565170000", + "1565170000", + "1565160000", + "1565160000", + "1565160000", + "1565150000", + "1565150000", + "1565150000", + "1565140000", + "1565140000", + "1565140000", + "1565130000", + "1565130000", + "1565130000", + "1565120000", + "1565120000", + "1565110000", + "1565110000", + "1565110000", + "1565100000", + "1565100000", + "1565100000", + "1565090000", + "1565090000", + "1565090000", + "1565080000", + "1565080000", + "1565070000", + "1565070000", + "1565070000", + "1565060000" + ] + }, + { + "d": "2019-08-12 23:59:45.000000", + "v": [ + "1565670000", + "1565660000", + "1565660000", + "1565650000", + "1565650000", + "1565650000", + "1565640000", + "1565640000", + "1565640000", + "1565630000", + "1565630000", + "1565630000", + "1565620000", + "1565620000", + "1565610000", + "1565610000", + "1565610000", + "1565600000", + "1565600000", + "1565600000", + "1565590000", + "1565590000", + "1565590000", + "1565580000", + "1565580000", + "1565580000", + "1565570000", + "1565570000", + "1565560000", + "1565560000", + "1565560000", + "1565550000", + "1565550000", + "1565550000", + "1565540000", + "1565540000", + "1565540000", + "1565530000", + "1565530000", + "1565520000", + "1565520000", + "1565520000", + "1565510000", + "1565510000", + "1565510000", + "1565500000", + "1565500000", + "1565500000", + "1565490000", + "1565490000", + "1565490000", + "1565480000", + "1565480000", + "1565470000", + "1565470000", + "1565470000", + "1565460000", + "1565460000", + "1565460000", + "1565450000", + "1565450000", + "1565450000", + "1565440000", + "1565440000", + "1565430000", + "1565430000", + "1565430000", + "1565420000", + "1565420000", + "1565420000", + "1565410000", + "1565410000", + "1565410000", + "1565400000", + "1565400000", + "1565400000", + "1565390000", + "1565390000", + "1565380000", + "1565380000", + "1565380000", + "1565370000", + "1565370000", + "1565370000", + "1565360000", + "1565360000", + "1565360000", + "1565350000", + "1565350000", + "1565340000", + "1565340000", + "1565340000", + "1565330000", + "1565330000", + "1565330000", + "1565320000", + "1565320000", + "1565320000", + "1565310000", + "1565310000", + "1565310000", + "1565300000", + "1565300000", + "1565290000", + "1565290000", + "1565290000", + "1565280000", + "1565280000", + "1565280000", + "1565270000", + "1565270000", + "1565270000", + "1565260000", + "1565260000", + "1565250000", + "1565250000", + "1565250000", + "1565240000", + "1565240000", + "1565240000", + "1565230000", + "1565230000", + "1565230000", + "1565220000", + "1565220000", + "1565220000", + "1565210000", + "1565210000", + "1565200000", + "1565200000", + "1565200000", + "1565190000", + "1565190000", + "1565190000", + "1565180000", + "1565180000", + "1565180000", + "1565170000", + "1565170000", + "1565160000", + "1565160000", + "1565160000", + "1565150000", + "1565150000", + "1565150000", + "1565140000", + "1565140000", + "1565140000", + "1565130000", + "1565130000", + "1565130000", + "1565120000", + "1565120000", + "1565110000", + "1565110000", + "1565110000", + "1565100000", + "1565100000", + "1565100000", + "1565090000", + "1565090000", + "1565090000", + "1565080000", + "1565080000", + "1565070000", + "1565070000", + "1565070000", + "1565060000" + ] + }, + { + "d": "2019-08-13 00:00:00.000000", + "v": [ + "1565670000", + "1565660000", + "1565660000", + "1565650000", + "1565650000", + "1565650000", + "1565640000", + "1565640000", + "1565640000", + "1565630000", + "1565630000", + "1565630000", + "1565620000", + "1565620000", + "1565610000", + "1565610000", + "1565610000", + "1565600000", + "1565600000", + "1565600000", + "1565590000", + "1565590000", + "1565590000", + "1565580000", + "1565580000", + "1565580000", + "1565570000", + "1565570000", + "1565560000", + "1565560000", + "1565560000", + "1565550000", + "1565550000", + "1565550000", + "1565540000", + "1565540000", + "1565540000", + "1565530000", + "1565530000", + "1565520000", + "1565520000", + "1565520000", + "1565510000", + "1565510000", + "1565510000", + "1565500000", + "1565500000", + "1565500000", + "1565490000", + "1565490000", + "1565490000", + "1565480000", + "1565480000", + "1565470000", + "1565470000", + "1565470000", + "1565460000", + "1565460000", + "1565460000", + "1565450000", + "1565450000", + "1565450000", + "1565440000", + "1565440000", + "1565430000", + "1565430000", + "1565430000", + "1565420000", + "1565420000", + "1565420000", + "1565410000", + "1565410000", + "1565410000", + "1565400000", + "1565400000", + "1565400000", + "1565390000", + "1565390000", + "1565380000", + "1565380000", + "1565380000", + "1565370000", + "1565370000", + "1565370000", + "1565360000", + "1565360000", + "1565360000", + "1565350000", + "1565350000", + "1565340000", + "1565340000", + "1565340000", + "1565330000", + "1565330000", + "1565330000", + "1565320000", + "1565320000", + "1565320000", + "1565310000", + "1565310000", + "1565310000", + "1565300000", + "1565300000", + "1565290000", + "1565290000", + "1565290000", + "1565280000", + "1565280000", + "1565280000", + "1565270000", + "1565270000", + "1565270000", + "1565260000", + "1565260000", + "1565250000", + "1565250000", + "1565250000", + "1565240000", + "1565240000", + "1565240000", + "1565230000", + "1565230000", + "1565230000", + "1565220000", + "1565220000", + "1565220000", + "1565210000", + "1565210000", + "1565200000", + "1565200000", + "1565200000", + "1565190000", + "1565190000", + "1565190000", + "1565180000", + "1565180000", + "1565180000", + "1565170000", + "1565170000", + "1565160000", + "1565160000", + "1565160000", + "1565150000", + "1565150000", + "1565150000", + "1565140000", + "1565140000", + "1565140000", + "1565130000", + "1565130000", + "1565130000", + "1565120000", + "1565120000", + "1565110000", + "1565110000", + "1565110000", + "1565100000", + "1565100000", + "1565100000", + "1565090000", + "1565090000", + "1565090000", + "1565080000", + "1565080000", + "1565070000", + "1565070000", + "1565070000", + "1565060000" + ] + } + ], + "returnCount": 5 + }, + "channel4": { + "metadata": { + "name": "channel4", + "datatype": "DBR_DOUBLE", + "datasize": 1, + "datahost": "mya", + "ioc": null, + "active": true + }, + "data": [ + { + "d": "2019-08-12 23:59:00.000000", + "t": "UNDEFINED" + }, + { + "d": "2019-08-12 23:59:15.000000", + "t": "UNDEFINED" + }, + { + "d": "2019-08-12 23:59:30.000000", + "t": "UNDEFINED" + }, + { + "d": "2019-08-12 23:59:45.000000", + "t": "UNDEFINED" + }, + { + "d": "2019-08-13 00:00:00.000000", + "t": "UNDEFINED" + } + ], + "returnCount": 5 + } + } + }"""; String exp; try(JsonReader r = Json.createReader(new StringReader(jsonString))) { exp = r.readObject().toString(); diff --git a/src/integration/java/org/jlab/myquery/MyStatsQueryTest.java b/src/integration/java/org/jlab/myquery/MyStatsQueryTest.java index d956de3..3bdca6f 100644 --- a/src/integration/java/org/jlab/myquery/MyStatsQueryTest.java +++ b/src/integration/java/org/jlab/myquery/MyStatsQueryTest.java @@ -18,82 +18,227 @@ public class MyStatsQueryTest { @Test public void basicUsageTest() throws IOException, InterruptedException { HttpClient client = HttpClient.newHttpClient(); - HttpRequest request = HttpRequest.newBuilder().uri(URI.create("http://localhost:8080/myquery/mystats?c=channel1&b=2019-08-12&e=2019-08-12+01%3A00%3A00&n=5&m=docker&f=3&v=2")).build(); + HttpRequest request = HttpRequest.newBuilder().uri(URI.create("http://localhost:8080/myquery/mystats?c=channel1,channel4&b=2019-08-12&e=2019-08-12+01%3A00%3A00&n=5&m=docker&f=3&v=2")).build(); HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); assertEquals(200, response.statusCode()); String jsonString = """ - { - "datatype": "DBR_DOUBLE", - "datasize": 1, - "datahost": "mya", - "ioc": null, - "active": true, - "data": [ - { - "begin": "2019-08-12 00:00:00.000", - "eventCount": 335, - "updateCount": 334, - "duration": 719.13, - "integration": 68755.14, - "max": 96.81, - "mean": 95.61, - "min": 94.43, - "rms": 95.61, - "stdev": 0.44 - }, - { - "begin": "2019-08-12 00:12:00.000", - "eventCount": 369, - "updateCount": 368, - "duration": 719.12, - "integration": 68944.14, - "max": 96.85, - "mean": 95.87, - "min": 94.94, - "rms": 95.87, - "stdev": 0.39 - }, - { - "begin": "2019-08-12 00:24:00.000", - "eventCount": 343, - "updateCount": 342, - "duration": 718.11, - "integration": 68751.67, - "max": 96.89, - "mean": 95.74, - "min": 95.01, - "rms": 95.74, - "stdev": 0.35 - }, - { - "begin": "2019-08-12 00:36:00.000", - "eventCount": 317, - "updateCount": 316, - "duration": 718.07, - "integration": 65907.67, - "max": 96.95, - "mean": 91.78, - "min": 0, - "rms": 93.27, - "stdev": 16.59 - }, - { - "begin": "2019-08-12 00:48:00.000", - "eventCount": 352, - "updateCount": 351, - "duration": 714.12, - "integration": 68422.19, - "max": 96.9, - "mean": 95.81, - "min": 94.85, - "rms": 95.81, - "stdev": 0.45 - } - ], - "returnCount": 5 - }"""; + { + "channels": { + "channel1": { + "metadata": { + "name": "channel1", + "datatype": "DBR_DOUBLE", + "datasize": 1, + "datahost": "mya", + "ioc": null, + "active": true + }, + "data": [ + { + "begin": "2019-08-12 00:00:00.000", + "eventCount": 335, + "updateCount": 334, + "duration": 719.13, + "integration": 68755.14, + "max": 96.81, + "mean": 95.61, + "min": 94.43, + "rms": 95.61, + "stdev": 0.44 + }, + { + "begin": "2019-08-12 00:12:00.000", + "eventCount": 369, + "updateCount": 368, + "duration": 719.12, + "integration": 68944.14, + "max": 96.85, + "mean": 95.87, + "min": 94.94, + "rms": 95.87, + "stdev": 0.39 + }, + { + "begin": "2019-08-12 00:24:00.000", + "eventCount": 343, + "updateCount": 342, + "duration": 718.11, + "integration": 68751.67, + "max": 96.89, + "mean": 95.74, + "min": 95.01, + "rms": 95.74, + "stdev": 0.35 + }, + { + "begin": "2019-08-12 00:36:00.000", + "eventCount": 317, + "updateCount": 316, + "duration": 718.07, + "integration": 65907.67, + "max": 96.95, + "mean": 91.78, + "min": 0, + "rms": 93.27, + "stdev": 16.59 + }, + { + "begin": "2019-08-12 00:48:00.000", + "eventCount": 352, + "updateCount": 351, + "duration": 714.12, + "integration": 68422.19, + "max": 96.9, + "mean": 95.81, + "min": 94.85, + "rms": 95.81, + "stdev": 0.45 + } + ], + "returnCount": 5 + }, + "channel4": { + "metadata": { + "name": "channel4", + "datatype": "DBR_DOUBLE", + "datasize": 1, + "datahost": "mya", + "ioc": null, + "active": true + }, + "data": [ + { + "begin": "2019-08-12 00:00:00.000", + "eventCount": 0, + "updateCount": 0, + "duration": null, + "integration": null, + "max": null, + "mean": null, + "min": null, + "rms": null, + "stdev": null + }, + { + "begin": "2019-08-12 00:12:00.000", + "eventCount": 0, + "updateCount": 0, + "duration": null, + "integration": null, + "max": null, + "mean": null, + "min": null, + "rms": null, + "stdev": null + }, + { + "begin": "2019-08-12 00:24:00.000", + "eventCount": 0, + "updateCount": 0, + "duration": null, + "integration": null, + "max": null, + "mean": null, + "min": null, + "rms": null, + "stdev": null + }, + { + "begin": "2019-08-12 00:36:00.000", + "eventCount": 0, + "updateCount": 0, + "duration": null, + "integration": null, + "max": null, + "mean": null, + "min": null, + "rms": null, + "stdev": null + }, + { + "begin": "2019-08-12 00:48:00.000", + "eventCount": 0, + "updateCount": 0, + "duration": null, + "integration": null, + "max": null, + "mean": null, + "min": null, + "rms": null, + "stdev": null + } + ], + "returnCount": 5 + } + } + }"""; + String exp; + try (JsonReader r = Json.createReader(new StringReader(jsonString))) { + exp = r.readObject().toString(); + } + + try (JsonReader reader = Json.createReader(new StringReader(response.body()))) { + JsonObject json = reader.readObject(); + assertEquals(exp, json.toString()); + } + } + + @Test + public void unsupportedTypeTest() throws IOException, InterruptedException { + HttpClient client = HttpClient.newHttpClient(); + HttpRequest request = HttpRequest.newBuilder().uri(URI.create("http://localhost:8080/myquery/mystats?c=channel1,channel2&b=2019-08-12+00%3A01%3A00&e=2019-08-19+02%3A00%3A00&n=2&m=docker&f=&v=")).build(); + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + + assertEquals(200, response.statusCode()); + + String jsonString = """ + { + "channels": { + "channel1": { + "metadata": { + "name": "channel1", + "datatype": "DBR_DOUBLE", + "datasize": 1, + "datahost": "mya", + "ioc": null, + "active": true + }, + "data": [ + { + "begin": "2019-08-12T00:01:00", + "eventCount": 32963, + "updateCount": 32962, + "duration": 305970, + "integration": 27214184.885566, + "max": 103.997002, + "mean": 88.943965, + "min": 0, + "rms": 91.141455, + "stdev": 19.893113 + }, + { + "begin": "2019-08-15T13:00:30", + "eventCount": 2, + "updateCount": 1, + "duration": 305970, + "integration": 29282429.886932, + "max": 95.703598, + "mean": 95.703598, + "min": 95.703598, + "rms": 95.703598, + "stdev": 0 + } + ], + "returnCount": 2 + }, + "channel2": { + "error": "This myStats only supports FloatEvents - not 'org.jlab.mya.event.IntEvent'." + } + } + }"""; + String exp; try (JsonReader r = Json.createReader(new StringReader(jsonString))) { exp = r.readObject().toString(); diff --git a/src/main/java/org/jlab/myquery/ChannelController.java b/src/main/java/org/jlab/myquery/ChannelController.java index 8f8f2a8..122a6dc 100644 --- a/src/main/java/org/jlab/myquery/ChannelController.java +++ b/src/main/java/org/jlab/myquery/ChannelController.java @@ -29,12 +29,11 @@ public class ChannelController extends QueryController { * * @param request servlet request * @param response servlet response - * @throws ServletException if a servlet-specific error occurs * @throws IOException if an I/O error occurs */ @Override protected void doGet(HttpServletRequest request, HttpServletResponse response) - throws ServletException, IOException { + throws IOException { String jsonp = request.getParameter("jsonp"); if (jsonp != null) { @@ -85,7 +84,7 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) metadataList = service.findChannel(q, limit, offset); - // Disable caching of response so we don't miss new channels added: + // Disable caching of response, so we don't miss new channels added: CacheAndEncodingFilter.disableCaching(response); } catch (Exception ex) { LOGGER.log(Level.SEVERE, "Unable to service request", ex); @@ -106,22 +105,7 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) gen.writeEnd(); } else { gen.writeStartArray(); - if (metadataList != null) { - for (Metadata metadata : metadataList) { - gen.writeStartObject(); - gen.write("name", metadata.getName()); - gen.write("datatype", metadata.getMyaType().name()); - gen.write("datasize", metadata.getSize()); - gen.write("datahost", metadata.getHost()); - if(metadata.getIoc() == null) { - gen.writeNull("ioc"); - } else { - gen.write("ioc", metadata.getIoc()); - } - gen.write("active", metadata.isActive()); - gen.writeEnd(); - } - } + generateMetadataStream(gen, metadataList); gen.writeEnd(); } diff --git a/src/main/java/org/jlab/myquery/IntervalWebService.java b/src/main/java/org/jlab/myquery/IntervalWebService.java index 9133bbd..dea715e 100644 --- a/src/main/java/org/jlab/myquery/IntervalWebService.java +++ b/src/main/java/org/jlab/myquery/IntervalWebService.java @@ -109,7 +109,7 @@ private EventStream doApplicationSampling(String sampleType, Metadat double endD = (end.getEpochSecond() + end.getNano() / 1_000_000_000d); double beginD = (begin.getEpochSecond() + begin.getNano() / 1_000_000_000d); long stepMillis = (long) (((endD - beginD) / (limit - 1)) * 1000); - stream = new MySamplerStream<>(stream, begin, stepMillis, limit, priorEvent, updatesOnly, FloatEvent.class); + stream = MySamplerStream.getMySamplerStream(stream, begin, stepMillis, limit, priorEvent, updatesOnly, FloatEvent.class); break; default: throw new IllegalArgumentException("Unrecognized sampleType - " + sampleType + ". Options include graphical, eventsimple, myget, mysampler"); diff --git a/src/main/java/org/jlab/myquery/MySamplerController.java b/src/main/java/org/jlab/myquery/MySamplerController.java index a1fb778..2080b5f 100644 --- a/src/main/java/org/jlab/myquery/MySamplerController.java +++ b/src/main/java/org/jlab/myquery/MySamplerController.java @@ -20,12 +20,14 @@ import java.time.LocalDateTime; import java.time.ZoneId; import java.time.format.DateTimeFormatter; -import java.util.List; +import java.util.*; import java.util.logging.Level; import java.util.logging.Logger; +import java.util.regex.PatternSyntaxException; /** * This method provides an end point for functionality similar to the mySampler command line application. + * * @author adamc */ @WebServlet(name = "MySamplerController", value = "/mysampler") @@ -38,13 +40,11 @@ public class MySamplerController extends QueryController { * * @param request servlet request * @param response servlet response - * @throws ServletException if a servlet-specific error occurs * @throws IOException if an I/O error occurs */ - @SuppressWarnings({"unchecked"}) @Override protected void doGet(HttpServletRequest request, HttpServletResponse response) - throws ServletException, IOException { + throws IOException, ServletException { String jsonp = request.getParameter("jsonp"); if (jsonp != null) { @@ -54,10 +54,12 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) } String errorReason = null; - EventStream stream = null; - Metadata metadata = null; - List enumLabels = null; - + List channels = null; + MySamplerWebService service = null; + long intervalMillis = -1; + long sampleCount = -1; + String deployment = "ops"; + Instant begin = null; String c = request.getParameter("c"); // channel String b = request.getParameter("b"); // begin @@ -71,6 +73,9 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) String a = request.getParameter("a"); // adjustMillisWithServerOffset String v = request.getParameter("v"); // decimalFormatter (value precision) + boolean updatesOnly = (d != null); + boolean enumsAsStrings = (e != null); + try { if (c == null || c.trim().isEmpty()) { throw new Exception("Channel (c) is required"); @@ -93,35 +98,34 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) b = b + "T00:00:00"; } - Instant begin = LocalDateTime.parse(b).atZone( - ZoneId.systemDefault()).toInstant(); + begin = LocalDateTime.parse(b).atZone(ZoneId.systemDefault()).toInstant(); - String deployment = "ops"; if (m != null && !m.trim().isEmpty()) { deployment = m; } - MySamplerWebService service = new MySamplerWebService(deployment); + service = new MySamplerWebService(deployment); - metadata = service.findMetadata(c); - if (metadata == null) { - throw new Exception("Unable to find channel: '" + c + "' in deployment: '" + deployment + "'"); + try { + channels = Arrays.asList(c.split(",")); + } catch (PatternSyntaxException ex) { + throw new Exception("Error parsing comma separated channel list: " + c); } - long intervalMillis, sampleCount; try { intervalMillis = Long.parseLong(s); } catch (NumberFormatException ex) { throw new Exception("Error parsing sample interval (s) in milliseconds: '" + s + "'"); } + if (intervalMillis > 315_360_000_000L) { + throw new IllegalArgumentException("Sample interval (s) must be less than 10 years."); + } try { sampleCount = Long.parseLong(n); } catch (NumberFormatException ex) { throw new Exception("Error parsing number of samples (n): '" + n + "'"); } - boolean updatesOnly = (d != null); - boolean enumsAsStrings = (e != null); // Don't tell client to cache response if contains future bounds! Instant end = begin.plusMillis(intervalMillis * (sampleCount - 1)); @@ -131,28 +135,9 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) response.setHeader("Cache-Control", "private"); } - // Get the sampled stream. If it's an enum convert to use string labels if requested. - stream = service.openEventStream(metadata, begin, intervalMillis, sampleCount, updatesOnly); - if(metadata.getMyaType() == MyaDataType.DBR_ENUM) { - enumLabels = service.findExtraInfo(metadata, "enum_strings", begin, end); - if (enumsAsStrings) { - stream = new LabeledEnumStream((EventStream) stream, enumLabels); - } - } - - } catch (Exception ex) { LOGGER.log(Level.SEVERE, "Unable to service request", ex); errorReason = ex.getMessage(); - - try { - if (stream != null) { - stream.close(); - stream = null; - } - } catch (Exception closeIssue) { - System.err.println("Unable to close stream"); - } } DateTimeFormatter timestampFormatter = FormatUtil.getInstantFormatter(f); @@ -167,74 +152,142 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) out.write((jsonp + "(").getBytes(StandardCharsets.UTF_8)); } + // Won't compile without the -1 setting/check because of possibly uninitialized variables. + if (errorReason == null) { + if ( sampleCount == -1) { + errorReason = "sampleCount (n) is required"; + } else if (intervalMillis == -1) { + errorReason = "intervalMillis (s) is required"; + } else if (begin == null) { + errorReason = "begin time (b) is required"; + } + } + + // The logic around multiple streams is getting a little confusing. If we've hit an error before we process + // any channels, let's write the error and close out. + if (errorReason != null) { + response.setStatus(HttpServletResponse.SC_BAD_REQUEST); + out.write(("{\"error\": \"" + errorReason + "\"}").getBytes(StandardCharsets.UTF_8)); + if (jsonp != null) { + out.write((");").getBytes(StandardCharsets.UTF_8)); + } + out.close(); + return; + } + try (JsonGenerator gen = Json.createGenerator(out)) { gen.writeStartObject(); - - if (errorReason != null) { - response.setStatus(HttpServletResponse.SC_BAD_REQUEST); - gen.write("error", errorReason); - } else { - if (metadata != null) { - gen.write("datatype", metadata.getMyaType().name()); - gen.write("datasize", metadata.getSize()); - gen.write("datahost", metadata.getHost()); - if (metadata.getIoc() == null) { - gen.writeNull("ioc"); - } else { - gen.write("ioc", metadata.getIoc()); - } - gen.write("active", metadata.isActive()); + boolean anyErrors = false; + gen.writeStartObject("channels"); + for(String channelName : channels) { + boolean error = processChannelRequest(service, deployment, gen, channelName, begin, + intervalMillis, sampleCount, updatesOnly, formatAsMillisSinceEpoch, + adjustMillisWithServerOffset, timestampFormatter, decimalFormatter, enumsAsStrings); + if (error) { + anyErrors = true; } + } + gen.writeEnd(); + if (anyErrors) { + response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + } + gen.writeEnd(); + gen.flush(); - if (enumLabels != null && enumLabels.size() > 0) { - gen.writeStartArray("labels"); - for (ExtraInfo info : enumLabels) { - gen.writeStartObject(); - FormatUtil.writeTimestampJSON(gen, "d", info.getTimestamp(), formatAsMillisSinceEpoch, adjustMillisWithServerOffset, timestampFormatter); - gen.writeStartArray("value"); - for (String token : info.getValueAsArray()) { - if (token != null && !token.isEmpty()) { - gen.write(token); - } - } - gen.writeEnd(); - gen.writeEnd(); - } - gen.writeEnd(); - } + if (jsonp != null) { + out.write((");").getBytes(StandardCharsets.UTF_8)); + } - gen.writeStartArray("data"); - - long dataLength = 0; - if (stream == null) { - // Didn't get a stream so presumably there is an errorReason - } else if (stream.getType() == IntEvent.class) { - dataLength = generateIntStream(gen, (EventStream) stream, formatAsMillisSinceEpoch, adjustMillisWithServerOffset, - timestampFormatter); - } else if (stream.getType() == FloatEvent.class) { - dataLength = generateFloatStream(gen, (EventStream) stream, formatAsMillisSinceEpoch, adjustMillisWithServerOffset, - timestampFormatter, decimalFormatter); - } else if (stream.getType() == AnalyzedFloatEvent.class) { - dataLength = generateAnalyzedFloatStream(gen, (EventStream) stream, formatAsMillisSinceEpoch, adjustMillisWithServerOffset, - timestampFormatter, decimalFormatter); - } else if (stream.getType() == LabeledEnumEvent.class) { - dataLength = generateLabeledEnumStream(gen, (EventStream) stream, formatAsMillisSinceEpoch, adjustMillisWithServerOffset, timestampFormatter); - } else if (stream.getType() == MultiStringEvent.class) { - dataLength = generateMultiStringStream(gen, (EventStream) stream, formatAsMillisSinceEpoch, adjustMillisWithServerOffset, - timestampFormatter); - } else { - throw new ServletException("Unsupported data type: " + stream.getClass()); - } - gen.writeEnd(); - gen.write("returnCount", dataLength); - } + } + } catch (Exception ex) { + response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + LOGGER.log(Level.SEVERE, "Unexepected error", ex); + throw ex; + } + } + + @SuppressWarnings("unchecked") + private boolean processChannelRequest(MySamplerWebService service, String deployment, JsonGenerator gen, String channel, + Instant begin, long intervalMillis, long sampleCount, boolean updatesOnly, + boolean formatAsMillisSinceEpoch, boolean adjustMillisWithServerOffset, + DateTimeFormatter timestampFormatter, DecimalFormat decimalFormatter, + boolean enumsAsStrings) throws ServletException { + gen.writeStartObject(channel); + boolean error = false; + + // Write out the channel's metadata. + Metadata metadata = null; + try { + metadata = service.findMetadata(channel); + if (metadata == null) { + throw new Exception("Unable to find channel: '" + channel + "' in deployment: '" + deployment + "'"); + } + writeMetadata("metadata", gen, metadata); + } catch (Exception ex) { + gen.write("error", ex.getMessage()); + gen.writeEnd(); + return true; + } + + // Write out the channels enum label's if channel was an enumerated type + List enumLabels = null; + if (metadata.getMyaType() == MyaDataType.DBR_ENUM) { + try { + enumLabels = service.findExtraInfo(metadata, "enum_strings", begin, + begin.plusMillis(intervalMillis * (sampleCount - 1))); + writeEnumLabels("labels", gen, enumLabels, formatAsMillisSinceEpoch, adjustMillisWithServerOffset, + timestampFormatter); + } catch (Exception ex) { + gen.write("error", ex.getMessage()); gen.writeEnd(); + return true; + } + } - gen.flush(); + // Write out the channel's data + EventStream stream = null; + try { + + // Cannot use try with resources since we may sometimes wrap stream in a LabeledEnumStream + stream = service.openEventStream(metadata, begin, intervalMillis, sampleCount, updatesOnly); + if (enumsAsStrings && metadata.getMyaType() == MyaDataType.DBR_ENUM && enumLabels != null && !enumLabels.isEmpty()) { + + stream = new LabeledEnumStream(stream, enumLabels); } - if (jsonp != null) { - out.write((");").getBytes(StandardCharsets.UTF_8)); + + gen.writeStartArray("data"); + long dataLength = 0; + if (stream.getType() == IntEvent.class) { + dataLength = generateIntStream(gen, (EventStream) stream, formatAsMillisSinceEpoch, adjustMillisWithServerOffset, + timestampFormatter); + } else if (stream.getType() == FloatEvent.class) { + dataLength = generateFloatStream(gen, (EventStream) stream, formatAsMillisSinceEpoch, adjustMillisWithServerOffset, + timestampFormatter, decimalFormatter); + } else if (stream.getType() == AnalyzedFloatEvent.class) { + dataLength = generateAnalyzedFloatStream(gen, (EventStream) stream, formatAsMillisSinceEpoch, adjustMillisWithServerOffset, + timestampFormatter, decimalFormatter); + } else if (stream.getType() == LabeledEnumEvent.class) { + dataLength = generateLabeledEnumStream(gen, (EventStream) stream, formatAsMillisSinceEpoch, adjustMillisWithServerOffset, timestampFormatter); + } else if (stream.getType() == MultiStringEvent.class) { + dataLength = generateMultiStringStream(gen, (EventStream) stream, formatAsMillisSinceEpoch, adjustMillisWithServerOffset, + timestampFormatter); + } else { + gen.writeEnd(); + throw new ServletException("Unsupported data type: " + stream.getClass()); + } + + gen.writeEnd(); + gen.write("returnCount", dataLength); + + } catch(Exception ex) { + // Can't just return or else we leave a connection open + error = true; + try { + gen.write("error", ex.getMessage()); + } catch(Exception writeEx) { + LOGGER.log(Level.SEVERE, "Error trying to write error message.", writeEx); + throw new ServletException("Error trying to write error message to JSON", writeEx); } } finally { try { @@ -242,8 +295,11 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) stream.close(); } } catch (Exception closeIssue) { - System.err.println("Unable to close stream"); + LOGGER.log(Level.SEVERE, "Unable to close stream. channel=" + metadata.getName()); + error = true; } } + gen.writeEnd(); + return error; } } diff --git a/src/main/java/org/jlab/myquery/MySamplerWebService.java b/src/main/java/org/jlab/myquery/MySamplerWebService.java index c551429..0ef6a8c 100644 --- a/src/main/java/org/jlab/myquery/MySamplerWebService.java +++ b/src/main/java/org/jlab/myquery/MySamplerWebService.java @@ -5,16 +5,19 @@ import org.jlab.mya.event.Event; import org.jlab.mya.nexus.DataNexus; import org.jlab.mya.stream.*; +import java.io.IOException; import java.sql.SQLException; import java.time.Instant; import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; /** * This class provides features similar to the mySampler command line utility. * @author adamc */ public class MySamplerWebService extends QueryWebService { - + private static final Logger LOGGER = Logger.getLogger((MySamplerWebService.class.getName())); private final DataNexus nexus; public MySamplerWebService(String deployment) { @@ -31,7 +34,7 @@ public List findExtraInfo(Metadata metadata, String type, Instant beg @SuppressWarnings("unchecked") - public EventStream openEventStream(Metadata metadata, Instant begin, long intervalMillis, + public MySamplerStream openEventStream(Metadata metadata, Instant begin, long intervalMillis, long sampleCount, boolean updatesOnly) throws SQLException, UnsupportedOperationException { PointWebService pws = new PointWebService(nexus.getDeployment()); @@ -40,6 +43,19 @@ public EventStream openEventStream(Metadata metadata, In Instant end = begin.plusMillis(intervalMillis * (sampleCount - 1)); EventStream stream = nexus.openEventStream(metadata, begin, end, DataNexus.IntervalQueryFetchStrategy.STREAM, updatesOnly); - return new MySamplerStream<>(stream, begin, intervalMillis, sampleCount, priorEvent, updatesOnly, metadata.getType()); + MySamplerStream out; + try { + out = MySamplerStream.getMySamplerStream(stream, begin, intervalMillis, sampleCount, priorEvent, updatesOnly, metadata.getType()); + } catch (Exception ex) { + if (stream != null) { + try { + stream.close(); + } catch (IOException closeIssue) { + LOGGER.log(Level.SEVERE, "Could not close stream", ex); + } + } + throw ex; + } + return out; } } diff --git a/src/main/java/org/jlab/myquery/MyStatsController.java b/src/main/java/org/jlab/myquery/MyStatsController.java index b0487ce..b36a68b 100644 --- a/src/main/java/org/jlab/myquery/MyStatsController.java +++ b/src/main/java/org/jlab/myquery/MyStatsController.java @@ -5,7 +5,6 @@ import jakarta.servlet.http.*; import jakarta.servlet.annotation.*; import org.jlab.mya.Metadata; -import org.jlab.mya.RunningStatistics; import org.jlab.mya.event.*; import org.jlab.mya.stream.FloatAnalysisStream; import java.io.IOException; @@ -19,6 +18,7 @@ import java.util.*; import java.util.logging.Level; import java.util.logging.Logger; +import java.util.regex.PatternSyntaxException; /** * This class provides functionality similar to the command line application myStats. @@ -48,11 +48,10 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) } String errorReason = null; - Metadata metadata = null; - Map stats = new TreeMap<>(); // TreeMap results in bin sorted JSON response. + List metadatas = null; + MyStatsResults results = new MyStatsResults(); - - String c = request.getParameter("c"); // channel + String c = request.getParameter("c"); // channels String b = request.getParameter("b"); // begin String e = request.getParameter("e"); // end String n = request.getParameter("n"); // number of bins @@ -65,7 +64,7 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) try { if (c == null || c.trim().isEmpty()) { - throw new Exception("Channel (c) is required"); + throw new Exception("Channel list (c) is required"); } if (b == null || b.trim().isEmpty()) { throw new Exception("Begin Date (b) is required"); @@ -102,13 +101,20 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) IntervalWebService service = new IntervalWebService(deployment); - metadata = service.findMetadata(c); - if (metadata == null) { - throw new Exception("Unable to find channel: '" + c + "' in deployment: '" + deployment + "'"); + metadatas = new ArrayList<>(); + List channels; + try { + channels = Arrays.asList(c.split(",")); + } catch (PatternSyntaxException ex) { + throw new Exception("Error parsing comma separated channel list: " + c); } - if (metadata.getType() != FloatEvent.class) { - throw new IllegalArgumentException("This myStats only supports FloatEvents - not '" + metadata.getType().getName() + "'."); + for (String channel : channels) { + Metadata metadata = service.findMetadata(channel); + if (metadata == null) { + throw new Exception("Unable to find channel: '" + channel + "' in deployment: '" + deployment + "'"); + } + metadatas.add(metadata); } long numBins; @@ -131,23 +137,35 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) } PointWebService pws = new PointWebService(deployment); - Event priorEvent = pws.findEvent(metadata, updatesOnly, begin, true, true, false); - - double interval = ((end.getEpochSecond() + end.getNano() / 1_000_000_000d) - (begin.getEpochSecond() + begin.getNano() / 1_000_000_000d)) / numBins; - Instant binBegin, binEnd = begin; - for (int i = 1; i <= numBins; i++) { - binBegin = binEnd; - if (i == numBins) { - binEnd = end; - } else { - binEnd = binBegin.plusSeconds((long) interval); + Map priorEvents = new HashMap<>(); + for (Metadata metadata : metadatas) { + priorEvents.put(metadata.getName(), pws.findEvent(metadata, updatesOnly, begin, true, true, false)); + } + + Event priorEvent; + for (Metadata metadata : metadatas) { + + if (metadata.getType() != FloatEvent.class) { + continue; } - // Since we provide a priorPoint, the underlying stream should be a BoundaryAwareStream. - try (FloatAnalysisStream fas = new FloatAnalysisStream(service.openEventStream(metadata, updatesOnly, binBegin, binEnd, priorEvent, metadata.getType()))) { - while (fas.read() != null) { - // Read through the entire stream. We only want statistics from it + + priorEvent = priorEvents.get(metadata.getName()); + double interval = ((end.getEpochSecond() + end.getNano() / 1_000_000_000d) - (begin.getEpochSecond() + begin.getNano() / 1_000_000_000d)) / numBins; + Instant binBegin, binEnd = begin; + for (int i = 1; i <= numBins; i++) { + binBegin = binEnd; + if (i == numBins) { + binEnd = end; + } else { + binEnd = binBegin.plusSeconds((long) interval); + } + // Since we provide a priorPoint, the underlying stream should be a BoundaryAwareStream. + try (FloatAnalysisStream fas = new FloatAnalysisStream(service.openEventStream(metadata, updatesOnly, binBegin, binEnd, priorEvent, metadata.getType()))) { + while (fas.read() != null) { + // Read through the entire stream. We only want statistics from it + } + results.add(metadata.getName(), binBegin, fas.getLatestStats()); } - stats.put(binBegin, fas.getLatestStats()); } } @@ -175,30 +193,27 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) response.setStatus(HttpServletResponse.SC_BAD_REQUEST); gen.write("error", errorReason); } else { - if (metadata != null) { - gen.write("datatype", metadata.getMyaType().name()); - gen.write("datasize", metadata.getSize()); - gen.write("datahost", metadata.getHost()); - if (metadata.getIoc() == null) { - gen.writeNull("ioc"); - } else { - gen.write("ioc", metadata.getIoc()); + if (metadatas != null) { + gen.writeStartObject("channels"); + for (Metadata metadata : metadatas) { + gen.writeStartObject(metadata.getName()); + + if (metadata.getType() != FloatEvent.class) { + gen.write("error", "This myStats only supports FloatEvents - not '" + + metadata.getType().getName() + "'."); + gen.writeEnd(); + continue; + } + + writeMetadata("metadata", gen, metadata); + long dataLength = generateStatisticsStream("data", gen, results.get(metadata.getName()), + timestampFormatter, decimalFormatter, + formatAsMillisSinceEpoch, adjustMillisWithServerOffset); + gen.write("returnCount", dataLength); + gen.writeEnd(); // metadata.getName() } - gen.write("active", metadata.isActive()); + gen.writeEnd(); // channels } - - gen.writeStartArray("data"); - - long dataLength = 0; - if (stats.isEmpty()) { - // Presumably there is an error reason - } else { - dataLength = generateStatisticsStream(gen, stats, timestampFormatter, decimalFormatter, - formatAsMillisSinceEpoch, adjustMillisWithServerOffset); - } - - gen.writeEnd(); - gen.write("returnCount", dataLength); } gen.writeEnd(); gen.flush(); diff --git a/src/main/java/org/jlab/myquery/MyStatsResults.java b/src/main/java/org/jlab/myquery/MyStatsResults.java new file mode 100644 index 0000000..eb11db1 --- /dev/null +++ b/src/main/java/org/jlab/myquery/MyStatsResults.java @@ -0,0 +1,51 @@ +package org.jlab.myquery; + +import org.jlab.mya.RunningStatistics; + +import java.time.Instant; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; + +/** + * A class for containing the RunningStatistics data on multiple channels over periods of time. This class is intended + * to be used to represent the situation where a contiguous channel history has been split into bins and summary + * statistics were calculated for each bin. This class supports tracking multiple channels with different binning, but + * you will probably be best served by keeping all of the bin sizes the same. + * @author adamc + */ +public class MyStatsResults { + private final Map> statMap; + + public MyStatsResults() { + statMap = new TreeMap<>(); + } + + /** + * Add statistics information about a channel for a given start time. + * @param channel The name of PV/channel being updates + * @param timestamp The timestamp for the beginning of the time period for which the statistics are valid + * @param stats The object containing the statistics values. + */ + public void add(String channel, Instant timestamp, RunningStatistics stats) { + statMap.putIfAbsent(channel, new TreeMap<>()); + statMap.get(channel).put(timestamp, stats); + } + + /** + * Get the binned statistics for a given channel + * @param channel Then name of the channel to fetch + * @return A map that pairs the statistics with the bin start time for that channel. + */ + public Map get(String channel) { + return statMap.get(channel); + } + + /** + * Get the set of channels currently tracked by this object. + * @return A set of channel names + */ + public Set getChannels() { + return statMap.keySet(); + } +} diff --git a/src/main/java/org/jlab/myquery/QueryController.java b/src/main/java/org/jlab/myquery/QueryController.java index f28e2d8..23fc681 100644 --- a/src/main/java/org/jlab/myquery/QueryController.java +++ b/src/main/java/org/jlab/myquery/QueryController.java @@ -5,9 +5,12 @@ import java.text.DecimalFormat; import java.time.Instant; import java.time.format.DateTimeFormatter; +import java.util.List; import java.util.Map; import jakarta.json.stream.JsonGenerator; import jakarta.servlet.http.HttpServlet; +import org.jlab.mya.ExtraInfo; +import org.jlab.mya.Metadata; import org.jlab.mya.RunningStatistics; import org.jlab.mya.event.*; import org.jlab.mya.stream.EventStream; @@ -130,6 +133,132 @@ public void writeMultiStringEvent(String name, JsonGenerator gen, MultiStringEve gen.writeEnd(); } + public void writeRunningStatistics(String name, JsonGenerator gen, RunningStatistics stat, Instant begin, + boolean formatAsMillisSinceEpoch, boolean adjustMillisWithServerOffset, + DateTimeFormatter timestampFormatter, DecimalFormat decimalFormatter){ + if (name == null) { + gen.writeStartObject(); + } else { + gen.writeStartObject(name); + } + + FormatUtil.writeTimestampJSON(gen, "begin", begin, formatAsMillisSinceEpoch, adjustMillisWithServerOffset, + timestampFormatter); + gen.write("eventCount", stat.getEventCount()); + gen.write("updateCount", stat.getUpdateCount()); + + if (stat.getDuration() == null) { + gen.writeNull("duration"); + } else { + gen.write("duration", new BigDecimal(decimalFormatter.format(stat.getDuration()))); + } + if (stat.getIntegration() == null) { + gen.writeNull("integration"); + } else { + gen.write("integration", new BigDecimal(decimalFormatter.format(stat.getIntegration()))); + } + + if (stat.getMax() == null) { + gen.writeNull("max"); + } else { + gen.write("max", new BigDecimal(decimalFormatter.format(stat.getMax()))); + } + if (stat.getMean() == null) { + gen.writeNull("mean"); + } else { + gen.write("mean", new BigDecimal(decimalFormatter.format(stat.getMean()))); + } + if (stat.getMin() == null) { + gen.writeNull("min"); + } else { + gen.write("min", new BigDecimal(decimalFormatter.format(stat.getMin()))); + } + if (stat.getRms() == null) { + gen.writeNull("rms"); + } else { + gen.write("rms", new BigDecimal(decimalFormatter.format(stat.getRms()))); + } + if (stat.getSigma() == null) { + gen.writeNull("stdev"); + } else { + gen.write("stdev", new BigDecimal(decimalFormatter.format(stat.getSigma()))); + } + gen.writeEnd(); + } + + /** + * Write out Metadata to a JSON generator + * @param gen The generator to write to + * @param metadata The metadata to write + */ + public void writeMetadata(String name, JsonGenerator gen, Metadata metadata) { + if (name == null) { + gen.writeStartObject(); + } else { + gen.writeStartObject(name); + } + gen.write("name", metadata.getName()); + gen.write("datatype", metadata.getMyaType().name()); + gen.write("datasize", metadata.getSize()); + gen.write("datahost", metadata.getHost()); + if (metadata.getIoc() == null) { + gen.writeNull("ioc"); + } else { + gen.write("ioc", metadata.getIoc()); + } + gen.write("active", metadata.isActive()); + gen.writeEnd(); + } + + /** + * Write out the enum labels for a single channel to a JSON generator + * @param name Optional name for the label list + * @param gen The JSON generator to write to + * @param enumLabelList The list of ExtraInfo, i.e., the enum labels over time + * @param formatAsMillisSinceEpoch Write timestamps in more precise Unix-like time format + * @param adjustMillisWithServerOffset Adjust timestamps for server + * @param timestampFormatter How to format the datetimes associated with values + */ + public void writeEnumLabels(String name, JsonGenerator gen, List enumLabelList, + boolean formatAsMillisSinceEpoch, boolean adjustMillisWithServerOffset, + DateTimeFormatter timestampFormatter) { + if (name != null) { + gen.writeStartArray(name); + } else { + gen.writeStartArray(); + } + for (ExtraInfo info : enumLabelList) { + gen.writeStartObject(); + FormatUtil.writeTimestampJSON(gen, "d", info.getTimestamp(), formatAsMillisSinceEpoch, adjustMillisWithServerOffset, timestampFormatter); + gen.writeStartArray("value"); + for (String token : info.getValueAsArray()) { + if (token != null && !token.isEmpty()) { + gen.write(token); + } + } + gen.writeEnd(); // value array + gen.writeEnd(); // object + } + gen.writeEnd(); // array + + } + + /** + * Write a list of metadata objects as a series of JSON objects. Assumes that a JSON array has been started/ended + * outside of this method. + * @param gen The JSON generator used to write the data + * @param metadataList The metadata objects to write + */ + public void generateMetadataStream(JsonGenerator gen, List metadataList) { + if (metadataList != null) { + for (Metadata metadata : metadataList) { + writeMetadata(null, gen, metadata); + } + } else { + gen.writeNull(); + } + } + /** * Write out the IntEventStream to a JsonGenerator. * @param gen @@ -151,7 +280,8 @@ public long generateIntStream(JsonGenerator gen, EventStream stream, } /** - * This method write a stream of RunningStatistics associated with a given start time to a JSON generator. + * This method write a stream of RunningStatistics associated with a given start time to a JSON generator. This + * creates the JSON array, and does not expect one to be started outside of the method. * @param gen The JSON generator to write them to * @param stats The Map of timestamps to RunningStatistics that will be written to the JSON generator * @param timestampFormatter How to format timestamps @@ -160,56 +290,21 @@ public long generateIntStream(JsonGenerator gen, EventStream stream, * @param adjustMillisWithServerOffset * @return The number of RunningStatistics written to the stream */ - public long generateStatisticsStream(JsonGenerator gen, Map stats, + public long generateStatisticsStream(String name, JsonGenerator gen, Map stats, DateTimeFormatter timestampFormatter, DecimalFormat decimalFormatter, boolean formatAsMillisSinceEpoch, boolean adjustMillisWithServerOffset) { + if (name == null) { + gen.writeStartArray(); + } else { + gen.writeStartArray(name); + } long count = 0; - for (Instant begin : stats.keySet()) { - RunningStatistics stat = stats.get(begin); - gen.writeStartObject(); - FormatUtil.writeTimestampJSON(gen, "begin", begin, formatAsMillisSinceEpoch, adjustMillisWithServerOffset, timestampFormatter); - gen.write("eventCount", stat.getEventCount()); - gen.write("updateCount", stat.getUpdateCount()); - - if (stat.getDuration() == null) { - gen.writeNull("duration"); - } else { - gen.write("duration", new BigDecimal(decimalFormatter.format(stat.getDuration()))); - } - if (stat.getIntegration() == null) { - gen.writeNull("integration"); - } else { - gen.write("integration", new BigDecimal(decimalFormatter.format(stat.getIntegration()))); - } - - if (stat.getMax() == null) { - gen.writeNull("max"); - } else { - gen.write("max", new BigDecimal(decimalFormatter.format(stat.getMax()))); - } - if (stat.getMean() == null) { - gen.writeNull("mean"); - } else { - gen.write("mean", new BigDecimal(decimalFormatter.format(stat.getMean()))); - } - if (stat.getMin() == null) { - gen.writeNull("min"); - } else { - gen.write("min", new BigDecimal(decimalFormatter.format(stat.getMin()))); - } - if (stat.getRms() == null) { - gen.writeNull("rms"); - } else { - gen.write("rms", new BigDecimal(decimalFormatter.format(stat.getRms()))); - } - if (stat.getSigma() == null) { - gen.writeNull("stdev"); - } else { - gen.write("stdev", new BigDecimal(decimalFormatter.format(stat.getSigma()))); - } - gen.writeEnd(); + for(Instant begin : stats.keySet()) { + writeRunningStatistics(null, gen, stats.get(begin), begin, formatAsMillisSinceEpoch, + adjustMillisWithServerOffset, timestampFormatter, decimalFormatter); count++; } + gen.writeEnd(); return count; } diff --git a/src/main/webapp/mysampler-form.html b/src/main/webapp/mysampler-form.html index 01b6094..c6b622e 100644 --- a/src/main/webapp/mysampler-form.html +++ b/src/main/webapp/mysampler-form.html @@ -13,7 +13,7 @@

mysampler query

  • - +
  • diff --git a/src/main/webapp/mystats-form.html b/src/main/webapp/mystats-form.html index 1e8de44..9c7a9ee 100644 --- a/src/main/webapp/mystats-form.html +++ b/src/main/webapp/mystats-form.html @@ -16,7 +16,7 @@

    mystats query

    • - +