PRIS-151: Added support for meta-data

docs/content/source/jdbc/_index.en.adoc

@@ -34,8 +34,11 @@ The following column-headers will be mapped from the result set to the OpenNMS r
 | `Location`                |          | The monitoring location for the node. When not set, the node is monitored from the _OpenNMS_ server, otherwise from the _Minion_ associated with the `Location`.
 | `Cat`                     |          | will be interpreted as a surveillance category for the node identified by the `Foreign_Id`
 | `Svc`                     |          | will be interpreted as a service on the interface of the node identified by the `Foreign_Id` and `IP_Address` field
+| `MetaData_`               |          | will be interpreted as node-level meta-data with the given key and the default context `requisition`. You can use `MetaData_Context:Key` to specify a custom context.
 |========================
 
+CAUTION: Please note, that this datasource only allows to specify node-level meta-data.
+
 This source also supports all asset fields by using `Asset_` as a prefix followed by the `asset-field-name`.
 The city field of the assets can be addressed like this: `yourvalue AS Asset_City` and is not case-sensitive.
 

docs/content/source/xls/_index.en.adoc

@@ -32,8 +32,11 @@ These `column names` have to start with certain prefixes to be recognized.
 | `InterfaceStatus`         |          | will be interpreted as interface status. Use `1` for monitored and `3` for not monitored.
 | `cat_`                    |          | will be interpreted as a surveillance category. Multiple comma-separated categories can be provided. It can be used multiple times per sheet.
 | `svc_`                    |          | will be interpreted as a service on the interface of the node. Multiple comma-separated services can be provided. It can be used multiple times per sheet.
+| `MetaData_`               |          | will be interpreted as node-level meta-data with the given key and the default context `requisition`. You can use `MetaData_Context:Key` to specify a custom context.
 |========================
 
+CAUTION: Please note, that this datasource only allows to specify node-level meta-data.
+
 This source also supports all asset-fields by using `Asset_` as a prefix followed by the `asset-field-name`.
 The city field of the assets can be addressed like this: `Asset_City`.
 This is not case-sensitive.

opennms-pris-plugins/opennms-pris-plugins-jdbc/src/main/java/org/opennms/opennms/pris/plugins/jdbc/source/JdbcSource.java

@@ -22,13 +22,15 @@
 import java.sql.Connection;
 import java.sql.DriverManager;
 import java.sql.ResultSet;
+import java.sql.ResultSetMetaData;
 import java.sql.SQLException;
 import java.sql.Statement;
 
 import org.kohsuke.MetaInfServices;
 import org.opennms.pris.api.InstanceConfiguration;
 import org.opennms.pris.api.Source;
 import org.opennms.pris.model.AssetField;
+import org.opennms.pris.model.MetaData;
 import org.opennms.pris.model.PrimaryType;
 import org.opennms.pris.model.Requisition;
 import org.opennms.pris.model.RequisitionAsset;
@@ -40,6 +42,8 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.google.common.base.Strings;
+
 /**
  * A JDBC data source allows to connect to an SQL database and extract data in a given format. The result set is mapped to an OpenNMS requisition.
  */
@@ -65,6 +69,7 @@ public class JdbcSource implements Source {
     private static final String COLUMN_PARENT_NODE_LABEL = "Parent_Node_Label";
     private static final String COLUMN_PARENT_FOREIGN_ID = "Parent_Foreign_Id";
     private static final String COLUMN_PARENT_FOREIGN_SOURCE = "Parent_Foreign_Source";
+    private static final String COLUMN_METADATA_PREFIX = "MetaData_";
 
     public JdbcSource(final InstanceConfiguration config) {
         this.config = config;
@@ -207,6 +212,29 @@ public Object dump() {
                             asset.setValue(assetValue);
                         }
                     }
+
+                    final ResultSetMetaData resultSetMetaData = resultSet.getMetaData();
+
+                    for (int i = 0; i < resultSetMetaData.getColumnCount(); i++) {
+                        final String columnName = resultSetMetaData.getColumnLabel(i + 1);
+
+                        if (columnName.toLowerCase().startsWith(COLUMN_METADATA_PREFIX.toLowerCase())) {
+                            final String value = getString(resultSet, columnName);
+
+                            if (!Strings.isNullOrEmpty(value)) {
+                                String context = "requisition";
+                                String key = columnName.substring(COLUMN_METADATA_PREFIX.length());
+                                final int index = key.indexOf(":");
+
+                                if (index != -1) {
+                                    context = key.substring(0, index);
+                                    key = key.substring(index + 1);
+                                }
+
+                                node.getMetaDatas().add(new MetaData(context, key, value));
+                            }
+                        }
+                    }
                 }
             } else {
                 LOGGER.error("ResultSet is null");

opennms-pris-plugins/opennms-pris-plugins-jdbc/src/test/java/org/opennms/opennms/pris/plugins/jdbc/source/JdbcSourceTest.java

@@ -6,13 +6,16 @@
 import java.sql.SQLException;
 import java.sql.Statement;
 
+import org.hamcrest.Matchers;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.opennms.pris.api.MockInstanceConfiguration;
+import org.opennms.pris.model.MetaData;
 import org.opennms.pris.model.Requisition;
+import org.opennms.pris.model.RequisitionNode;
 
 public class JdbcSourceTest {
 
@@ -38,6 +41,8 @@ public class JdbcSourceTest {
             + "state VARCHAR(255),"
             + "serviceName VARCHAR(255),"
             + "categoryName VARCHAR(255),"
+            + "metaDataColumn1 VARCHAR(255),"
+            + "metaDataColumn2 VARCHAR(255),"
             + "PRIMARY KEY (foreignId))";
 
     private final String SQL_SELECT_STATEMENT_TEST_1 = "SELECT foreignId AS Foreign_Id, "
@@ -55,6 +60,8 @@ public class JdbcSourceTest {
             + "city AS Asset_City,"
             + "state AS Asset_State,"
             + "serviceName AS Svc,"
+            + "metaDataColumn1 AS \"MetaData_keyWithoutContext\","
+            + "metaDataColumn2 AS \"MetaData_Context:keyWithContext\","
             + "categoryName AS Cat FROM node";
 
     @BeforeClass
@@ -69,13 +76,13 @@ public void setUp() throws ClassNotFoundException, SQLException {
         Statement stmnt = connection.createStatement();
         stmnt.executeUpdate(SQL_CREATE_ALL);
 
-        insertRow("1", "node1", null, null, null, null, "192.168.0.1", "P", "1", "description1", "city1", "state1", "service1", "category1");
-        insertRow("2", "node2", "Test-Location", "ParentNodeLabel", "", "", "192.168.0.2", "P", "3", "description2", "city2", "state2", "service2", "category2");
-        insertRow("3", "node3", null, null, "ParentId", "ParentSouce", "192.168.0.3", "P", "1", "description3", "city3", "state3", "service3", "category3");
+        insertRow("1", "node1", null, null, null, null, "192.168.0.1", "P", "1", "description1", "city1", "state1", "service1", "category1", "Foo1", "Bar1");
+        insertRow("2", "node2", "Test-Location", "ParentNodeLabel", "", "", "192.168.0.2", "P", "3", "description2", "city2", "state2", "service2", "category2", "Foo2", "Bar2");
+        insertRow("3", "node3", null, null, "ParentId", "ParentSouce", "192.168.0.3", "P", "1", "description3", "city3", "state3", "service3", "category3", "Foo3", "Bar3");
     }
 
-    private void insertRow(String foreignId, String nodeLabel, String location, String parentNodeLabel, String parentForeignId, String parentForeignSource, String ipAddress, String ifType, String ifStatus, String description, String city, String state, String serviceName, String categoryName) throws SQLException {
-        String DML = "INSERT INTO node (foreignId, nodeLabel, location, parentNodeLabel, parentForeignId, parentForeignSource, ipAddress, ifType, ifStatus, description, city, state, serviceName, categoryName) VALUES (" + foreignId + ", '" + nodeLabel + "', '" + location + "', '" + parentNodeLabel + "', '" + parentForeignId + "', '" + parentForeignSource + "', '" + ipAddress + "', '" + ifType + "', '" + ifStatus + "', '" + description + "', '" + city + "', '" + state + "', '" + serviceName + "', '" + categoryName + "')";
+    private void insertRow(String foreignId, String nodeLabel, String location, String parentNodeLabel, String parentForeignId, String parentForeignSource, String ipAddress, String ifType, String ifStatus, String description, String city, String state, String serviceName, String categoryName, String metaDataColumn1, String metaDataColumn2) throws SQLException {
+        String DML = "INSERT INTO node (foreignId, nodeLabel, location, parentNodeLabel, parentForeignId, parentForeignSource, ipAddress, ifType, ifStatus, description, city, state, serviceName, categoryName, metaDataColumn1, metaDataColumn2) VALUES (" + foreignId + ", '" + nodeLabel + "', '" + location + "', '" + parentNodeLabel + "', '" + parentForeignId + "', '" + parentForeignSource + "', '" + ipAddress + "', '" + ifType + "', '" + ifStatus + "', '" + description + "', '" + city + "', '" + state + "', '" + serviceName + "', '" + categoryName + "', '" + metaDataColumn1 +"', '" + metaDataColumn2 +"')";
         Statement stmnt = connection.createStatement();
         stmnt.executeUpdate(DML);
     }
@@ -115,5 +122,13 @@ public void testDump() {
         Assert.assertEquals(3, requisition.getNodes().size());
         Assert.assertEquals("Test-Location", requisition.getNodes().get(1).getLocation());
         Assert.assertEquals("ParentNodeLabel", requisition.getNodes().get(1).getParentNodeLabel());
+
+        final RequisitionNode node1 = requisition.getNodes().stream().filter(n -> n.getForeignId().equals("1")).findFirst().get();
+        final RequisitionNode node2 = requisition.getNodes().stream().filter(n -> n.getForeignId().equals("2")).findFirst().get();
+        final RequisitionNode node3 = requisition.getNodes().stream().filter(n -> n.getForeignId().equals("3")).findFirst().get();
+
+        Assert.assertThat(node1.getMetaDatas(), Matchers.containsInAnyOrder(new MetaData("requisition", "keyWithoutContext", "Foo1"), new MetaData("Context", "keyWithContext", "Bar1")));
+        Assert.assertThat(node2.getMetaDatas(), Matchers.containsInAnyOrder(new MetaData("requisition", "keyWithoutContext", "Foo2"), new MetaData("Context", "keyWithContext", "Bar2")));
+        Assert.assertThat(node3.getMetaDatas(), Matchers.containsInAnyOrder(new MetaData("requisition", "keyWithoutContext", "Foo3"), new MetaData("Context", "keyWithContext", "Bar3")));
     }
 }

opennms-pris-plugins/opennms-pris-plugins-xls/src/main/java/org/opennms/opennms/pris/plugins/xls/source/XlsSource.java

@@ -42,6 +42,7 @@
 import org.opennms.pris.api.InstanceConfiguration;
 import org.opennms.pris.api.Source;
 import org.opennms.pris.model.AssetField;
+import org.opennms.pris.model.MetaData;
 import org.opennms.pris.model.PrimaryType;
 import org.opennms.pris.model.Requisition;
 import org.opennms.pris.model.RequisitionAsset;
@@ -52,6 +53,8 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.google.common.base.Strings;
+
 public class XlsSource implements Source {
 
 	private static final Logger LOGGER = LoggerFactory
@@ -62,13 +65,17 @@ public class XlsSource implements Source {
 	private final String WITHIN_SPLITTER = ",";
 
 	private final String PREFIX_FOR_ASSETS = "Asset_";
+	private final String PREFIX_FOR_METADATA = "MetaData_";
+
 	private final String INTERFACE_TYPE_PRIMARY = "P";
 	private final String INTERFACE_TYPE_SECONDARY = "S";
 
 	private Map<String, Integer> requiredColumns;
 	private Map<String, List<Integer>> optionalMultiColumns;
 	private Map<String, Integer> optionalUniquHeaders;
 	private Map<String, Integer> assetColumns;
+	private Map<String, List<Integer>> metaDataColumns;
+
 
 	private File xls;
 	private final String encoding;
@@ -159,6 +166,7 @@ public Object dump() throws MissingRequiredColumnHeaderException, Exception {
 		optionalMultiColumns = initializeOptionalMultiColumns(sheet);
 		optionalUniquHeaders = initializeOptionalUniquHeaders(sheet);
 		assetColumns = initializeAssetColumns(sheet);
+		metaDataColumns = initializeMetaDataColumns(sheet);
 
 		RequisitionInterface reqInterface;
 		Iterator<Row> rowiterator = sheet.rowIterator();
@@ -214,6 +222,9 @@ public Object dump() throws MissingRequiredColumnHeaderException, Exception {
 			// adding assets
 			node.getAssets().addAll(getAssetsByRow(row));
 
+			// adding meta-data
+			node.getMetaDatas().addAll(getMetaDataByRow(row));
+
 			// Add interface
 			reqInterface = getInterfaceByRow(row);
 
@@ -305,6 +316,24 @@ private Map<String, List<Integer>> initializeOptionalMultiColumns(
 		return result;
 	}
 
+	private Map<String, List<Integer>> initializeMetaDataColumns(final Sheet sheet) {
+		final Map<String, List<Integer>> result = new HashMap<>();
+		final Row row = sheet.getRow(0);
+		final Iterator<Cell> celliterator = row.cellIterator();
+		while (celliterator.hasNext()) {
+			Cell cell = celliterator.next();
+			final String cellName = cell.getStringCellValue();
+			if (cellName.toLowerCase().startsWith(PREFIX_FOR_METADATA.toLowerCase())) {
+				final String metaDataKey = cellName.substring(PREFIX_FOR_METADATA.length());
+				if (!result.containsKey(metaDataKey)) {
+					result.put(metaDataKey, new ArrayList<>());
+				}
+				result.get(metaDataKey).add(cell.getColumnIndex());
+			}
+		}
+		return result;
+	}
+
 	private Map<String, Integer> initializeAssetColumns(Sheet sheet) {
 		Map<String, Integer> result = new HashMap<>();
 		for (AssetField prefix : AssetField.values()) {
@@ -392,6 +421,30 @@ private Set<RequisitionAsset> getAssetsByRow(Row row) {
 		return assets;
 	}
 
+	private Set<MetaData> getMetaDataByRow(Row row) {
+		Set<MetaData> metaData = new HashSet<>();
+		for (final Map.Entry<String, List<Integer>> entry : metaDataColumns.entrySet()) {
+			for (final int columnIndex : entry.getValue()) {
+				final Cell cell = row.getCell(columnIndex);
+				if (cell == null) {
+					continue;
+				}
+				final String value = XlsSource.getStringValueFromCell(cell);
+				if (!Strings.isNullOrEmpty(value)) {
+					String context = "requisition";
+					String key = entry.getKey();
+					final int index = entry.getKey().indexOf(":");
+					if (index != -1) {
+						context = entry.getKey().substring(0, index);
+						key = entry.getKey().substring(index + 1);
+					}
+					metaData.add(new MetaData(context, key, value));
+				}
+			}
+		}
+		return metaData;
+	}
+
 	private RequisitionInterface getInterfaceByRow(Row row)
 			throws InvalidInterfaceException {
 		RequisitionInterface reqInterface = new RequisitionInterface();

opennms-pris-plugins/opennms-pris-plugins-xls/src/test/java/org/opennms/opennms/pris/plugins/xls/source/XlsSourceTest.java

@@ -1,11 +1,17 @@
 package org.opennms.opennms.pris.plugins.xls.source;
 
 import java.nio.file.Paths;
+
+import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
+
 import org.junit.Before;
 import org.junit.Test;
 import org.opennms.pris.api.MockInstanceConfiguration;
 import org.opennms.pris.model.AssetField;
+import org.opennms.pris.model.MetaData;
 import org.opennms.pris.model.PrimaryType;
 import org.opennms.pris.model.Requisition;
 import org.opennms.pris.model.RequisitionCategory;
@@ -58,6 +64,10 @@ public void basicTest() throws Exception {
 
         RequisitionCategory findCategory = RequisitionUtils.findCategory(resultNode, "Test");
         assertEquals("Test", findCategory.getName());
+
+        assertThat(resultNode.getMetaDatas(), containsInAnyOrder(
+                new MetaData("requisition", "KeyWithoutContext", "Foo"),
+                new MetaData("Context", "KeyWithContext", "Bar")));
     }
 
     @Test

pom.xml

@@ -33,14 +33,15 @@
         <commonsIoVersion>2.6</commonsIoVersion>
         <groovyVersion>2.4.15</groovyVersion>
         <guavaVersion>27.0-jre</guavaVersion>
+        <hamcrestVersion>2.1</hamcrestVersion>
         <httpComponentsVersion>4.5.6</httpComponentsVersion>
         <jettyServerVersion>9.4.20.v20190813</jettyServerVersion>
         <junitVersion>4.12</junitVersion>
         <mavenResourcesPluginVersion>3.1.0</mavenResourcesPluginVersion>
         <mavenCompilerPluginVersion>3.8.0</mavenCompilerPluginVersion>
         <metainfServicesVersion>1.8</metainfServicesVersion>
         <mysqlConnectorVersion>8.0.13</mysqlConnectorVersion>
-        <opennmsVersion>22.0.4</opennmsVersion>
+        <opennmsVersion>24.1.3</opennmsVersion>
         <postgresqlDriverVersion>42.2.5</postgresqlDriverVersion>
         <slf4jVersion>1.7.25</slf4jVersion>
         <updatePolicy>interval:60</updatePolicy>
@@ -77,6 +78,13 @@
             <version>${junitVersion}</version>
             <scope>test</scope>
         </dependency>
+
+        <dependency>
+            <groupId>org.hamcrest</groupId>
+            <artifactId>hamcrest</artifactId>
+            <version>${hamcrestVersion}</version>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 
     <dependencyManagement>