From 84dcca0bb10267276d13fd336668499dff4da969 Mon Sep 17 00:00:00 2001 From: Asif Sohail Mohammed Date: Mon, 29 Jan 2024 14:01:11 -0600 Subject: [PATCH] Added when condition and fixed building reader on each event (#4002) * Added when condition and fixed building reader on each event Signed-off-by: Asif Sohail Mohammed --- .../processor/GeoIPProcessorUrlServiceIT.java | 5 +- .../plugins/processor/GeoIPProcessor.java | 56 +++++---- .../processor/GeoIPProcessorConfig.java | 10 ++ .../processor/configuration/EntryConfig.java | 3 +- .../processor/databaseenrich/GetGeoData.java | 12 +- .../databaseenrich/GetGeoIP2Data.java | 20 +-- .../databaseenrich/GetGeoLite2Data.java | 46 +++---- .../extension/DefaultGeoIpConfigSupplier.java | 2 - .../GeoIPProcessorService.java | 57 +++++---- .../extension/GeoIpConfigSupplier.java | 2 - .../databasedownload/DBSource.java | 8 +- .../databasedownload/DBSourceOptions.java | 2 +- .../DatabaseReaderCreate.java | 9 +- .../databasedownload/GeoDataFactory.java | 39 ++++++ .../HttpDBDownloadService.java | 2 +- .../databasedownload/LicenseTypeOptions.java | 2 +- .../LocalDBDownloadService.java | 9 +- .../databasedownload/S3DBService.java | 2 +- .../utils/DbSourceIdentification.java | 2 +- ...ationcheck.java => IPValidationCheck.java} | 9 +- .../processor/utils/LicenseTypeCheck.java | 7 +- .../processor/GeoIPProcessorConfigTest.java | 12 +- .../plugins/processor/GeoIPProcessorTest.java | 114 +++++++++++++++--- .../{EntryTest.java => EntryConfigTest.java} | 10 +- .../DatabaseReaderCreateTest.java | 26 ---- .../databaseenrich/GetGeoIP2DataTest.java | 4 +- .../databaseenrich/GetGeoLite2DataTest.java | 4 +- .../DefaultGeoIpConfigSupplierTest.java | 1 - .../GeoIPProcessorServiceTest.java | 4 +- .../databasedownload/DBSourceOptionsTest.java | 2 +- .../databasedownload/DBSourceTest.java | 2 +- .../DatabaseReaderCreateTest.java | 43 +++++++ .../databasedownload/GeoDataFactoryTest.java | 51 ++++++++ .../HttpDBDownloadServiceTest.java | 2 +- .../LicenseTypeOptionsTest.java | 2 +- .../LocalDBDownloadServiceTest.java | 4 +- .../databasedownload/S3DBServiceTest.java | 2 +- .../utils/DbSourceIdentificationTest.java | 2 +- ...ckTest.java => IPValidationCheckTest.java} | 8 +- .../processor/utils/LicenseTypeCheckTest.java | 16 ++- 40 files changed, 415 insertions(+), 198 deletions(-) rename data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/{ => extension}/GeoIPProcessorService.java (66%) rename data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/{ => extension}/databasedownload/DBSource.java (91%) rename data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/{ => extension}/databasedownload/DBSourceOptions.java (91%) rename data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/{ => extension}/databasedownload/DatabaseReaderCreate.java (67%) create mode 100644 data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/GeoDataFactory.java rename data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/{ => extension}/databasedownload/HttpDBDownloadService.java (98%) rename data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/{ => extension}/databasedownload/LicenseTypeOptions.java (91%) rename data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/{ => extension}/databasedownload/LocalDBDownloadService.java (75%) rename data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/{ => extension}/databasedownload/S3DBService.java (98%) rename data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/utils/{IPValidationcheck.java => IPValidationCheck.java} (75%) rename data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/configuration/{EntryTest.java => EntryConfigTest.java} (85%) delete mode 100644 data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/databasedownload/DatabaseReaderCreateTest.java rename data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/{ => extension}/GeoIPProcessorServiceTest.java (96%) rename data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/{ => extension}/databasedownload/DBSourceOptionsTest.java (90%) rename data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/{ => extension}/databasedownload/DBSourceTest.java (94%) create mode 100644 data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/DatabaseReaderCreateTest.java create mode 100644 data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/GeoDataFactoryTest.java rename data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/{ => extension}/databasedownload/HttpDBDownloadServiceTest.java (92%) rename data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/{ => extension}/databasedownload/LicenseTypeOptionsTest.java (91%) rename data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/{ => extension}/databasedownload/LocalDBDownloadServiceTest.java (92%) rename data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/{ => extension}/databasedownload/S3DBServiceTest.java (93%) rename data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/utils/{IPValidationcheckTest.java => IPValidationCheckTest.java} (81%) diff --git a/data-prepper-plugins/geoip-processor/src/integrationTest/java/org/opensearch/dataprepper/plugins/processor/GeoIPProcessorUrlServiceIT.java b/data-prepper-plugins/geoip-processor/src/integrationTest/java/org/opensearch/dataprepper/plugins/processor/GeoIPProcessorUrlServiceIT.java index 3c90b1936e..70a80c0684 100644 --- a/data-prepper-plugins/geoip-processor/src/integrationTest/java/org/opensearch/dataprepper/plugins/processor/GeoIPProcessorUrlServiceIT.java +++ b/data-prepper-plugins/geoip-processor/src/integrationTest/java/org/opensearch/dataprepper/plugins/processor/GeoIPProcessorUrlServiceIT.java @@ -14,7 +14,8 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.junit.jupiter.MockitoExtension; -import org.opensearch.dataprepper.plugins.processor.utils.IPValidationcheck; +import org.opensearch.dataprepper.plugins.processor.extension.GeoIPProcessorService; +import org.opensearch.dataprepper.plugins.processor.utils.IPValidationCheck; import java.io.File; import java.net.InetAddress; @@ -89,7 +90,7 @@ void verify_enrichment_of_data_from_maxmind_url() throws UnknownHostException { Map geoData = new HashMap<>(); this.geoIPProcessorService = createObjectUnderTest(); String ipAddress = geoIPInputJson.getPeer().getIp(); - if (IPValidationcheck.isPublicIpAddress(ipAddress)) { + if (IPValidationCheck.isPublicIpAddress(ipAddress)) { InetAddress inetAddress = InetAddress.getByName(ipAddress); //All attributes are considered by default with the null value geoData = geoIPProcessorService.getGeoData(inetAddress, null); diff --git a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/GeoIPProcessor.java b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/GeoIPProcessor.java index e9fd27ee3c..3b3e42f9d0 100644 --- a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/GeoIPProcessor.java +++ b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/GeoIPProcessor.java @@ -6,17 +6,19 @@ package org.opensearch.dataprepper.plugins.processor; import io.micrometer.core.instrument.Counter; +import org.opensearch.dataprepper.expression.ExpressionEvaluator; +import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor; -import org.opensearch.dataprepper.model.configuration.PluginSetting; import org.opensearch.dataprepper.model.event.Event; import org.opensearch.dataprepper.model.processor.AbstractProcessor; import org.opensearch.dataprepper.model.processor.Processor; import org.opensearch.dataprepper.model.record.Record; import org.opensearch.dataprepper.plugins.processor.configuration.EntryConfig; import org.opensearch.dataprepper.plugins.processor.databaseenrich.EnrichFailedException; +import org.opensearch.dataprepper.plugins.processor.extension.GeoIPProcessorService; import org.opensearch.dataprepper.plugins.processor.extension.GeoIpConfigSupplier; -import org.opensearch.dataprepper.plugins.processor.utils.IPValidationcheck; +import org.opensearch.dataprepper.plugins.processor.utils.IPValidationCheck; import org.opensearch.dataprepper.logging.DataPrepperMarkers; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -35,28 +37,34 @@ public class GeoIPProcessor extends AbstractProcessor, Record> { private static final Logger LOG = LoggerFactory.getLogger(GeoIPProcessor.class); - private static final String GEO_IP_PROCESSING_MATCH = "geoIpProcessingMatch"; - private static final String GEO_IP_PROCESSING_MISMATCH = "geoIpProcessingMismatch"; + //TODO: rename metrics + static final String GEO_IP_PROCESSING_MATCH = "geoIpProcessingMatch"; + static final String GEO_IP_PROCESSING_MISMATCH = "geoIpProcessingMismatch"; private final Counter geoIpProcessingMatchCounter; private final Counter geoIpProcessingMismatchCounter; private final GeoIPProcessorConfig geoIPProcessorConfig; - private final List tagsOnSourceNotFoundFailure; - private GeoIPProcessorService geoIPProcessorService; + private final List tagsOnFailure; + private final GeoIPProcessorService geoIPProcessorService; + private final String whenCondition; + private final ExpressionEvaluator expressionEvaluator; /** * GeoIPProcessor constructor for initialization of required attributes - * @param pluginSetting pluginSetting + * @param pluginMetrics pluginMetrics * @param geoIPProcessorConfig geoIPProcessorConfig * @param geoIpConfigSupplier geoIpConfigSupplier */ @DataPrepperPluginConstructor - public GeoIPProcessor(PluginSetting pluginSetting, + public GeoIPProcessor(final PluginMetrics pluginMetrics, final GeoIPProcessorConfig geoIPProcessorConfig, - final GeoIpConfigSupplier geoIpConfigSupplier) { - super(pluginSetting); + final GeoIpConfigSupplier geoIpConfigSupplier, + final ExpressionEvaluator expressionEvaluator) { + super(pluginMetrics); this.geoIPProcessorConfig = geoIPProcessorConfig; this.geoIPProcessorService = geoIpConfigSupplier.getGeoIPProcessorService(); - this.tagsOnSourceNotFoundFailure = geoIPProcessorConfig.getTagsOnFailure(); + this.tagsOnFailure = geoIPProcessorConfig.getTagsOnFailure(); + this.whenCondition = geoIPProcessorConfig.getWhenCondition(); + this.expressionEvaluator = expressionEvaluator; this.geoIpProcessingMatchCounter = pluginMetrics.counter(GEO_IP_PROCESSING_MATCH); this.geoIpProcessingMismatchCounter = pluginMetrics.counter(GEO_IP_PROCESSING_MISMATCH); } @@ -67,33 +75,35 @@ public GeoIPProcessor(PluginSetting pluginSetting, * @return collection of record events */ @Override - public Collection> doExecute(Collection> records) { - + public Collection> doExecute(final Collection> records) { Map geoData; for (final Record eventRecord : records) { - Event event = eventRecord.getData(); + final Event event = eventRecord.getData(); + if (whenCondition != null && !expressionEvaluator.evaluateConditional(whenCondition, event)) { + continue; + } for (EntryConfig entry : geoIPProcessorConfig.getEntries()) { - String source = entry.getSource(); - List attributes = entry.getFields(); - String ipAddress = event.get(source, String.class); + final String source = entry.getSource(); + final List attributes = entry.getFields(); + final String ipAddress = event.get(source, String.class); //Lookup from DB - if (ipAddress != null && (!(ipAddress.isEmpty()))) { + if (ipAddress != null && !ipAddress.isEmpty()) { try { - if (IPValidationcheck.isPublicIpAddress(ipAddress)) { + if (IPValidationCheck.isPublicIpAddress(ipAddress)) { geoData = geoIPProcessorService.getGeoData(InetAddress.getByName(ipAddress), attributes); eventRecord.getData().put(entry.getTarget(), geoData); geoIpProcessingMatchCounter.increment(); } - } catch (IOException | EnrichFailedException ex) { + } catch (final IOException | EnrichFailedException ex) { geoIpProcessingMismatchCounter.increment(); - event.getMetadata().addTags(tagsOnSourceNotFoundFailure); - LOG.error(DataPrepperMarkers.EVENT, "Failed to get Geo data for event: [{}] for the IP address [{}]", event, ipAddress, ex); + event.getMetadata().addTags(tagsOnFailure); + LOG.error(DataPrepperMarkers.EVENT, "Failed to get Geo data for event: [{}] for the IP address [{}]", event, ipAddress, ex); } } else { //No Enrichment. - event.getMetadata().addTags(tagsOnSourceNotFoundFailure); + event.getMetadata().addTags(tagsOnFailure); } } } diff --git a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/GeoIPProcessorConfig.java b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/GeoIPProcessorConfig.java index 0dbbc2b566..ca0b00e8e7 100644 --- a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/GeoIPProcessorConfig.java +++ b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/GeoIPProcessorConfig.java @@ -27,6 +27,9 @@ public class GeoIPProcessorConfig { @JsonProperty("tags_on_failure") private List tagsOnFailure; + @JsonProperty("geoip_when") + private String whenCondition; + /** * Get List of entries @@ -44,4 +47,11 @@ public List getTagsOnFailure() { return tagsOnFailure; } + /** + * Get when condition + * @return String When condition + */ + public String getWhenCondition() { + return whenCondition; + } } diff --git a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/configuration/EntryConfig.java b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/configuration/EntryConfig.java index 84695d4b4b..c1e95373a3 100644 --- a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/configuration/EntryConfig.java +++ b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/configuration/EntryConfig.java @@ -11,12 +11,13 @@ import java.util.List; public class EntryConfig { + static final String DEFAULT_TARGET = "geolocation"; @JsonProperty("source") @NotEmpty private String source; @JsonProperty("target") - private String target; + private String target = DEFAULT_TARGET; @JsonProperty("fields") private List fields; diff --git a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/databaseenrich/GetGeoData.java b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/databaseenrich/GetGeoData.java index 5ac4e5bbe8..3730be7990 100644 --- a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/databaseenrich/GetGeoData.java +++ b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/databaseenrich/GetGeoData.java @@ -20,9 +20,9 @@ public interface GetGeoData { public final String GeoLite2AsnDB = "GeoLite2-ASN.mmdb"; public final String GeoIP2EnterpriseDB = "GeoIP2-Enterprise.mmdb"; - public void switchDatabaseReader(); - public void closeReader(); - public Map getGeoData(InetAddress inetAddress, List attributes, String tempDestDir); + void switchDatabaseReader(); + void closeReader(); + Map getGeoData(InetAddress inetAddress, List attributes, String tempDestDir); /** * Enrich attributes @@ -30,7 +30,7 @@ public interface GetGeoData { * @param attributeName attributeName * @param attributeValue attributeValue */ - public default void enrichData(Map geoData,String attributeName, String attributeValue) { + default void enrichData(final Map geoData, final String attributeName, final String attributeValue) { if (attributeValue != null) { geoData.put(attributeName, attributeValue); } @@ -42,7 +42,7 @@ public default void enrichData(Map geoData,String attributeName, * @param countryIso countryIso * @param subdivisionIso subdivisionIso */ - public default void enrichRegionIsoCode(Map geoData, String countryIso, String subdivisionIso) { + default void enrichRegionIsoCode(final Map geoData, final String countryIso, final String subdivisionIso) { if (countryIso != null && subdivisionIso != null) { enrichData(geoData, "region_iso_code", countryIso + "-" + subdivisionIso); } @@ -54,7 +54,7 @@ public default void enrichRegionIsoCode(Map geoData, String coun * @param latitude latitude * @param longitude longitude */ - public default void enrichLocationData(Map geoData, Double latitude, Double longitude) { + default void enrichLocationData(final Map geoData, final Double latitude, final Double longitude) { if (latitude != null && longitude != null) { Map locationObject = new HashMap<>(); locationObject.put("lat", latitude); diff --git a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/databaseenrich/GetGeoIP2Data.java b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/databaseenrich/GetGeoIP2Data.java index 6fb09b6f65..70d888f587 100644 --- a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/databaseenrich/GetGeoIP2Data.java +++ b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/databaseenrich/GetGeoIP2Data.java @@ -14,9 +14,9 @@ import com.maxmind.geoip2.record.Subdivision; import com.maxmind.geoip2.record.Location; import com.maxmind.geoip2.record.Postal; -import org.opensearch.dataprepper.plugins.processor.GeoIPProcessorService; -import org.opensearch.dataprepper.plugins.processor.databasedownload.DatabaseReaderCreate; -import org.opensearch.dataprepper.plugins.processor.databasedownload.DBSource; +import org.opensearch.dataprepper.plugins.processor.extension.GeoIPProcessorService; +import org.opensearch.dataprepper.plugins.processor.extension.databasedownload.DatabaseReaderCreate; +import org.opensearch.dataprepper.plugins.processor.extension.databasedownload.DBSource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -44,7 +44,7 @@ public class GetGeoIP2Data implements GetGeoData { public static final String TIMEZONE = "timezone"; public static final String LOCATION = "location"; public static final String POSTAL = "postal"; - private DatabaseReader.Builder readerEnterprise; + private DatabaseReader readerEnterprise; private Country country; private Continent continent; private City city; @@ -70,7 +70,11 @@ public GetGeoIP2Data(final String dbPath, final int cacheSize) { * Initialise all the DatabaseReader */ public void initDatabaseReader() { - readerEnterprise = DatabaseReaderCreate.createLoader(Path.of(dbPath + File.separator + tempDestDir + File.separator + GeoIP2EnterpriseDB), cacheSize); + try { + readerEnterprise = DatabaseReaderCreate.createLoader(Path.of(dbPath + File.separator + tempDestDir + File.separator + GeoIP2EnterpriseDB), cacheSize); + } catch (final IOException ex) { + LOG.error("Exception while creating GeoIP2 DatabaseReader: {0}", ex); + } } /** @@ -78,7 +82,7 @@ public void initDatabaseReader() { */ @Override public void switchDatabaseReader() { - LOG.info("Switch DatabaseReader"); + LOG.info("Switching GeoIP2 DatabaseReader"); closeReader(); System.gc(); File file = new File(dbPath); @@ -100,7 +104,7 @@ public Map getGeoData(InetAddress inetAddress, List attr switchDatabaseReader(); } try { - EnterpriseResponse enterpriseResponse = readerEnterprise.build().enterprise(inetAddress); + EnterpriseResponse enterpriseResponse = readerEnterprise.enterprise(inetAddress); country = enterpriseResponse.getCountry(); subdivision = enterpriseResponse.getMostSpecificSubdivision(); city = enterpriseResponse.getCity(); @@ -179,7 +183,7 @@ public Map getGeoData(InetAddress inetAddress, List attr public void closeReader() { try { if (readerEnterprise != null) - readerEnterprise.build().close(); + readerEnterprise.close(); } catch (IOException ex) { LOG.info("Close Enterprise DatabaseReader Exception : {0}", ex); } diff --git a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/databaseenrich/GetGeoLite2Data.java b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/databaseenrich/GetGeoLite2Data.java index 3ccc6f0f82..0d719541e0 100644 --- a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/databaseenrich/GetGeoLite2Data.java +++ b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/databaseenrich/GetGeoLite2Data.java @@ -16,9 +16,9 @@ import com.maxmind.geoip2.record.Country; import com.maxmind.geoip2.record.Subdivision; import com.maxmind.geoip2.record.Location; -import org.opensearch.dataprepper.plugins.processor.GeoIPProcessorService; -import org.opensearch.dataprepper.plugins.processor.databasedownload.DBSource; -import org.opensearch.dataprepper.plugins.processor.databasedownload.DatabaseReaderCreate; +import org.opensearch.dataprepper.plugins.processor.extension.GeoIPProcessorService; +import org.opensearch.dataprepper.plugins.processor.extension.databasedownload.DBSource; +import org.opensearch.dataprepper.plugins.processor.extension.databasedownload.DatabaseReaderCreate; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -48,9 +48,9 @@ public class GetGeoLite2Data implements GetGeoData { public static final String TIMEZONE = "timezone"; public static final String LOCATION = "location"; public static final String ASN = "asn"; - private DatabaseReader.Builder readerCity; - private DatabaseReader.Builder readerCountry; - private DatabaseReader.Builder readerAsn; + private DatabaseReader readerCity; + private DatabaseReader readerCountry; + private DatabaseReader readerAsn; private Country country; private Continent continent; private City city; @@ -72,7 +72,7 @@ public class GetGeoLite2Data implements GetGeoData { * @param dbPath dbPath * @param cacheSize cacheSize */ - public GetGeoLite2Data(String dbPath, int cacheSize) { + public GetGeoLite2Data(final String dbPath, final int cacheSize) { this.dbPath = dbPath; this.cacheSize = cacheSize; initDatabaseReader(); @@ -82,9 +82,13 @@ public GetGeoLite2Data(String dbPath, int cacheSize) { * Initialise all the DatabaseReader */ private void initDatabaseReader() { - readerCity = DatabaseReaderCreate.createLoader(Path.of(dbPath + File.separator + GeoLite2CityDB), cacheSize); - readerCountry = DatabaseReaderCreate.createLoader(Path.of(dbPath + File.separator + GeoLite2CountryDB), cacheSize); - readerAsn = DatabaseReaderCreate.createLoader(Path.of(dbPath + File.separator + GeoLite2AsnDB), cacheSize); + try { + readerCity = DatabaseReaderCreate.createLoader(Path.of(dbPath + File.separator + GeoLite2CityDB), cacheSize); + readerCountry = DatabaseReaderCreate.createLoader(Path.of(dbPath + File.separator + GeoLite2CountryDB), cacheSize); + readerAsn = DatabaseReaderCreate.createLoader(Path.of(dbPath + File.separator + GeoLite2AsnDB), cacheSize); + } catch (final IOException ex) { + LOG.error("Exception while creating GeoLite2 DatabaseReader: {0}", ex); + } } /** @@ -92,12 +96,12 @@ private void initDatabaseReader() { */ @Override public void switchDatabaseReader() { - LOG.info("Switch DatabaseReader"); + LOG.info("Switching GeoLite2 DatabaseReaders"); closeReaderCity(); closeReaderCountry(); closeReaderAsn(); System.gc(); - File file = new File(dbPath); + final File file = new File(dbPath); DBSource.deleteDirectory(file); dbPath = tempDestDir; initDatabaseReader(); @@ -110,7 +114,7 @@ public void switchDatabaseReader() { * @return enriched data Map */ @Override - public Map getGeoData(InetAddress inetAddress, List attributes, String tempDestDir) { + public Map getGeoData(final InetAddress inetAddress, final List attributes, final String tempDestDir) { Map geoData = new HashMap<>(); if (GeoIPProcessorService.downloadReady) { this.tempDestDir = tempDestDir; @@ -118,16 +122,16 @@ public Map getGeoData(InetAddress inetAddress, List attr switchDatabaseReader(); } try { - responseCountry = readerCountry.build().country(inetAddress); + responseCountry = readerCountry.country(inetAddress); country = responseCountry.getCountry(); continent = responseCountry.getContinent(); - responseCity = readerCity.build().city(inetAddress); + responseCity = readerCity.city(inetAddress); city = responseCity.getCity(); location = responseCity.getLocation(); subdivision = responseCity.getMostSpecificSubdivision(); - responseAsn = readerAsn.build().asn(inetAddress); + responseAsn = readerAsn.asn(inetAddress); asn = responseAsn.getAutonomousSystemNumber(); organizationName = responseAsn.getAutonomousSystemOrganization(); network = responseAsn.getNetwork(); @@ -220,10 +224,10 @@ public void closeReader() { /** * Close the City DatabaseReader */ - private void closeReaderCity(){ + private void closeReaderCity() { try { if (readerCity != null) - readerCity.build().close(); + readerCity.close(); readerCity = null; } catch (IOException ex) { LOG.info("Close City DatabaseReader Exception : {0}", ex); @@ -233,10 +237,10 @@ private void closeReaderCity(){ /** * Close the Country DatabaseReader */ - private void closeReaderCountry(){ + private void closeReaderCountry() { try { if (readerCountry != null) - readerCountry.build().close(); + readerCountry.close(); readerCountry = null; } catch (IOException ex) { LOG.info("Close Country DatabaseReader Exception : {0}", ex); @@ -249,7 +253,7 @@ private void closeReaderCountry(){ private void closeReaderAsn() { try { if (readerAsn != null) - readerAsn.build().close(); + readerAsn.close(); readerAsn = null; } catch (IOException ex) { LOG.info("Close Asn DatabaseReader Exception : {0}", ex); diff --git a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/DefaultGeoIpConfigSupplier.java b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/DefaultGeoIpConfigSupplier.java index 2b98e22fba..08e50381bf 100644 --- a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/DefaultGeoIpConfigSupplier.java +++ b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/DefaultGeoIpConfigSupplier.java @@ -5,8 +5,6 @@ package org.opensearch.dataprepper.plugins.processor.extension; -import org.opensearch.dataprepper.plugins.processor.GeoIPProcessorService; - public class DefaultGeoIpConfigSupplier implements GeoIpConfigSupplier { private final GeoIpServiceConfig geoIpServiceConfig; diff --git a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/GeoIPProcessorService.java b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/GeoIPProcessorService.java similarity index 66% rename from data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/GeoIPProcessorService.java rename to data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/GeoIPProcessorService.java index 995b1a0469..7df7af684a 100644 --- a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/GeoIPProcessorService.java +++ b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/GeoIPProcessorService.java @@ -3,20 +3,16 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.dataprepper.plugins.processor; +package org.opensearch.dataprepper.plugins.processor.extension; -import org.opensearch.dataprepper.plugins.processor.databasedownload.DBSourceOptions; -import org.opensearch.dataprepper.plugins.processor.databasedownload.LicenseTypeOptions; -import org.opensearch.dataprepper.plugins.processor.databasedownload.S3DBService; -import org.opensearch.dataprepper.plugins.processor.databasedownload.DBSource; -import org.opensearch.dataprepper.plugins.processor.databasedownload.HttpDBDownloadService; -import org.opensearch.dataprepper.plugins.processor.databasedownload.LocalDBDownloadService; +import org.opensearch.dataprepper.plugins.processor.extension.databasedownload.DBSourceOptions; +import org.opensearch.dataprepper.plugins.processor.extension.databasedownload.GeoDataFactory; +import org.opensearch.dataprepper.plugins.processor.extension.databasedownload.S3DBService; +import org.opensearch.dataprepper.plugins.processor.extension.databasedownload.DBSource; +import org.opensearch.dataprepper.plugins.processor.extension.databasedownload.HttpDBDownloadService; +import org.opensearch.dataprepper.plugins.processor.extension.databasedownload.LocalDBDownloadService; import org.opensearch.dataprepper.plugins.processor.databaseenrich.DownloadFailedException; import org.opensearch.dataprepper.plugins.processor.databaseenrich.GetGeoData; -import org.opensearch.dataprepper.plugins.processor.databaseenrich.GetGeoIP2Data; -import org.opensearch.dataprepper.plugins.processor.databaseenrich.GetGeoLite2Data; -import org.opensearch.dataprepper.plugins.processor.extension.GeoIpServiceConfig; -import org.opensearch.dataprepper.plugins.processor.extension.MaxMindConfig; import org.opensearch.dataprepper.plugins.processor.utils.DbSourceIdentification; import org.opensearch.dataprepper.plugins.processor.utils.LicenseTypeCheck; import org.slf4j.Logger; @@ -41,19 +37,19 @@ public class GeoIPProcessorService { private static final Logger LOG = LoggerFactory.getLogger(GeoIPProcessorService.class); public static final String DATABASE_1 = "first_database_path"; public static final String DATABASE_2 = "second_database_path"; - private static final String TEMP_PATH_FOLDER = "GeoIP"; - private GeoIPProcessorConfig geoIPProcessorConfig; - private LicenseTypeOptions licenseType; + private static final String TEMP_PATH_FOLDER = "geoip"; + private final GeoDataFactory geoDataFactory; private GetGeoData geoData; private List databasePaths; private final String tempPath; private final ScheduledExecutorService scheduledExecutorService; private final DBSourceOptions dbSourceOptions; private final MaxMindConfig maxMindConfig; + private final LicenseTypeCheck licenseTypeCheck; public static volatile boolean downloadReady; private boolean toggle; private String flipDatabase; - + private boolean isDuringInitialization; /** * GeoIPProcessorService constructor for initialization of required attributes @@ -64,9 +60,12 @@ public GeoIPProcessorService(final GeoIpServiceConfig geoIpServiceConfig) { this.toggle = false; this.maxMindConfig = geoIpServiceConfig.getMaxMindConfig(); this.databasePaths = maxMindConfig.getDatabasePaths(); + this.isDuringInitialization = true; flipDatabase = DATABASE_1; - this.tempPath = System.getProperty("java.io.tmpdir")+ File.separator + TEMP_PATH_FOLDER; + licenseTypeCheck = new LicenseTypeCheck(); + geoDataFactory = new GeoDataFactory(maxMindConfig, licenseTypeCheck); + this.tempPath = System.getProperty("java.io.tmpdir") + File.separator + TEMP_PATH_FOLDER; dbSourceOptions = DbSourceIdentification.getDatabasePathType(databasePaths); final Duration checkInterval = Objects.requireNonNull(maxMindConfig.getDatabaseRefreshInterval()); @@ -79,17 +78,12 @@ public GeoIPProcessorService(final GeoIpServiceConfig geoIpServiceConfig) { while (!downloadReady) { wait(); } - } catch (InterruptedException ex) { - LOG.info("InterruptedException {0} ", ex); + } catch (final InterruptedException ex) { + LOG.info("Thread interrupted while waiting for download to complete: {0}", ex); Thread.currentThread().interrupt(); } - String finalPath = tempPath + File.separator; - licenseType = LicenseTypeCheck.isGeoLite2OrEnterpriseLicense(finalPath.concat(flipDatabase)); - if (licenseType.equals(LicenseTypeOptions.FREE)) { - geoData = new GetGeoLite2Data(finalPath.concat(flipDatabase), maxMindConfig.getCacheSize()); - } - else if (licenseType.equals(LicenseTypeOptions.ENTERPRISE)) { - geoData = new GetGeoIP2Data(finalPath.concat(flipDatabase), maxMindConfig.getCacheSize()); + if (downloadReady) { + geoData = geoDataFactory.create(flipDatabase); } } downloadReady = false; @@ -120,14 +114,19 @@ public synchronized void downloadThroughURLandS3() { downloadReady = true; break; case PATH: - dbSource = new LocalDBDownloadService(tempPath, flipDatabase); + dbSource = new LocalDBDownloadService(flipDatabase); dbSource.initiateDownload(databasePaths); downloadReady = true; break; } - } catch (Exception ex) { - throw new DownloadFailedException("Download failed: " + ex); + } catch (final Exception ex) { + if (isDuringInitialization) { + throw new DownloadFailedException("Download failed due to: " + ex); + } else { + LOG.error("Download failed due to: {0}. Using previously loaded database files.", ex); + } } + isDuringInitialization = false; notifyAll(); } @@ -137,7 +136,7 @@ public synchronized void downloadThroughURLandS3() { * @param attributes attributes * @return Enriched Map */ - public Map getGeoData(InetAddress inetAddress, List attributes) { + public Map getGeoData(final InetAddress inetAddress, final List attributes) { return geoData.getGeoData(inetAddress, attributes, tempPath + File.separator + flipDatabase); } } diff --git a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/GeoIpConfigSupplier.java b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/GeoIpConfigSupplier.java index c7adb00c2f..a754c9a745 100644 --- a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/GeoIpConfigSupplier.java +++ b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/GeoIpConfigSupplier.java @@ -5,8 +5,6 @@ package org.opensearch.dataprepper.plugins.processor.extension; -import org.opensearch.dataprepper.plugins.processor.GeoIPProcessorService; - /** * Interface for supplying {@link GeoIPProcessorService} to {@link GeoIpConfigExtension} * diff --git a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/databasedownload/DBSource.java b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/DBSource.java similarity index 91% rename from data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/databasedownload/DBSource.java rename to data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/DBSource.java index fec988c92a..3ead66744c 100644 --- a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/databasedownload/DBSource.java +++ b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/DBSource.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.dataprepper.plugins.processor.databasedownload; +package org.opensearch.dataprepper.plugins.processor.extension.databasedownload; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -36,7 +36,7 @@ public interface DBSource { * @param outputFilePath Output File Path * @return File */ - public static File createFolderIfNotExist(String outputFilePath) { + static File createFolderIfNotExist(String outputFilePath) { final File destFile = new File(outputFilePath); try { if (!destFile.exists()) { @@ -53,7 +53,7 @@ public static File createFolderIfNotExist(String outputFilePath) { * Delete Directory * @param file file */ - public static void deleteDirectory(File file) { + static void deleteDirectory(File file) { if (file.exists()) { for (final File subFile : file.listFiles()) { @@ -71,7 +71,7 @@ public static void deleteDirectory(File file) { * @throws NoSuchAlgorithmException NoSuchAlgorithmException * @throws KeyManagementException KeyManagementException */ - public default void initiateSSL() throws NoSuchAlgorithmException, KeyManagementException { + default void initiateSSL() throws NoSuchAlgorithmException, KeyManagementException { final TrustManager[] trustAllCerts = new TrustManager[]{ new X509TrustManager() { public X509Certificate[] getAcceptedIssuers() { diff --git a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/databasedownload/DBSourceOptions.java b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/DBSourceOptions.java similarity index 91% rename from data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/databasedownload/DBSourceOptions.java rename to data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/DBSourceOptions.java index 76b5d2f218..485274dd9d 100644 --- a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/databasedownload/DBSourceOptions.java +++ b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/DBSourceOptions.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.dataprepper.plugins.processor.databasedownload; +package org.opensearch.dataprepper.plugins.processor.extension.databasedownload; import com.fasterxml.jackson.annotation.JsonCreator; diff --git a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/databasedownload/DatabaseReaderCreate.java b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/DatabaseReaderCreate.java similarity index 67% rename from data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/databasedownload/DatabaseReaderCreate.java rename to data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/DatabaseReaderCreate.java index 85b0f560f4..47546c6c9f 100644 --- a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/databasedownload/DatabaseReaderCreate.java +++ b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/DatabaseReaderCreate.java @@ -3,12 +3,13 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.dataprepper.plugins.processor.databasedownload; +package org.opensearch.dataprepper.plugins.processor.extension.databasedownload; import com.maxmind.db.CHMCache; import com.maxmind.db.Reader; import com.maxmind.geoip2.DatabaseReader; +import java.io.IOException; import java.nio.file.Path; /** @@ -22,10 +23,10 @@ public class DatabaseReaderCreate { * @param cacheSize cacheSize * @return DatabaseReader */ - public static DatabaseReader.Builder createLoader(final Path databasePath, final int cacheSize) { - + public static DatabaseReader createLoader(final Path databasePath, final int cacheSize) throws IOException { return new DatabaseReader.Builder(databasePath.toFile()) .fileMode(Reader.FileMode.MEMORY_MAPPED) - .withCache(new CHMCache(cacheSize)); + .withCache(new CHMCache(cacheSize)) + .build(); } } diff --git a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/GeoDataFactory.java b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/GeoDataFactory.java new file mode 100644 index 0000000000..e05453d37f --- /dev/null +++ b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/GeoDataFactory.java @@ -0,0 +1,39 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.processor.extension.databasedownload; + +import org.opensearch.dataprepper.plugins.processor.databaseenrich.GetGeoData; +import org.opensearch.dataprepper.plugins.processor.databaseenrich.GetGeoIP2Data; +import org.opensearch.dataprepper.plugins.processor.databaseenrich.GetGeoLite2Data; +import org.opensearch.dataprepper.plugins.processor.extension.MaxMindConfig; +import org.opensearch.dataprepper.plugins.processor.utils.LicenseTypeCheck; + +import java.io.File; + +import static org.opensearch.dataprepper.plugins.processor.extension.databasedownload.DBSource.tempFolderPath; + +public class GeoDataFactory { + private final MaxMindConfig maxMindConfig; + private final LicenseTypeCheck licenseTypeCheck; + + public GeoDataFactory(final MaxMindConfig maxMindConfig, final LicenseTypeCheck licenseTypeCheck) { + this.maxMindConfig = maxMindConfig; + this.licenseTypeCheck = licenseTypeCheck; + } + + /** + * Creates GetGeoData class based on LicenseTypeOptions + */ + public GetGeoData create(final String databasePath) { + final String finalPath = tempFolderPath + File.separator + databasePath; + final LicenseTypeOptions licenseType = licenseTypeCheck.isGeoLite2OrEnterpriseLicense(finalPath); + if (licenseType.equals(LicenseTypeOptions.FREE)) { + return new GetGeoLite2Data(finalPath, maxMindConfig.getCacheSize()); + } else { + return new GetGeoIP2Data(finalPath, maxMindConfig.getCacheSize()); + } + } +} diff --git a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/databasedownload/HttpDBDownloadService.java b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/HttpDBDownloadService.java similarity index 98% rename from data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/databasedownload/HttpDBDownloadService.java rename to data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/HttpDBDownloadService.java index d06e911192..d407f9bdb7 100644 --- a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/databasedownload/HttpDBDownloadService.java +++ b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/HttpDBDownloadService.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.dataprepper.plugins.processor.databasedownload; +package org.opensearch.dataprepper.plugins.processor.extension.databasedownload; import org.apache.commons.compress.archivers.tar.TarArchiveEntry; import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; diff --git a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/databasedownload/LicenseTypeOptions.java b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/LicenseTypeOptions.java similarity index 91% rename from data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/databasedownload/LicenseTypeOptions.java rename to data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/LicenseTypeOptions.java index 003135bf94..59f83c37ab 100644 --- a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/databasedownload/LicenseTypeOptions.java +++ b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/LicenseTypeOptions.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.dataprepper.plugins.processor.databasedownload; +package org.opensearch.dataprepper.plugins.processor.extension.databasedownload; import com.fasterxml.jackson.annotation.JsonCreator; diff --git a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/databasedownload/LocalDBDownloadService.java b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/LocalDBDownloadService.java similarity index 75% rename from data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/databasedownload/LocalDBDownloadService.java rename to data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/LocalDBDownloadService.java index d089febc06..d4a85218fa 100644 --- a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/databasedownload/LocalDBDownloadService.java +++ b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/LocalDBDownloadService.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.dataprepper.plugins.processor.databasedownload; +package org.opensearch.dataprepper.plugins.processor.extension.databasedownload; import org.apache.commons.io.FileUtils; @@ -15,16 +15,13 @@ */ public class LocalDBDownloadService implements DBSource { - private String tempPath; private final String prefixDir; /** * LocalDBDownloadService constructor for initialisation of attributes - * @param tempPath tempPath * @param prefixDir prefixDir */ - public LocalDBDownloadService(String tempPath, String prefixDir) { - this.tempPath = tempPath; + public LocalDBDownloadService(final String prefixDir) { this.prefixDir = prefixDir; } @@ -34,7 +31,7 @@ public LocalDBDownloadService(String tempPath, String prefixDir) { */ @Override public void initiateDownload(List config) throws Exception { - String destPath = tempPath + File.separator + prefixDir; + String destPath = tempFolderPath + File.separator + prefixDir; DBSource.createFolderIfNotExist(destPath); File srcDatabaseConfigPath = new File(config.get(0)); File destDatabaseConfigPath = new File(destPath); diff --git a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/databasedownload/S3DBService.java b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/S3DBService.java similarity index 98% rename from data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/databasedownload/S3DBService.java rename to data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/S3DBService.java index a581e46d36..11793a8189 100644 --- a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/databasedownload/S3DBService.java +++ b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/S3DBService.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.dataprepper.plugins.processor.databasedownload; +package org.opensearch.dataprepper.plugins.processor.extension.databasedownload; import org.opensearch.dataprepper.plugins.processor.databaseenrich.DownloadFailedException; import org.opensearch.dataprepper.plugins.processor.extension.AwsAuthenticationOptionsConfig; diff --git a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/utils/DbSourceIdentification.java b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/utils/DbSourceIdentification.java index a7428858fb..e7eaffe75d 100644 --- a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/utils/DbSourceIdentification.java +++ b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/utils/DbSourceIdentification.java @@ -5,7 +5,7 @@ package org.opensearch.dataprepper.plugins.processor.utils; -import org.opensearch.dataprepper.plugins.processor.databasedownload.DBSourceOptions; +import org.opensearch.dataprepper.plugins.processor.extension.databasedownload.DBSourceOptions; import java.net.MalformedURLException; import java.net.URI; diff --git a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/utils/IPValidationcheck.java b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/utils/IPValidationCheck.java similarity index 75% rename from data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/utils/IPValidationcheck.java rename to data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/utils/IPValidationCheck.java index c1d59cc92e..d717ebc711 100644 --- a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/utils/IPValidationcheck.java +++ b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/utils/IPValidationCheck.java @@ -14,17 +14,16 @@ * Implementation of class for checking IP validation * IP should be public it can be either IPV4 or IPV6 */ -public class IPValidationcheck { +public class IPValidationCheck { /** * Check for IP is valid or not - * @param input input + * @param ipAddress ipAddress * @return boolean * @throws UnknownHostException UnknownHostException */ - public static boolean isPublicIpAddress(String input) throws UnknownHostException { - String ipAddress = input; - InetAddress address = InetAddress.getByName(input); + public static boolean isPublicIpAddress(final String ipAddress) throws UnknownHostException { + InetAddress address = InetAddress.getByName(ipAddress); if (address instanceof Inet6Address || address instanceof Inet4Address) { return !address.isSiteLocalAddress() && !address.isLoopbackAddress(); } diff --git a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/utils/LicenseTypeCheck.java b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/utils/LicenseTypeCheck.java index 7922a2b0fd..1e75d4c4ca 100644 --- a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/utils/LicenseTypeCheck.java +++ b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/utils/LicenseTypeCheck.java @@ -5,7 +5,7 @@ package org.opensearch.dataprepper.plugins.processor.utils; -import org.opensearch.dataprepper.plugins.processor.databasedownload.LicenseTypeOptions; +import org.opensearch.dataprepper.plugins.processor.extension.databasedownload.LicenseTypeOptions; import java.io.File; import java.nio.file.Path; @@ -25,16 +25,17 @@ public class LicenseTypeCheck { private static final String geoIP2EnterpriseDB = "GeoIP2-Enterprise.mmdb"; - private LicenseTypeCheck() { + public LicenseTypeCheck() { } /** * Get the license type based on the maxmind mmdb file name + * * @param databasePath databasePath * @return license type options */ - public static LicenseTypeOptions isGeoLite2OrEnterpriseLicense(String databasePath) { + public LicenseTypeOptions isGeoLite2OrEnterpriseLicense(final String databasePath) { LicenseTypeOptions licenseTypeOptions = LicenseTypeOptions.ENTERPRISE; File directory = new File(databasePath); // list all files present in the directory diff --git a/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/GeoIPProcessorConfigTest.java b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/GeoIPProcessorConfigTest.java index 33b17795e2..1709e39234 100644 --- a/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/GeoIPProcessorConfigTest.java +++ b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/GeoIPProcessorConfigTest.java @@ -12,7 +12,6 @@ import org.opensearch.dataprepper.plugins.processor.configuration.EntryConfig; import org.opensearch.dataprepper.test.helper.ReflectivelySetField; -import java.util.ArrayList; import java.util.List; import static org.hamcrest.CoreMatchers.equalTo; @@ -32,14 +31,21 @@ void setUp() { void testDefaultConfig() { assertThat(geoIPProcessorConfig.getEntries(), equalTo(null)); assertThat(geoIPProcessorConfig.getTagsOnFailure(), equalTo(null)); + assertThat(geoIPProcessorConfig.getWhenCondition(), equalTo(null)); } @Test void testGetEntries() throws NoSuchFieldException, IllegalAccessException { - List entries = new ArrayList<>(); - entries.add(new EntryConfig()); + final List entries = List.of(new EntryConfig()); + final List tagsOnFailure = List.of("tag1", "tag2"); + final String whenCondition = "/ip == 1.2.3.4"; + ReflectivelySetField.setField(GeoIPProcessorConfig.class, geoIPProcessorConfig, "entries", entries); + ReflectivelySetField.setField(GeoIPProcessorConfig.class, geoIPProcessorConfig, "tagsOnFailure", tagsOnFailure); + ReflectivelySetField.setField(GeoIPProcessorConfig.class, geoIPProcessorConfig, "whenCondition", whenCondition); assertThat(geoIPProcessorConfig.getEntries(), equalTo(entries)); + assertThat(geoIPProcessorConfig.getTagsOnFailure(), equalTo(tagsOnFailure)); + assertThat(geoIPProcessorConfig.getWhenCondition(), equalTo(whenCondition)); } } diff --git a/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/GeoIPProcessorTest.java b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/GeoIPProcessorTest.java index 775c4270fe..d6b8a7bf01 100644 --- a/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/GeoIPProcessorTest.java +++ b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/GeoIPProcessorTest.java @@ -5,18 +5,22 @@ package org.opensearch.dataprepper.plugins.processor; +import io.micrometer.core.instrument.Counter; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import org.opensearch.dataprepper.model.configuration.PluginSetting; +import org.opensearch.dataprepper.expression.ExpressionEvaluator; +import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.event.Event; import org.opensearch.dataprepper.model.event.JacksonEvent; import org.opensearch.dataprepper.model.log.JacksonLog; import org.opensearch.dataprepper.model.record.Record; import org.opensearch.dataprepper.plugins.processor.configuration.EntryConfig; import org.opensearch.dataprepper.plugins.processor.databaseenrich.EnrichFailedException; +import org.opensearch.dataprepper.plugins.processor.extension.GeoIPProcessorService; import org.opensearch.dataprepper.plugins.processor.extension.GeoIpConfigSupplier; import org.opensearch.dataprepper.test.helper.ReflectivelySetField; @@ -26,6 +30,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.MatcherAssert.assertThat; @@ -33,16 +38,17 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; +import static org.opensearch.dataprepper.plugins.processor.GeoIPProcessor.GEO_IP_PROCESSING_MATCH; +import static org.opensearch.dataprepper.plugins.processor.GeoIPProcessor.GEO_IP_PROCESSING_MISMATCH; @ExtendWith(MockitoExtension.class) class GeoIPProcessorTest { - - public static final int REFRESH_SCHEDULE = 10; public static final String SOURCE = "/peer/ip"; - public static final String TARGET = "location"; - public static final String PROCESSOR_PLUGIN_NAME = "geoip"; - public static final String PROCESSOR_PIPELINE_NAME = "geoIP-processor-pipeline"; + public static final String TARGET = "geolocation"; @Mock private GeoIPProcessorService geoIPProcessorService; @Mock @@ -50,17 +56,72 @@ class GeoIPProcessorTest { @Mock private GeoIpConfigSupplier geoIpConfigSupplier; @Mock - private PluginSetting pluginSetting; - @Mock private EntryConfig entry; + @Mock + private ExpressionEvaluator expressionEvaluator; + @Mock + private PluginMetrics pluginMetrics; + @Mock + private Counter geoIpProcessingMatch; + @Mock + private Counter geoIpProcessingMismatch; @BeforeEach - void setUp() throws NoSuchFieldException, IllegalAccessException { + void setUp() { + when(geoIpConfigSupplier.getGeoIPProcessorService()).thenReturn(geoIPProcessorService); + lenient().when(pluginMetrics.counter(GEO_IP_PROCESSING_MATCH)).thenReturn(geoIpProcessingMatch); + lenient().when(pluginMetrics.counter(GEO_IP_PROCESSING_MISMATCH)).thenReturn(geoIpProcessingMismatch); + } - when(pluginSetting.getName()).thenReturn(PROCESSOR_PLUGIN_NAME); - when(pluginSetting.getPipelineName()).thenReturn(PROCESSOR_PIPELINE_NAME); + @AfterEach + void tearDown() { + verifyNoMoreInteractions(geoIpProcessingMatch); + verifyNoMoreInteractions(geoIpProcessingMismatch); + } - when(geoIpConfigSupplier.getGeoIPProcessorService()).thenReturn(geoIPProcessorService); + private GeoIPProcessor createObjectUnderTest() { + return new GeoIPProcessor(pluginMetrics, geoIPProcessorConfig, geoIpConfigSupplier, expressionEvaluator); + } + + @Test + void doExecuteTest_with_when_condition_should_only_enrich_events_that_match_when_condition() throws NoSuchFieldException, IllegalAccessException { + final String whenCondition = "/peer/status == success"; + + when(geoIPProcessorConfig.getEntries()).thenReturn(List.of(entry)); + when(geoIPProcessorConfig.getWhenCondition()).thenReturn(whenCondition); + when(entry.getSource()).thenReturn("/peer/ip"); + when(entry.getTarget()).thenReturn(TARGET); + when(entry.getFields()).thenReturn(setFields()); + + final GeoIPProcessor geoIPProcessor = createObjectUnderTest(); + + when(geoIPProcessorService.getGeoData(any(), any())).thenReturn(prepareGeoData()); + + ReflectivelySetField.setField(GeoIPProcessor.class, geoIPProcessor, "geoIPProcessorService", geoIPProcessorService); + + final Record record1 = createCustomRecord("success"); + final Record record2 = createCustomRecord("failed"); + List> recordsIn = new ArrayList<>(); + recordsIn.add(record1); + recordsIn.add(record2); + + when(expressionEvaluator.evaluateConditional(whenCondition, record1.getData())).thenReturn(true); + when(expressionEvaluator.evaluateConditional(whenCondition, record2.getData())).thenReturn(false); + + final Collection> records = geoIPProcessor.doExecute(recordsIn); + + assertThat(records.size(), equalTo(2)); + + final Collection> recordsWithLocation = records.stream().filter(record -> record.getData().containsKey(TARGET)) + .collect(Collectors.toList()); + + assertThat(recordsWithLocation.size(), equalTo(1)); + + for (final Record record : recordsWithLocation) { + final Event event = record.getData(); + assertThat(event.get("/peer/status", String.class), equalTo("success")); + } + verify(geoIpProcessingMatch).increment(); } @Test @@ -70,19 +131,20 @@ void doExecuteTest() throws NoSuchFieldException, IllegalAccessException { when(entry.getTarget()).thenReturn(TARGET); when(entry.getFields()).thenReturn(setFields()); - GeoIPProcessor geoIPProcessor = createObjectUnderTest(); + final GeoIPProcessor geoIPProcessor = createObjectUnderTest(); when(geoIPProcessorService.getGeoData(any(), any())).thenReturn(prepareGeoData()); ReflectivelySetField.setField(GeoIPProcessor.class, geoIPProcessor, "geoIPProcessorService", geoIPProcessorService); Collection> records = geoIPProcessor.doExecute(setEventQueue()); for (final Record record : records) { - Event event = record.getData(); + final Event event = record.getData(); assertThat(event.get("/peer/ip", String.class), equalTo("136.226.242.205")); + assertThat(event.containsKey("geolocation"), equalTo(true)); + verify(geoIpProcessingMatch).increment(); } } - @Test void test_tags_when_enrich_fails() { when(entry.getSource()).thenReturn(SOURCE); @@ -103,6 +165,7 @@ void test_tags_when_enrich_fails() { for (final Record record : records) { Event event = record.getData(); assertTrue(event.getMetadata().hasTags(testTags)); + verify(geoIpProcessingMismatch).increment(); } } @@ -127,10 +190,6 @@ private Map prepareGeoData() { return geoDataMap; } - private GeoIPProcessor createObjectUnderTest() { - return new GeoIPProcessor(pluginSetting, geoIPProcessorConfig, geoIpConfigSupplier); - } - private List setFields() { final List attributes = new ArrayList<>(); attributes.add("city_name"); @@ -150,4 +209,21 @@ private static Record createRecord() { final JacksonEvent event = JacksonLog.builder().withData(json).build(); return new Record<>(event); } + + private Record createCustomRecord(final String customFieldValue) { + Map innerMap = new HashMap<>(); + innerMap.put("ip", "136.226.242.205"); + innerMap.put("host", "example.org"); + innerMap.put("status", customFieldValue); + + final Map eventMap1 = new HashMap<>(); + eventMap1.put("peer", innerMap); + + final Event firstEvent = JacksonEvent.builder() + .withData(eventMap1) + .withEventType("event") + .build(); + + return new Record<>(firstEvent); + } } diff --git a/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/configuration/EntryTest.java b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/configuration/EntryConfigTest.java similarity index 85% rename from data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/configuration/EntryTest.java rename to data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/configuration/EntryConfigTest.java index 20e99b0d28..3863663c7a 100644 --- a/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/configuration/EntryTest.java +++ b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/configuration/EntryConfigTest.java @@ -16,8 +16,9 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.nullValue; +import static org.opensearch.dataprepper.plugins.processor.configuration.EntryConfig.DEFAULT_TARGET; -class EntryTest { +class EntryConfigTest { public static final String SOURCE_VALUE = "source"; public static final String TARGET_VALUE = "target"; public static final List FIELDS_VALUE = List.of("city", "country"); @@ -31,7 +32,8 @@ void setUp() { @Test void testDefaultConfig() { assertThat(entryConfig.getSource(), is(nullValue())); - assertThat(entryConfig.getTarget(), is(nullValue())); + assertThat(entryConfig.getTarget(), equalTo(DEFAULT_TARGET)); + assertThat(entryConfig.getFields(), is(nullValue())); } @Test @@ -44,6 +46,4 @@ void testCustomConfig() throws NoSuchFieldException, IllegalAccessException { assertThat(entryConfig.getTarget(), equalTo(TARGET_VALUE)); assertThat(entryConfig.getFields(), equalTo(FIELDS_VALUE)); } - - -} \ No newline at end of file +} diff --git a/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/databasedownload/DatabaseReaderCreateTest.java b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/databasedownload/DatabaseReaderCreateTest.java deleted file mode 100644 index c216720c57..0000000000 --- a/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/databasedownload/DatabaseReaderCreateTest.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.dataprepper.plugins.processor.databasedownload; - -import com.maxmind.geoip2.DatabaseReader; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.junit.jupiter.MockitoExtension; - -import java.nio.file.Path; - -@ExtendWith(MockitoExtension.class) -class DatabaseReaderCreateTest { - - private final Path databasePath = Path.of("/tmp/"); - - @Test - void createLoaderTest_with_cache() { - DatabaseReader.Builder builder = DatabaseReaderCreate.createLoader(databasePath, 4096); - Assertions.assertNotNull(builder); - } -} diff --git a/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/databaseenrich/GetGeoIP2DataTest.java b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/databaseenrich/GetGeoIP2DataTest.java index 2f24976eb1..5493082d08 100644 --- a/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/databaseenrich/GetGeoIP2DataTest.java +++ b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/databaseenrich/GetGeoIP2DataTest.java @@ -12,8 +12,8 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.dataprepper.plugins.processor.GeoIPProcessorConfig; -import org.opensearch.dataprepper.plugins.processor.GeoIPProcessorService; -import org.opensearch.dataprepper.plugins.processor.databasedownload.DBSource; +import org.opensearch.dataprepper.plugins.processor.extension.GeoIPProcessorService; +import org.opensearch.dataprepper.plugins.processor.extension.databasedownload.DBSource; import org.opensearch.dataprepper.plugins.processor.extension.GeoIpServiceConfig; import org.opensearch.dataprepper.plugins.processor.extension.MaxMindConfig; diff --git a/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/databaseenrich/GetGeoLite2DataTest.java b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/databaseenrich/GetGeoLite2DataTest.java index ef2291b0e0..7a58317a5d 100644 --- a/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/databaseenrich/GetGeoLite2DataTest.java +++ b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/databaseenrich/GetGeoLite2DataTest.java @@ -12,8 +12,8 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import org.opensearch.dataprepper.plugins.processor.GeoIPProcessorService; -import org.opensearch.dataprepper.plugins.processor.databasedownload.DBSource; +import org.opensearch.dataprepper.plugins.processor.extension.GeoIPProcessorService; +import org.opensearch.dataprepper.plugins.processor.extension.databasedownload.DBSource; import org.opensearch.dataprepper.plugins.processor.extension.GeoIpServiceConfig; import org.opensearch.dataprepper.plugins.processor.extension.MaxMindConfig; diff --git a/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/extension/DefaultGeoIpConfigSupplierTest.java b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/extension/DefaultGeoIpConfigSupplierTest.java index af0920817d..9d42f5b6f2 100644 --- a/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/extension/DefaultGeoIpConfigSupplierTest.java +++ b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/extension/DefaultGeoIpConfigSupplierTest.java @@ -10,7 +10,6 @@ import org.mockito.Mock; import org.mockito.MockedConstruction; import org.mockito.junit.jupiter.MockitoExtension; -import org.opensearch.dataprepper.plugins.processor.GeoIPProcessorService; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.instanceOf; diff --git a/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/GeoIPProcessorServiceTest.java b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/extension/GeoIPProcessorServiceTest.java similarity index 96% rename from data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/GeoIPProcessorServiceTest.java rename to data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/extension/GeoIPProcessorServiceTest.java index 5ce41a775b..232f98edec 100644 --- a/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/GeoIPProcessorServiceTest.java +++ b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/extension/GeoIPProcessorServiceTest.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.dataprepper.plugins.processor; +package org.opensearch.dataprepper.plugins.processor.extension; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; @@ -11,8 +11,8 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.plugins.processor.GeoIPProcessorConfig; import org.opensearch.dataprepper.plugins.processor.databaseenrich.GetGeoData; -import org.opensearch.dataprepper.plugins.processor.extension.MaxMindConfig; import org.opensearch.dataprepper.test.helper.ReflectivelySetField; import java.io.File; diff --git a/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/databasedownload/DBSourceOptionsTest.java b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/DBSourceOptionsTest.java similarity index 90% rename from data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/databasedownload/DBSourceOptionsTest.java rename to data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/DBSourceOptionsTest.java index c6c634a22b..531efca0a9 100644 --- a/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/databasedownload/DBSourceOptionsTest.java +++ b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/DBSourceOptionsTest.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.dataprepper.plugins.processor.databasedownload; +package org.opensearch.dataprepper.plugins.processor.extension.databasedownload; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; diff --git a/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/databasedownload/DBSourceTest.java b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/DBSourceTest.java similarity index 94% rename from data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/databasedownload/DBSourceTest.java rename to data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/DBSourceTest.java index 401a0f700b..3b7ed49d87 100644 --- a/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/databasedownload/DBSourceTest.java +++ b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/DBSourceTest.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.dataprepper.plugins.processor.databasedownload; +package org.opensearch.dataprepper.plugins.processor.extension.databasedownload; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; diff --git a/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/DatabaseReaderCreateTest.java b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/DatabaseReaderCreateTest.java new file mode 100644 index 0000000000..621498e8e2 --- /dev/null +++ b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/DatabaseReaderCreateTest.java @@ -0,0 +1,43 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.processor.extension.databasedownload; + +import com.maxmind.geoip2.DatabaseReader; +import org.apache.commons.io.FileUtils; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.io.BufferedInputStream; +import java.io.File; +import java.io.IOException; +import java.net.URL; +import java.nio.file.Path; + +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class DatabaseReaderCreateTest { + @Mock + private Path path; + + @Test + void createLoaderTest() throws IOException { + final String testFileURL = "https://github.com/maxmind/MaxMind-DB/raw/main/test-data/GeoLite2-City-Test.mmdb"; + final File file = File.createTempFile( "GeoIP2-City-Test", ".mmdb"); + + final BufferedInputStream in = new BufferedInputStream(new URL(testFileURL).openStream()); + FileUtils.copyInputStreamToFile(in, file); + when(path.toFile()).thenReturn(file); + + DatabaseReader databaseReader = DatabaseReaderCreate.createLoader(path, 4096); + Assertions.assertNotNull(databaseReader); + in.close(); + file.deleteOnExit(); + } +} diff --git a/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/GeoDataFactoryTest.java b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/GeoDataFactoryTest.java new file mode 100644 index 0000000000..83be77f774 --- /dev/null +++ b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/GeoDataFactoryTest.java @@ -0,0 +1,51 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.processor.extension.databasedownload; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.plugins.processor.databaseenrich.GetGeoData; +import org.opensearch.dataprepper.plugins.processor.databaseenrich.GetGeoIP2Data; +import org.opensearch.dataprepper.plugins.processor.databaseenrich.GetGeoLite2Data; +import org.opensearch.dataprepper.plugins.processor.extension.MaxMindConfig; +import org.opensearch.dataprepper.plugins.processor.utils.LicenseTypeCheck; + +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class GeoDataFactoryTest { + + @Mock + private MaxMindConfig maxMindConfig; + + @Mock + private LicenseTypeCheck licenseTypeCheck; + + @Test + void testCreateWithFreeLicense() { + when(licenseTypeCheck.isGeoLite2OrEnterpriseLicense(anyString())).thenReturn(LicenseTypeOptions.FREE); + final GeoDataFactory geoDataFactory = new GeoDataFactory(maxMindConfig, licenseTypeCheck); + final String databasePath = "testPath"; + + final GetGeoData getGeoData = geoDataFactory.create(databasePath); + assertInstanceOf(GetGeoLite2Data.class, getGeoData); + } + + @Test + void testCreateWithEnterpriseLicense() { + when(licenseTypeCheck.isGeoLite2OrEnterpriseLicense(anyString())).thenReturn(LicenseTypeOptions.ENTERPRISE); + final GeoDataFactory geoDataFactory = new GeoDataFactory(maxMindConfig, licenseTypeCheck); + final String databasePath = "testPath"; + + final GetGeoData getGeoData = geoDataFactory.create(databasePath); + assertInstanceOf(GetGeoIP2Data.class, getGeoData); + } + +} \ No newline at end of file diff --git a/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/databasedownload/HttpDBDownloadServiceTest.java b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/HttpDBDownloadServiceTest.java similarity index 92% rename from data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/databasedownload/HttpDBDownloadServiceTest.java rename to data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/HttpDBDownloadServiceTest.java index 47971b13f9..8b2bbf47cd 100644 --- a/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/databasedownload/HttpDBDownloadServiceTest.java +++ b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/HttpDBDownloadServiceTest.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.dataprepper.plugins.processor.databasedownload; +package org.opensearch.dataprepper.plugins.processor.extension.databasedownload; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; diff --git a/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/databasedownload/LicenseTypeOptionsTest.java b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/LicenseTypeOptionsTest.java similarity index 91% rename from data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/databasedownload/LicenseTypeOptionsTest.java rename to data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/LicenseTypeOptionsTest.java index 0b40ad601f..6a47db5143 100644 --- a/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/databasedownload/LicenseTypeOptionsTest.java +++ b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/LicenseTypeOptionsTest.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.dataprepper.plugins.processor.databasedownload; +package org.opensearch.dataprepper.plugins.processor.extension.databasedownload; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; diff --git a/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/databasedownload/LocalDBDownloadServiceTest.java b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/LocalDBDownloadServiceTest.java similarity index 92% rename from data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/databasedownload/LocalDBDownloadServiceTest.java rename to data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/LocalDBDownloadServiceTest.java index 2af7846875..eaee24774c 100644 --- a/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/databasedownload/LocalDBDownloadServiceTest.java +++ b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/LocalDBDownloadServiceTest.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.dataprepper.plugins.processor.databasedownload; +package org.opensearch.dataprepper.plugins.processor.extension.databasedownload; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -36,7 +36,7 @@ void initiateDownloadTest() throws IOException { } private LocalDBDownloadService createObjectUnderTest() { - return new LocalDBDownloadService(tempFolderPath, PREFIX_DIR); + return new LocalDBDownloadService(PREFIX_DIR); } private static void createFolder(String folderName) { diff --git a/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/databasedownload/S3DBServiceTest.java b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/S3DBServiceTest.java similarity index 93% rename from data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/databasedownload/S3DBServiceTest.java rename to data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/S3DBServiceTest.java index ed49f657db..815d2eb624 100644 --- a/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/databasedownload/S3DBServiceTest.java +++ b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/S3DBServiceTest.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.dataprepper.plugins.processor.databasedownload; +package org.opensearch.dataprepper.plugins.processor.extension.databasedownload; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; diff --git a/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/utils/DbSourceIdentificationTest.java b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/utils/DbSourceIdentificationTest.java index 2e1bd855cb..7be0b00db9 100644 --- a/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/utils/DbSourceIdentificationTest.java +++ b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/utils/DbSourceIdentificationTest.java @@ -7,7 +7,7 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import org.opensearch.dataprepper.plugins.processor.databasedownload.DBSourceOptions; +import org.opensearch.dataprepper.plugins.processor.extension.databasedownload.DBSourceOptions; import java.util.List; diff --git a/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/utils/IPValidationcheckTest.java b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/utils/IPValidationCheckTest.java similarity index 81% rename from data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/utils/IPValidationcheckTest.java rename to data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/utils/IPValidationCheckTest.java index 9ebbbcbe83..487f131f81 100644 --- a/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/utils/IPValidationcheckTest.java +++ b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/utils/IPValidationCheckTest.java @@ -13,7 +13,7 @@ import java.net.UnknownHostException; @ExtendWith(MockitoExtension.class) -class IPValidationcheckTest { +class IPValidationCheckTest { private static final String PRIVATE_IP_ADDRESS = "192.168.29.233"; private static final String PUBLIC_IP_ADDRESS = "2001:4860:4860::8888"; @@ -21,16 +21,16 @@ class IPValidationcheckTest { @Test void ipValidationcheckTest_positive() throws UnknownHostException { - Assertions.assertTrue(IPValidationcheck.isPublicIpAddress(PUBLIC_IP_ADDRESS)); + Assertions.assertTrue(IPValidationCheck.isPublicIpAddress(PUBLIC_IP_ADDRESS)); } @Test void ipValidationcheckTest_negative() throws UnknownHostException { - Assertions.assertFalse(IPValidationcheck.isPublicIpAddress(PRIVATE_IP_ADDRESS)); + Assertions.assertFalse(IPValidationCheck.isPublicIpAddress(PRIVATE_IP_ADDRESS)); } @Test void ipValidationcheckTest_invalid() throws UnknownHostException { - Assertions.assertTrue(IPValidationcheck.isPublicIpAddress(INVALID_IP_ADDRESS)); + Assertions.assertTrue(IPValidationCheck.isPublicIpAddress(INVALID_IP_ADDRESS)); } } \ No newline at end of file diff --git a/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/utils/LicenseTypeCheckTest.java b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/utils/LicenseTypeCheckTest.java index ad1ca4f631..02d40cdf1e 100644 --- a/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/utils/LicenseTypeCheckTest.java +++ b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/utils/LicenseTypeCheckTest.java @@ -8,7 +8,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.junit.jupiter.MockitoExtension; -import org.opensearch.dataprepper.plugins.processor.databasedownload.LicenseTypeOptions; +import org.opensearch.dataprepper.plugins.processor.extension.databasedownload.LicenseTypeOptions; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.MatcherAssert.assertThat; @@ -20,21 +20,27 @@ class LicenseTypeCheckTest { private static final String FOLDER_PATH_GEO_LITE2 = "./src/test/resources/mmdb-file/geo-lite2"; private static final String FOLDER_PATH_GEO_ENTERPRISE = "./src/test/resources/mmdb-file/geo-enterprise"; + private LicenseTypeCheck createObjectUnderTest() { + return new LicenseTypeCheck(); + } + @Test void isGeoLite2OrEnterpriseLicenseTest_positive() { - LicenseTypeOptions licenseTypeOptionsFree = LicenseTypeCheck.isGeoLite2OrEnterpriseLicense(FOLDER_PATH_GEO_LITE2); + final LicenseTypeCheck objectUnderTest = createObjectUnderTest(); + LicenseTypeOptions licenseTypeOptionsFree = objectUnderTest.isGeoLite2OrEnterpriseLicense(FOLDER_PATH_GEO_LITE2); assertThat(licenseTypeOptionsFree, equalTo(LicenseTypeOptions.FREE)); - LicenseTypeOptions licenseTypeOptionsEnterprise = LicenseTypeCheck.isGeoLite2OrEnterpriseLicense(FOLDER_PATH_GEO_ENTERPRISE); + LicenseTypeOptions licenseTypeOptionsEnterprise = objectUnderTest.isGeoLite2OrEnterpriseLicense(FOLDER_PATH_GEO_ENTERPRISE); assertThat(licenseTypeOptionsEnterprise, equalTo(LicenseTypeOptions.ENTERPRISE)); } @Test void isGeoLite2OrEnterpriseLicenseTest_negative() { - LicenseTypeOptions licenseTypeOptionsFree = LicenseTypeCheck.isGeoLite2OrEnterpriseLicense(FOLDER_PATH_GEO_ENTERPRISE); + final LicenseTypeCheck objectUnderTest = createObjectUnderTest(); + LicenseTypeOptions licenseTypeOptionsFree = objectUnderTest.isGeoLite2OrEnterpriseLicense(FOLDER_PATH_GEO_ENTERPRISE); assertThat(licenseTypeOptionsFree, not(equalTo(LicenseTypeOptions.FREE))); - LicenseTypeOptions licenseTypeOptionsEnterprise = LicenseTypeCheck.isGeoLite2OrEnterpriseLicense(FOLDER_PATH_GEO_LITE2); + LicenseTypeOptions licenseTypeOptionsEnterprise = objectUnderTest.isGeoLite2OrEnterpriseLicense(FOLDER_PATH_GEO_LITE2); assertThat(licenseTypeOptionsEnterprise, not(equalTo(LicenseTypeOptions.ENTERPRISE))); } } \ No newline at end of file