From 310c0161d558ee34d8359b7f339122568403dcfb Mon Sep 17 00:00:00 2001 From: Roberto Benitez Date: Thu, 16 Dec 2021 16:11:09 -0600 Subject: [PATCH 01/28] Initial Commit. Added basic back-end implementation--sans IP validation --- graylog2-server/pom.xml | 5 + .../map/config/GeoIpResolverConfig.java | 47 +++++-- .../plugins/map/geoip/GeoAsnInformation.java | 34 +++++ .../plugins/map/geoip/GeoIpResolver.java | 71 ++++++++++ .../map/geoip/GeoIpResolverEngine.java | 132 ++++++++---------- .../map/geoip/GeoIpResolverFactory.java | 78 +++++++++++ .../map/geoip/GeoLocationInformation.java | 41 ++++++ .../map/geoip/IpInfoIpAsnResolver.java | 63 +++++++++ .../map/geoip/IpInfoIpLocationResolver.java | 68 +++++++++ .../map/geoip/MaxMindIpAsnResolver.java | 65 +++++++++ .../map/geoip/MaxMindIpLocationResolver.java | 75 ++++++++++ .../map/geoip/GeoIpResolverEngineTest.java | 12 +- pom.xml | 1 + 13 files changed, 602 insertions(+), 90 deletions(-) create mode 100644 graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoAsnInformation.java create mode 100644 graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpResolver.java create mode 100644 graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpResolverFactory.java create mode 100644 graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoLocationInformation.java create mode 100644 graylog2-server/src/main/java/org/graylog/plugins/map/geoip/IpInfoIpAsnResolver.java create mode 100644 graylog2-server/src/main/java/org/graylog/plugins/map/geoip/IpInfoIpLocationResolver.java create mode 100644 graylog2-server/src/main/java/org/graylog/plugins/map/geoip/MaxMindIpAsnResolver.java create mode 100644 graylog2-server/src/main/java/org/graylog/plugins/map/geoip/MaxMindIpLocationResolver.java diff --git a/graylog2-server/pom.xml b/graylog2-server/pom.xml index 071b7e740b88..83d27b4885ec 100644 --- a/graylog2-server/pom.xml +++ b/graylog2-server/pom.xml @@ -564,6 +564,11 @@ com.maxmind.geoip2 geoip2 + + io.ipinfo + ipinfo-api + ${ipinfo.version} + org.graylog.cef cef-parser diff --git a/graylog2-server/src/main/java/org/graylog/plugins/map/config/GeoIpResolverConfig.java b/graylog2-server/src/main/java/org/graylog/plugins/map/config/GeoIpResolverConfig.java index 922fe6d6c1ea..4161a41b20a3 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/map/config/GeoIpResolverConfig.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/map/config/GeoIpResolverConfig.java @@ -16,12 +16,11 @@ */ package org.graylog.plugins.map.config; -import com.google.auto.value.AutoValue; - import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.auto.value.AutoValue; @JsonAutoDetect @JsonIgnoreProperties(ignoreUnknown = true) @@ -32,27 +31,39 @@ public abstract class GeoIpResolverConfig { public abstract boolean enabled(); @JsonProperty("db_type") - public abstract DatabaseType dbType(); + public abstract DatabaseType cityDbType(); @JsonProperty("db_path") - public abstract String dbPath(); + public abstract String cityDbPath(); + + @JsonProperty("asn_db_type") + public abstract DatabaseType asnDbType(); + + @JsonProperty("asn_db_path") + public abstract String asnDbPath(); @JsonCreator - public static GeoIpResolverConfig create(@JsonProperty("enabled") boolean enabled, + public static GeoIpResolverConfig create(@JsonProperty("enabled") boolean cityEnabled, @JsonProperty("db_type") DatabaseType dbType, - @JsonProperty("db_path") String dbPath) { + @JsonProperty("db_path") String cityDbPath, + @JsonProperty("asn_db_type") DatabaseType asnDbType, + @JsonProperty("asn_db_path") String asnDbPath) { return builder() - .enabled(enabled) - .dbType(dbType) - .dbPath(dbPath) + .enabled(cityEnabled) + .cityDbType(dbType) + .cityDbPath(cityDbPath) + .asnDbType(asnDbType) + .asnDbPath(asnDbPath) .build(); } public static GeoIpResolverConfig defaultConfig() { return builder() .enabled(false) - .dbType(DatabaseType.MAXMIND_CITY) - .dbPath("/etc/graylog/server/GeoLite2-City.mmdb") + .cityDbType(DatabaseType.MAXMIND_CITY) + .cityDbPath("/etc/graylog/server/GeoLite2-City.mmdb") + .asnDbType(DatabaseType.MAXMIND_ASN) + .asnDbPath("/etc/graylog/server/GeoLite2-ASN.mmdb") .build(); } @@ -63,11 +74,17 @@ public static Builder builder() { public abstract Builder toBuilder(); @AutoValue.Builder - public static abstract class Builder { + public abstract static class Builder { public abstract Builder enabled(boolean enabled); - public abstract Builder dbType(DatabaseType dbType); - public abstract Builder dbPath(String dbPath); + + public abstract Builder cityDbType(DatabaseType dbType); + + public abstract Builder cityDbPath(String dbPath); + + public abstract Builder asnDbType(DatabaseType dbType); + + public abstract Builder asnDbPath(String asnDBPath); public abstract GeoIpResolverConfig build(); } -} \ No newline at end of file +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoAsnInformation.java b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoAsnInformation.java new file mode 100644 index 000000000000..75949c5ee544 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoAsnInformation.java @@ -0,0 +1,34 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ + +package org.graylog.plugins.map.geoip; + +import com.google.auto.value.AutoValue; + +@AutoValue +public abstract class GeoAsnInformation { + + public abstract String organization(); + + public abstract String type(); + + public abstract String asn(); + + public static GeoAsnInformation create(String organization, String type, String asn) { + return new AutoValue_GeoAsnInformation(organization, type, asn); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpResolver.java b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpResolver.java new file mode 100644 index 000000000000..cbad11cd3663 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpResolver.java @@ -0,0 +1,71 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ + +package org.graylog.plugins.map.geoip; + +import com.codahale.metrics.Timer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.net.InetAddress; +import java.nio.file.Files; +import java.util.Optional; + +public abstract class GeoIpResolver { + + private static final Logger LOG = LoggerFactory.getLogger(GeoIpResolver.class); + + protected final Timer resolveTime; + private final boolean enabled; + protected final P dataProvider; + + GeoIpResolver(Timer resolveTime, String configPath, boolean enabled) { + + this.resolveTime = resolveTime; + if (enabled) { + final File configFile = new File(configPath); + if (Files.exists(configFile.toPath())) { + this.dataProvider = createDataProvider(configFile); + this.enabled = true; + } else { + LOG.warn("'{}' database file does not exist: {}", getClass().getName(), configPath); + this.enabled = false; + this.dataProvider = null; + } + } else { + this.enabled = false; + this.dataProvider = null; + } + } + + public boolean isEnabled() { + return enabled; + } + + abstract P createDataProvider(File configFile); + + public Optional getGeoIpData(InetAddress address) { + if (!enabled || dataProvider == null || address == null) { + return Optional.empty(); + } + + return doGetGeoIpData(address); + } + + protected abstract Optional doGetGeoIpData(InetAddress address); +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpResolverEngine.java b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpResolverEngine.java index 670a388e8ce1..3a00fac304d1 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpResolverEngine.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpResolverEngine.java @@ -18,53 +18,52 @@ import com.codahale.metrics.MetricRegistry; import com.codahale.metrics.Timer; -import com.google.auto.value.AutoValue; import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableMap; import com.google.common.net.InetAddresses; -import com.maxmind.geoip2.DatabaseReader; -import com.maxmind.geoip2.model.CityResponse; -import com.maxmind.geoip2.record.City; -import com.maxmind.geoip2.record.Country; -import com.maxmind.geoip2.record.Location; import org.graylog.plugins.map.config.GeoIpResolverConfig; import org.graylog2.plugin.Message; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.annotation.Nullable; -import java.io.File; -import java.io.IOException; import java.net.InetAddress; -import java.nio.file.Files; +import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.stream.Collectors; import static com.codahale.metrics.MetricRegistry.name; public class GeoIpResolverEngine { private static final Logger LOG = LoggerFactory.getLogger(GeoIpResolverEngine.class); + /** + * A mapping of fields that to search that contain IP addresses. ONLY these fields will be checked + * to see if they have valid Geo IP information. + */ + private final Map ipAddressFields = new ImmutableMap.Builder() + .put("source_ip", "source") + .put("host_ip", "host") + .put("destination_ip", "destination") + .build(); + private final Timer resolveTime; - private DatabaseReader databaseReader; + private final GeoIpResolver ipLocationResolver; + private final GeoIpResolver ipAsnResolver; private boolean enabled; public GeoIpResolverEngine(GeoIpResolverConfig config, MetricRegistry metricRegistry) { this.resolveTime = metricRegistry.timer(name(GeoIpResolverEngine.class, "resolveTime")); - try { - final File database = new File(config.dbPath()); - if (Files.exists(database.toPath())) { - this.databaseReader = new DatabaseReader.Builder(database).build(); - this.enabled = config.enabled(); - } else { - LOG.warn("GeoIP database file does not exist: {}", config.dbPath()); - this.enabled = false; - } - } catch (IOException e) { - LOG.error("Could not open GeoIP database {}", config.dbPath(), e); - this.enabled = false; - } + GeoIpResolverFactory resolverFactory = GeoIpResolverFactory.getInstance(); + ipLocationResolver = resolverFactory.createLocationResolver(resolveTime, config); + ipAsnResolver = resolverFactory.createIpAsnResolver(resolveTime, config); + + //TODO: Confirm with Dan/Rob/et. al, if enabled here should be if either (any) resolver is working/enabled + this.enabled = ipLocationResolver.isEnabled() && ipAsnResolver.isEnabled(); + } public boolean filter(Message message) { @@ -72,24 +71,51 @@ public boolean filter(Message message) { return false; } - for (Map.Entry field : message.getFields().entrySet()) { - final String key = field.getKey(); - if (!key.startsWith(Message.INTERNAL_FIELD_PREFIX)) { - final Optional geoLocationInformation = extractGeoLocationInformation(field.getValue()); - geoLocationInformation.ifPresent(locationInformation -> { - // We will store the coordinates as a "lat,long" string - message.addField(key + "_geolocation", locationInformation.latitude() + "," + locationInformation.longitude()); - message.addField(key + "_country_code", locationInformation.countryIsoCode()); - message.addField(key + "_city_name", locationInformation.cityName()); - }); + List ipFields = getIpAddressFields(message); + + for (String key : ipFields) { + Object fieldValue = message.getField(key); + final InetAddress address = getValidRoutableInetAddress(fieldValue); + if (address == null) { + continue; } + + final String prefix = ipAddressFields.get(key); + ipLocationResolver.getGeoIpData(address).ifPresent(locationInformation -> { + message.addField(prefix + "_geo_coordinates", locationInformation.latitude() + "," + locationInformation.longitude()); + message.addField(prefix + "_geo_country", locationInformation.countryIsoCode()); + message.addField(prefix + "_geo_city", locationInformation.cityName()); + message.addField(prefix + "_geo_region", locationInformation.region()); + message.addField(prefix + "_geo_timeZone", locationInformation.timeZone()); + }); + + ipAsnResolver.getGeoIpData(address).ifPresent(info -> { + + message.addField(prefix + "_as_organization", info.organization()); + message.addField(prefix + "_as_number", info.asn()); + }); + } - return false; + return true; } + private List getIpAddressFields(Message message) { + return message.getFieldNames() + .stream() + .filter(e -> ipAddressFields.containsKey(e) + && !e.startsWith(Message.INTERNAL_FIELD_PREFIX)) + .collect(Collectors.toList()); + } + + //TODO: remove this and unit tests--test the resolvers individually instead @VisibleForTesting - Optional extractGeoLocationInformation(Object fieldValue) { + Optional extractGeoLocationInformation(InetAddress address) { + + return ipLocationResolver.getGeoIpData(address); + } + + private InetAddress getValidRoutableInetAddress(Object fieldValue) { final InetAddress ipAddress; if (fieldValue instanceof InetAddress) { ipAddress = (InetAddress) fieldValue; @@ -98,26 +124,7 @@ Optional extractGeoLocationInformation(Object fieldValue } else { ipAddress = null; } - - GeoLocationInformation geoLocationInformation = null; - if (ipAddress != null) { - try (Timer.Context ignored = resolveTime.time()) { - final CityResponse response = databaseReader.city(ipAddress); - final Location location = response.getLocation(); - final Country country = response.getCountry(); - final City city = response.getCity(); - - geoLocationInformation = GeoLocationInformation.create( - location.getLatitude(), location.getLongitude(), - country.getGeoNameId() != null ? country.getIsoCode() : "N/A", - city.getGeoNameId() != null ? city.getName() : "N/A" // calling to .getName() may throw a NPE - ); - } catch (Exception e) { - LOG.debug("Could not get location from IP {}", ipAddress.getHostAddress(), e); - } - } - - return Optional.ofNullable(geoLocationInformation); + return ipAddress; } @Nullable @@ -131,19 +138,4 @@ InetAddress getIpFromFieldValue(String fieldValue) { return null; } - - @AutoValue - static abstract class GeoLocationInformation { - public abstract double latitude(); - - public abstract double longitude(); - - public abstract String countryIsoCode(); - - public abstract String cityName(); - - public static GeoLocationInformation create(double latitude, double longitude, String countryIsoCode, String cityName) { - return new AutoValue_GeoIpResolverEngine_GeoLocationInformation(latitude, longitude, countryIsoCode, cityName); - } - } } diff --git a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpResolverFactory.java b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpResolverFactory.java new file mode 100644 index 000000000000..2895cf87227c --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpResolverFactory.java @@ -0,0 +1,78 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ + +package org.graylog.plugins.map.geoip; + +import com.codahale.metrics.Timer; +import org.graylog.plugins.map.config.DatabaseType; +import org.graylog.plugins.map.config.GeoIpResolverConfig; + +public class GeoIpResolverFactory { + private static GeoIpResolverFactory INSTANCE; + + private GeoIpResolverFactory() { + } + + //TODO: Find a way around this wild card--remove param from types? + public GeoIpResolver createLocationResolver(Timer timer, GeoIpResolverConfig config) { + + final GeoIpResolver resolver; + switch (config.cityDbType()) { + case MAXMIND_CITY: + resolver = new MaxMindIpLocationResolver(timer, config.cityDbPath(), config.enabled()); + break; + case IPINFO_STANDARD_LOCATION: + resolver = new IpInfoIpLocationResolver(timer, config.cityDbPath(), config.enabled()); + break; + default: + String opts = String.join(",", DatabaseType.MAXMIND_CITY.name(), DatabaseType.IPINFO_STANDARD_LOCATION.name()); + String error = String.format("'%s' is not a valid DatabaseType for a GeoLocation Resolver. Valid options are: %s", opts); + throw new IllegalArgumentException(error); + } + + return resolver; + } + + //TODO: Find a way around this wild card--remove param from types? + public GeoIpResolver createIpAsnResolver(Timer timer, GeoIpResolverConfig config) { + + final GeoIpResolver resolver; + + switch (config.asnDbType()) { + case IPINFO_ASN: + resolver = new IpInfoIpAsnResolver(timer, config.asnDbPath(), config.enabled()); + break; + case MAXMIND_ASN: + resolver = new MaxMindIpAsnResolver(timer, config.asnDbPath(), config.enabled()); + break; + default: + String opts = String.join(",", DatabaseType.MAXMIND_ASN.name(), DatabaseType.IPINFO_ASN.name()); + String error = String.format("'%s' is not a valid DatabaseType for a GeoLocation Resolver. Valid options are: %s", config.asnDbType(), opts); + throw new IllegalArgumentException(error); + } + + return resolver; + } + + public static synchronized GeoIpResolverFactory getInstance() { + if (INSTANCE == null) { + INSTANCE = new GeoIpResolverFactory(); + } + + return INSTANCE; + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoLocationInformation.java b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoLocationInformation.java new file mode 100644 index 000000000000..5f68fca477fb --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoLocationInformation.java @@ -0,0 +1,41 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ + +package org.graylog.plugins.map.geoip; + +import com.google.auto.value.AutoValue; + +@AutoValue +abstract class GeoLocationInformation { + public abstract double latitude(); + + public abstract double longitude(); + + public abstract String countryIsoCode(); + + public abstract String cityName(); + + public abstract String region(); + + public abstract String timeZone(); + + public static GeoLocationInformation create(double latitude, double longitude, String countryIsoCode, String cityName, + String region, String timeZone) { + return new AutoValue_GeoLocationInformation(latitude, longitude, countryIsoCode, cityName, + region, timeZone); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/IpInfoIpAsnResolver.java b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/IpInfoIpAsnResolver.java new file mode 100644 index 000000000000..4ccbbd99c856 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/IpInfoIpAsnResolver.java @@ -0,0 +1,63 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ + +package org.graylog.plugins.map.geoip; + +import com.codahale.metrics.Timer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.net.InetAddress; +import java.util.Optional; + +public class IpInfoIpAsnResolver extends GeoIpResolver { + private static Logger LOG = LoggerFactory.getLogger(IpInfoIpAsnResolver.class); + + public IpInfoIpAsnResolver(Timer timer, String configPath, boolean enabled) { + super(timer, configPath, enabled); + } + + @Override + IPinfoIPLocationDatabaseAdapter createDataProvider(File configFile) { + + IPinfoIPLocationDatabaseAdapter adapter; + try { + adapter = new IPinfoIPLocationDatabaseAdapter(configFile); + } catch (IOException e) { + String error = String.format("Error creating '%s'. %s", getClass(), e.getMessage()); + LOG.warn(error, e); + adapter = null; + } + return adapter; + } + + @Override + protected Optional doGetGeoIpData(InetAddress address) { + + GeoAsnInformation info; + try { + final IPinfoASN ipInfoASN = dataProvider.ipInfoASN(address); + info = GeoAsnInformation.create(ipInfoASN.name(), ipInfoASN.type(), ipInfoASN.asn()); + } catch (Exception e) { + info = null; + } + return Optional.ofNullable(info); + + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/IpInfoIpLocationResolver.java b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/IpInfoIpLocationResolver.java new file mode 100644 index 000000000000..7b6b0de3d2c1 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/IpInfoIpLocationResolver.java @@ -0,0 +1,68 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ + +package org.graylog.plugins.map.geoip; + +import com.codahale.metrics.Timer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.net.InetAddress; +import java.util.Optional; + +public class IpInfoIpLocationResolver extends GeoIpResolver { + + private static final Logger LOG = LoggerFactory.getLogger(IpInfoIpLocationResolver.class); + + public IpInfoIpLocationResolver(Timer resolveTime, String configPath, boolean enabled) { + super(resolveTime, configPath, enabled); + } + + @Override + IPinfoIPLocationDatabaseAdapter createDataProvider(File configFile) { + + IPinfoIPLocationDatabaseAdapter adapter; + try { + adapter = new IPinfoIPLocationDatabaseAdapter(configFile); + } catch (IOException e) { + String error = String.format("Error creating '%s'. %s", getClass(), configFile); + LOG.error(error); + adapter = null; + } + + return adapter; + } + + @Override + protected Optional doGetGeoIpData(InetAddress address) { + GeoLocationInformation info; + + try { + IPinfoStandardLocation loc = dataProvider.ipInfoStandardLocation(address); + info = GeoLocationInformation.create(loc.latitude(), loc.longitude(), loc.country(), + loc.city(), loc.region(), loc.timezone()); + + } catch (Exception e) { + String error = String.format("Error getting IP location info for '%s'. %s", address, e.getMessage()); + LOG.error(error, e); + info = null; + } + return Optional.ofNullable(info); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/MaxMindIpAsnResolver.java b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/MaxMindIpAsnResolver.java new file mode 100644 index 000000000000..64315b182165 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/MaxMindIpAsnResolver.java @@ -0,0 +1,65 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ + +package org.graylog.plugins.map.geoip; + +import com.codahale.metrics.Timer; +import com.maxmind.geoip2.DatabaseReader; +import com.maxmind.geoip2.exception.GeoIp2Exception; +import com.maxmind.geoip2.model.AsnResponse; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.net.InetAddress; +import java.util.Optional; + +public class MaxMindIpAsnResolver extends GeoIpResolver { + + private static final Logger LOG = LoggerFactory.getLogger(MaxMindIpAsnResolver.class); + + public MaxMindIpAsnResolver(Timer resolveTime, String configPath, boolean enabled) { + super(resolveTime, configPath, enabled); + } + + @Override + DatabaseReader createDataProvider(File configFile) { + try { + return new DatabaseReader.Builder(configFile).build(); + } catch (IOException e) { + String error = String.format("Error creating '%s'. %s", getClass().getName(), e.getMessage()); + throw new IllegalStateException(error, e); + } + } + + @Override + protected Optional doGetGeoIpData(InetAddress address) { + GeoAsnInformation asn; + try { + AsnResponse response = dataProvider.asn(address); + String number = response.getAutonomousSystemNumber() == null ? "N/A" : response.getAutonomousSystemNumber().toString(); + asn = GeoAsnInformation.create(response.getAutonomousSystemOrganization(), "N/A", number); + } catch (GeoIp2Exception | IOException | UnsupportedOperationException e) { + String error = String.format("Error getting ASN for IP Address '%s'. %s", address, e.getMessage()); + LOG.warn(error, e); + asn = null; + } + + return Optional.ofNullable(asn); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/MaxMindIpLocationResolver.java b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/MaxMindIpLocationResolver.java new file mode 100644 index 000000000000..33c3057f1034 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/MaxMindIpLocationResolver.java @@ -0,0 +1,75 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ + +package org.graylog.plugins.map.geoip; + +import com.codahale.metrics.Timer; +import com.maxmind.geoip2.DatabaseReader; +import com.maxmind.geoip2.model.CityResponse; +import com.maxmind.geoip2.record.City; +import com.maxmind.geoip2.record.Country; +import com.maxmind.geoip2.record.Location; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.net.InetAddress; +import java.util.Optional; + +public class MaxMindIpLocationResolver extends GeoIpResolver { + + private static final Logger LOG = LoggerFactory.getLogger(MaxMindIpLocationResolver.class); + + public MaxMindIpLocationResolver(Timer resolveTime, String configPath, boolean enabled) { + super(resolveTime, configPath, enabled); + } + + @Override + DatabaseReader createDataProvider(File configFile) { + try { + return new DatabaseReader.Builder(configFile).build(); + } catch (IOException e) { + String error = String.format("Error creating '%s'. %s", getClass().getName(), e.getMessage()); + throw new IllegalStateException(error, e); + } + } + + @Override + public Optional doGetGeoIpData(InetAddress address) { + + GeoLocationInformation info; + try (Timer.Context ignored = resolveTime.time()) { + final CityResponse response = dataProvider.city(address); + final Location location = response.getLocation(); + final Country country = response.getCountry(); + final City city = response.getCity(); + + info = GeoLocationInformation.create( + location.getLatitude(), location.getLongitude(), + country.getGeoNameId() == null ? "N/A" : country.getIsoCode(), + city.getGeoNameId() == null ? "N/A" : city.getName(),// calling to .getName() may throw a NPE + "N/A", + "N/A"); + } catch (Exception e) { + LOG.debug("Could not get location from IP {}", address.getHostAddress(), e); + info = null; + } + + return Optional.ofNullable(info); + } +} diff --git a/graylog2-server/src/test/java/org/graylog/plugins/map/geoip/GeoIpResolverEngineTest.java b/graylog2-server/src/test/java/org/graylog/plugins/map/geoip/GeoIpResolverEngineTest.java index d1a409eac521..e721f047ea06 100644 --- a/graylog2-server/src/test/java/org/graylog/plugins/map/geoip/GeoIpResolverEngineTest.java +++ b/graylog2-server/src/test/java/org/graylog/plugins/map/geoip/GeoIpResolverEngineTest.java @@ -50,7 +50,7 @@ public class GeoIpResolverEngineTest { @Before public void setUp() throws URISyntaxException { - config = GeoIpResolverConfig.defaultConfig().toBuilder().enabled(true).dbPath(this.getTestDatabasePath()).build(); + config = GeoIpResolverConfig.defaultConfig().toBuilder().enabled(true).cityDbPath(this.getTestDatabasePath()).build(); metricRegistry = new MetricRegistry(); } @@ -84,11 +84,13 @@ public void trimFieldValueBeforeLookup() { @Test public void extractGeoLocationInformation() { + final GeoIpResolverEngine resolver = new GeoIpResolverEngine(config, metricRegistry); - assertTrue("Should extract geo location information from public addresses", resolver.extractGeoLocationInformation("1.2.3.4").isPresent()); - assertFalse("Should not extract geo location information from private addresses", resolver.extractGeoLocationInformation("192.168.0.1").isPresent()); - assertFalse("Should not extract geo location information numeric fields", resolver.extractGeoLocationInformation(42).isPresent()); + //TODO: update these tests to pass InetAddress--use separate test to test Object-to-InetAddress converter + //assertTrue("Should extract geo location information from public addresses", resolver.extractGeoLocationInformation("1.2.3.4").isPresent()); + //assertFalse("Should not extract geo location information from private addresses", resolver.extractGeoLocationInformation("192.168.0.1").isPresent()); + //assertFalse("Should not extract geo location information numeric fields", resolver.extractGeoLocationInformation(42).isPresent()); assertTrue("Should extract geo location information IP address fields", resolver.extractGeoLocationInformation(InetAddresses.forString("1.2.3.4")).isPresent()); } @@ -146,4 +148,4 @@ public void filterResolvesIpGeoLocation() { assertFieldResolved(message, "extracted_ip", "Should have resolved public IP"); assertFieldResolved(message, "ipv6", "Should have resolved public IPv6"); } -} \ No newline at end of file +} diff --git a/pom.xml b/pom.xml index a6daeeaa53ea..117fdc3d62db 100644 --- a/pom.xml +++ b/pom.xml @@ -114,6 +114,7 @@ 2.1.12 6.1.2.Final 2.6.1 + 2.1 2.9.10.20200411 0.13.0 0.9.0 From f6e2da896c9b8ed4190ad97456d548df6b996f56 Mon Sep 17 00:00:00 2001 From: Roberto Benitez Date: Tue, 21 Dec 2021 14:13:10 -0600 Subject: [PATCH 02/28] Updated GeoIpResolverConfig to use a DatabaseVendorType instead of individual db types --- .../map/config/DatabaseVendorType.java | 39 +++++++++++++++++++ .../map/config/GeoIpResolverConfig.java | 20 +++------- .../map/geoip/GeoIpResolverEngine.java | 3 +- .../map/geoip/GeoIpResolverFactory.java | 10 +++-- 4 files changed, 52 insertions(+), 20 deletions(-) create mode 100644 graylog2-server/src/main/java/org/graylog/plugins/map/config/DatabaseVendorType.java diff --git a/graylog2-server/src/main/java/org/graylog/plugins/map/config/DatabaseVendorType.java b/graylog2-server/src/main/java/org/graylog/plugins/map/config/DatabaseVendorType.java new file mode 100644 index 000000000000..e9d39093ef60 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/map/config/DatabaseVendorType.java @@ -0,0 +1,39 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ + +package org.graylog.plugins.map.config; + +public enum DatabaseVendorType { + MAXMIND(DatabaseType.MAXMIND_CITY, DatabaseType.MAXMIND_ASN), + IPINFO(DatabaseType.IPINFO_STANDARD_LOCATION, DatabaseType.IPINFO_ASN); + + private final DatabaseType cityDbType; + private final DatabaseType asnDbType; + + DatabaseVendorType(DatabaseType cityDbType, DatabaseType asnDbType) { + this.cityDbType = cityDbType; + this.asnDbType = asnDbType; + } + + public DatabaseType getCityDbType() { + return cityDbType; + } + + public DatabaseType getAsnDbType() { + return asnDbType; + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/map/config/GeoIpResolverConfig.java b/graylog2-server/src/main/java/org/graylog/plugins/map/config/GeoIpResolverConfig.java index 4161a41b20a3..a3b8204c1dc8 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/map/config/GeoIpResolverConfig.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/map/config/GeoIpResolverConfig.java @@ -30,29 +30,24 @@ public abstract class GeoIpResolverConfig { @JsonProperty("enabled") public abstract boolean enabled(); - @JsonProperty("db_type") - public abstract DatabaseType cityDbType(); + @JsonProperty("database_vendor_type") + public abstract DatabaseVendorType databaseVendorType(); @JsonProperty("db_path") public abstract String cityDbPath(); - @JsonProperty("asn_db_type") - public abstract DatabaseType asnDbType(); - @JsonProperty("asn_db_path") public abstract String asnDbPath(); @JsonCreator public static GeoIpResolverConfig create(@JsonProperty("enabled") boolean cityEnabled, - @JsonProperty("db_type") DatabaseType dbType, + @JsonProperty("database_vendor_type") DatabaseVendorType databaseVendorType, @JsonProperty("db_path") String cityDbPath, - @JsonProperty("asn_db_type") DatabaseType asnDbType, @JsonProperty("asn_db_path") String asnDbPath) { return builder() .enabled(cityEnabled) - .cityDbType(dbType) + .databaseVendorType(databaseVendorType) .cityDbPath(cityDbPath) - .asnDbType(asnDbType) .asnDbPath(asnDbPath) .build(); } @@ -60,9 +55,8 @@ public static GeoIpResolverConfig create(@JsonProperty("enabled") boolean cityEn public static GeoIpResolverConfig defaultConfig() { return builder() .enabled(false) - .cityDbType(DatabaseType.MAXMIND_CITY) + .databaseVendorType(DatabaseVendorType.MAXMIND) .cityDbPath("/etc/graylog/server/GeoLite2-City.mmdb") - .asnDbType(DatabaseType.MAXMIND_ASN) .asnDbPath("/etc/graylog/server/GeoLite2-ASN.mmdb") .build(); } @@ -77,12 +71,10 @@ public static Builder builder() { public abstract static class Builder { public abstract Builder enabled(boolean enabled); - public abstract Builder cityDbType(DatabaseType dbType); + public abstract Builder databaseVendorType(DatabaseVendorType type); public abstract Builder cityDbPath(String dbPath); - public abstract Builder asnDbType(DatabaseType dbType); - public abstract Builder asnDbPath(String asnDBPath); public abstract GeoIpResolverConfig build(); diff --git a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpResolverEngine.java b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpResolverEngine.java index 3a00fac304d1..4fadd649b2bb 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpResolverEngine.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpResolverEngine.java @@ -48,14 +48,13 @@ public class GeoIpResolverEngine { .put("destination_ip", "destination") .build(); - private final Timer resolveTime; private final GeoIpResolver ipLocationResolver; private final GeoIpResolver ipAsnResolver; private boolean enabled; public GeoIpResolverEngine(GeoIpResolverConfig config, MetricRegistry metricRegistry) { - this.resolveTime = metricRegistry.timer(name(GeoIpResolverEngine.class, "resolveTime")); + Timer resolveTime = metricRegistry.timer(name(GeoIpResolverEngine.class, "resolveTime")); GeoIpResolverFactory resolverFactory = GeoIpResolverFactory.getInstance(); ipLocationResolver = resolverFactory.createLocationResolver(resolveTime, config); diff --git a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpResolverFactory.java b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpResolverFactory.java index a0182122fb15..1411de7af920 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpResolverFactory.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpResolverFactory.java @@ -33,7 +33,8 @@ private GeoIpResolverFactory() { public GeoIpResolver createLocationResolver(Timer timer, GeoIpResolverConfig config) { final GeoIpResolver resolver; - switch (config.cityDbType()) { + DatabaseType dbType = config.databaseVendorType().getCityDbType(); + switch (dbType) { case MAXMIND_CITY: resolver = new MaxMindIpLocationResolver(timer, config.cityDbPath(), config.enabled()); break; @@ -42,7 +43,7 @@ public GeoIpResolver createLocationResolver(Timer tim break; default: String opts = String.join(",", DatabaseType.MAXMIND_CITY.name(), DatabaseType.IPINFO_STANDARD_LOCATION.name()); - String error = String.format(Locale.US, "'%s' is not a valid DatabaseType for a GeoLocation Resolver. Valid options are: %s", config.cityDbType(), opts); + String error = String.format(Locale.US, "'%s' is not a valid DatabaseType for a GeoLocation Resolver. Valid options are: %s", dbType, opts); throw new IllegalArgumentException(error); } @@ -54,7 +55,8 @@ public GeoIpResolver createIpAsnResolver(Timer timer, GeoI final GeoIpResolver resolver; - switch (config.asnDbType()) { + final DatabaseType dbType = config.databaseVendorType().getAsnDbType(); + switch (dbType) { case IPINFO_ASN: resolver = new IpInfoIpAsnResolver(timer, config.asnDbPath(), config.enabled()); break; @@ -63,7 +65,7 @@ public GeoIpResolver createIpAsnResolver(Timer timer, GeoI break; default: String opts = String.join(",", DatabaseType.MAXMIND_ASN.name(), DatabaseType.IPINFO_ASN.name()); - String error = String.format(Locale.US, "'%s' is not a valid DatabaseType for a GeoLocation Resolver. Valid options are: %s", config.asnDbType(), opts); + String error = String.format(Locale.US, "'%s' is not a valid DatabaseType for a GeoLocation Resolver. Valid options are: %s", dbType, opts); throw new IllegalArgumentException(error); } From c751444c15efba51ddcf8abff1e95bbf93456eec Mon Sep 17 00:00:00 2001 From: Roberto Benitez Date: Wed, 22 Dec 2021 10:50:44 -0600 Subject: [PATCH 03/28] Added database migration to remove field 'db_type' and add field 'database_vendor_type' for GeoIpResolverConfig. Updated GeoIpResolverFactory to be non-singleton. --- .../map/config/GeoIpResolverConfig.java | 2 +- .../plugins/map/geoip/GeoIpResolver.java | 12 +-- .../map/geoip/GeoIpResolverEngine.java | 33 +++---- .../map/geoip/GeoIpResolverFactory.java | 48 +++++----- .../map/geoip/IpInfoIpAsnResolver.java | 28 ++---- .../plugins/map/geoip/IpInfoIpResolver.java | 47 ++++++++++ ...olver.java => IpInfoLocationResolver.java} | 30 ++----- .../map/geoip/MaxMindIpAsnResolver.java | 23 ++--- .../map/geoip/MaxMindIpLocationResolver.java | 25 ++---- .../plugins/map/geoip/MaxMindIpResolver.java | 48 ++++++++++ .../graylog2/migrations/MigrationsModule.java | 3 +- ...21144300_GeoIpResolverConfigMigration.java | 87 +++++++++++++++++++ .../map/geoip/GeoIpResolverEngineTest.java | 19 ++-- .../map/geoip/GeoIpResolverFactoryTest.java | 86 ++++++++++++++++++ 14 files changed, 346 insertions(+), 145 deletions(-) create mode 100644 graylog2-server/src/main/java/org/graylog/plugins/map/geoip/IpInfoIpResolver.java rename graylog2-server/src/main/java/org/graylog/plugins/map/geoip/{IpInfoIpLocationResolver.java => IpInfoLocationResolver.java} (60%) create mode 100644 graylog2-server/src/main/java/org/graylog/plugins/map/geoip/MaxMindIpResolver.java create mode 100644 graylog2-server/src/main/java/org/graylog2/migrations/V20211221144300_GeoIpResolverConfigMigration.java create mode 100644 graylog2-server/src/test/java/org/graylog/plugins/map/geoip/GeoIpResolverFactoryTest.java diff --git a/graylog2-server/src/main/java/org/graylog/plugins/map/config/GeoIpResolverConfig.java b/graylog2-server/src/main/java/org/graylog/plugins/map/config/GeoIpResolverConfig.java index a3b8204c1dc8..4a0cf6b61af9 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/map/config/GeoIpResolverConfig.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/map/config/GeoIpResolverConfig.java @@ -46,7 +46,7 @@ public static GeoIpResolverConfig create(@JsonProperty("enabled") boolean cityEn @JsonProperty("asn_db_path") String asnDbPath) { return builder() .enabled(cityEnabled) - .databaseVendorType(databaseVendorType) + .databaseVendorType(databaseVendorType == null ? DatabaseVendorType.MAXMIND : databaseVendorType) .cityDbPath(cityDbPath) .asnDbPath(asnDbPath) .build(); diff --git a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpResolver.java b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpResolver.java index cbad11cd3663..ab663120ed71 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpResolver.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpResolver.java @@ -26,13 +26,12 @@ import java.nio.file.Files; import java.util.Optional; -public abstract class GeoIpResolver { +public abstract class GeoIpResolver { private static final Logger LOG = LoggerFactory.getLogger(GeoIpResolver.class); protected final Timer resolveTime; private final boolean enabled; - protected final P dataProvider; GeoIpResolver(Timer resolveTime, String configPath, boolean enabled) { @@ -40,16 +39,13 @@ public abstract class GeoIpResolver { if (enabled) { final File configFile = new File(configPath); if (Files.exists(configFile.toPath())) { - this.dataProvider = createDataProvider(configFile); - this.enabled = true; + this.enabled = createDataProvider(configFile); } else { LOG.warn("'{}' database file does not exist: {}", getClass().getName(), configPath); this.enabled = false; - this.dataProvider = null; } } else { this.enabled = false; - this.dataProvider = null; } } @@ -57,10 +53,10 @@ public boolean isEnabled() { return enabled; } - abstract P createDataProvider(File configFile); + abstract boolean createDataProvider(File configFile); public Optional getGeoIpData(InetAddress address) { - if (!enabled || dataProvider == null || address == null) { + if (!enabled || address == null) { return Optional.empty(); } diff --git a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpResolverEngine.java b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpResolverEngine.java index 4fadd649b2bb..01d5e7c6cd0a 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpResolverEngine.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpResolverEngine.java @@ -29,8 +29,8 @@ import javax.annotation.Nullable; import java.net.InetAddress; import java.util.List; +import java.util.Locale; import java.util.Map; -import java.util.Optional; import java.util.stream.Collectors; import static com.codahale.metrics.MetricRegistry.name; @@ -48,20 +48,23 @@ public class GeoIpResolverEngine { .put("destination_ip", "destination") .build(); - private final GeoIpResolver ipLocationResolver; - private final GeoIpResolver ipAsnResolver; + private final GeoIpResolver ipLocationResolver; + private final GeoIpResolver ipAsnResolver; private boolean enabled; public GeoIpResolverEngine(GeoIpResolverConfig config, MetricRegistry metricRegistry) { Timer resolveTime = metricRegistry.timer(name(GeoIpResolverEngine.class, "resolveTime")); - GeoIpResolverFactory resolverFactory = GeoIpResolverFactory.getInstance(); - ipLocationResolver = resolverFactory.createLocationResolver(resolveTime, config); - ipAsnResolver = resolverFactory.createIpAsnResolver(resolveTime, config); + GeoIpResolverFactory resolverFactory = new GeoIpResolverFactory(config, resolveTime); + ipLocationResolver = resolverFactory.createIpCityResolver(); + ipAsnResolver = resolverFactory.createIpAsnResolver(); - //TODO: Confirm with Dan/Rob/et. al, if enabled here should be if either (any) resolver is working/enabled - this.enabled = ipLocationResolver.isEnabled() && ipAsnResolver.isEnabled(); + LOG.info("Created Geo IP Resolvers for '{}'", resolverFactory.getDatabaseVendorType()); + LOG.info("'{}' Status Enabled: {}", ipLocationResolver.getClass().getSimpleName(), ipLocationResolver.isEnabled()); + LOG.info("'{}' Status Enabled: {}", ipAsnResolver.getClass().getSimpleName(), ipAsnResolver.isEnabled()); + + this.enabled = ipLocationResolver.isEnabled() || ipAsnResolver.isEnabled(); } @@ -79,6 +82,8 @@ public boolean filter(Message message) { continue; } + //TODO: Tag any reserved IP addresses with 'reserved_ip: true' once ReservedIpChecker is merged + final String prefix = ipAddressFields.get(key); ipLocationResolver.getGeoIpData(address).ifPresent(locationInformation -> { message.addField(prefix + "_geo_coordinates", locationInformation.latitude() + "," + locationInformation.longitude()); @@ -86,6 +91,11 @@ public boolean filter(Message message) { message.addField(prefix + "_geo_city", locationInformation.cityName()); message.addField(prefix + "_geo_region", locationInformation.region()); message.addField(prefix + "_geo_timeZone", locationInformation.timeZone()); + + if (!(locationInformation.countryIsoCode() == null || locationInformation.cityName() == null)) { + String name = String.format(Locale.ENGLISH, "%s, %s", locationInformation.cityName(), locationInformation.countryIsoCode()); + message.addField(prefix + "_geo_name", name); + } }); ipAsnResolver.getGeoIpData(address).ifPresent(info -> { @@ -107,13 +117,6 @@ private List getIpAddressFields(Message message) { .collect(Collectors.toList()); } - //TODO: remove this and unit tests--test the resolvers individually instead - @VisibleForTesting - Optional extractGeoLocationInformation(InetAddress address) { - - return ipLocationResolver.getGeoIpData(address); - } - private InetAddress getValidRoutableInetAddress(Object fieldValue) { final InetAddress ipAddress; if (fieldValue instanceof InetAddress) { diff --git a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpResolverFactory.java b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpResolverFactory.java index 1411de7af920..1569a26bb03b 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpResolverFactory.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpResolverFactory.java @@ -19,64 +19,68 @@ import com.codahale.metrics.Timer; import org.graylog.plugins.map.config.DatabaseType; +import org.graylog.plugins.map.config.DatabaseVendorType; import org.graylog.plugins.map.config.GeoIpResolverConfig; import java.util.Locale; +import java.util.Objects; +/** + * A factory to create ASN and Location {@link GeoIpResolver} resolvers based on the {@link DatabaseVendorType} contained in + * the current {@link GeoIpResolverConfig}. + */ public class GeoIpResolverFactory { - private static GeoIpResolverFactory INSTANCE; - private GeoIpResolverFactory() { + private final GeoIpResolverConfig config; + private final Timer resolveTime; + + public GeoIpResolverFactory(GeoIpResolverConfig config, Timer resolveTime) { + this.config = Objects.requireNonNull(config, "Configuration must not be null"); + this.resolveTime = resolveTime; + } + + public DatabaseVendorType getDatabaseVendorType() { + return config.databaseVendorType(); } - //TODO: Find a way around this wild card--remove param from types? - public GeoIpResolver createLocationResolver(Timer timer, GeoIpResolverConfig config) { + public GeoIpResolver createIpCityResolver() { - final GeoIpResolver resolver; + final GeoIpResolver resolver; DatabaseType dbType = config.databaseVendorType().getCityDbType(); switch (dbType) { case MAXMIND_CITY: - resolver = new MaxMindIpLocationResolver(timer, config.cityDbPath(), config.enabled()); + resolver = new MaxMindIpLocationResolver(resolveTime, config.cityDbPath(), config.enabled()); break; case IPINFO_STANDARD_LOCATION: - resolver = new IpInfoIpLocationResolver(timer, config.cityDbPath(), config.enabled()); + resolver = new IpInfoLocationResolver(resolveTime, config.cityDbPath(), config.enabled()); break; default: String opts = String.join(",", DatabaseType.MAXMIND_CITY.name(), DatabaseType.IPINFO_STANDARD_LOCATION.name()); - String error = String.format(Locale.US, "'%s' is not a valid DatabaseType for a GeoLocation Resolver. Valid options are: %s", dbType, opts); + String error = String.format(Locale.US, "'%s' is not a valid DatabaseType for a City Geo IP Resolver. Valid options are: %s", dbType, opts); throw new IllegalArgumentException(error); } return resolver; } - //TODO: Find a way around this wild card--remove param from types? - public GeoIpResolver createIpAsnResolver(Timer timer, GeoIpResolverConfig config) { + public GeoIpResolver createIpAsnResolver() { - final GeoIpResolver resolver; + final GeoIpResolver resolver; final DatabaseType dbType = config.databaseVendorType().getAsnDbType(); switch (dbType) { case IPINFO_ASN: - resolver = new IpInfoIpAsnResolver(timer, config.asnDbPath(), config.enabled()); + resolver = new IpInfoIpAsnResolver(resolveTime, config.asnDbPath(), config.enabled()); break; case MAXMIND_ASN: - resolver = new MaxMindIpAsnResolver(timer, config.asnDbPath(), config.enabled()); + resolver = new MaxMindIpAsnResolver(resolveTime, config.asnDbPath(), config.enabled()); break; default: String opts = String.join(",", DatabaseType.MAXMIND_ASN.name(), DatabaseType.IPINFO_ASN.name()); - String error = String.format(Locale.US, "'%s' is not a valid DatabaseType for a GeoLocation Resolver. Valid options are: %s", dbType, opts); + String error = String.format(Locale.US, "'%s' is not a valid DatabaseType for an ASN Geo IP Resolver. Valid options are: %s", dbType, opts); throw new IllegalArgumentException(error); } return resolver; } - - public static synchronized GeoIpResolverFactory getInstance() { - if (INSTANCE == null) { - INSTANCE = new GeoIpResolverFactory(); - } - - return INSTANCE; - } } diff --git a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/IpInfoIpAsnResolver.java b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/IpInfoIpAsnResolver.java index be98d9b400be..cb0ed061e2a6 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/IpInfoIpAsnResolver.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/IpInfoIpAsnResolver.java @@ -18,47 +18,29 @@ package org.graylog.plugins.map.geoip; import com.codahale.metrics.Timer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import java.io.File; -import java.io.IOException; import java.net.InetAddress; -import java.util.Locale; import java.util.Optional; -public class IpInfoIpAsnResolver extends GeoIpResolver { - private static Logger LOG = LoggerFactory.getLogger(IpInfoIpAsnResolver.class); +/** + * A {@link GeoIpResolver} to load IP ASN data from {@link org.graylog.plugins.map.config.DatabaseVendorType#IPINFO}. + */ +public class IpInfoIpAsnResolver extends IpInfoIpResolver { public IpInfoIpAsnResolver(Timer timer, String configPath, boolean enabled) { super(timer, configPath, enabled); } - @Override - IPinfoIPLocationDatabaseAdapter createDataProvider(File configFile) { - - IPinfoIPLocationDatabaseAdapter adapter; - try { - adapter = new IPinfoIPLocationDatabaseAdapter(configFile); - } catch (IOException e) { - String error = String.format(Locale.US, "Error creating '%s'. %s", getClass(), e.getMessage()); - LOG.warn(error, e); - adapter = null; - } - return adapter; - } - @Override protected Optional doGetGeoIpData(InetAddress address) { GeoAsnInformation info; try { - final IPinfoASN ipInfoASN = dataProvider.ipInfoASN(address); + final IPinfoASN ipInfoASN = adapter.ipInfoASN(address); info = GeoAsnInformation.create(ipInfoASN.name(), ipInfoASN.type(), ipInfoASN.asn()); } catch (Exception e) { info = null; } return Optional.ofNullable(info); - } } diff --git a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/IpInfoIpResolver.java b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/IpInfoIpResolver.java new file mode 100644 index 000000000000..7a5cc59bd470 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/IpInfoIpResolver.java @@ -0,0 +1,47 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ + +package org.graylog.plugins.map.geoip; + +import com.codahale.metrics.Timer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; + +abstract class IpInfoIpResolver extends GeoIpResolver { + protected static final Logger LOG = LoggerFactory.getLogger(IpInfoIpResolver.class); + protected IPinfoIPLocationDatabaseAdapter adapter; + + IpInfoIpResolver(Timer resolveTime, String configPath, boolean enabled) { + super(resolveTime, configPath, enabled); + } + + @Override + boolean createDataProvider(File configFile) { + + try { + adapter = new IPinfoIPLocationDatabaseAdapter(configFile); + } catch (IOException e) { + LOG.warn("Error creating IPinfoIPLocationDatabaseAdapter for '{}' from file '{}'", getClass().getSimpleName(), configFile); + adapter = null; + } + + return adapter != null; + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/IpInfoIpLocationResolver.java b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/IpInfoLocationResolver.java similarity index 60% rename from graylog2-server/src/main/java/org/graylog/plugins/map/geoip/IpInfoIpLocationResolver.java rename to graylog2-server/src/main/java/org/graylog/plugins/map/geoip/IpInfoLocationResolver.java index 640ac71f62c0..312738140b9b 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/IpInfoIpLocationResolver.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/IpInfoLocationResolver.java @@ -18,44 +18,26 @@ package org.graylog.plugins.map.geoip; import com.codahale.metrics.Timer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import java.io.File; -import java.io.IOException; import java.net.InetAddress; import java.util.Locale; import java.util.Optional; -public class IpInfoIpLocationResolver extends GeoIpResolver { - - private static final Logger LOG = LoggerFactory.getLogger(IpInfoIpLocationResolver.class); +/** + * A {@link GeoIpResolver} to load IP Location data from {@link org.graylog.plugins.map.config.DatabaseVendorType#IPINFO}. + */ +public class IpInfoLocationResolver extends IpInfoIpResolver { - public IpInfoIpLocationResolver(Timer resolveTime, String configPath, boolean enabled) { + public IpInfoLocationResolver(Timer resolveTime, String configPath, boolean enabled) { super(resolveTime, configPath, enabled); } - @Override - IPinfoIPLocationDatabaseAdapter createDataProvider(File configFile) { - - IPinfoIPLocationDatabaseAdapter adapter; - try { - adapter = new IPinfoIPLocationDatabaseAdapter(configFile); - } catch (IOException e) { - String error = String.format(Locale.US, "Error creating '%s'. %s", getClass(), configFile); - LOG.error(error); - adapter = null; - } - - return adapter; - } - @Override protected Optional doGetGeoIpData(InetAddress address) { GeoLocationInformation info; try { - IPinfoStandardLocation loc = dataProvider.ipInfoStandardLocation(address); + IPinfoStandardLocation loc = adapter.ipInfoStandardLocation(address); info = GeoLocationInformation.create(loc.latitude(), loc.longitude(), loc.country(), loc.city(), loc.region(), loc.timezone()); diff --git a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/MaxMindIpAsnResolver.java b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/MaxMindIpAsnResolver.java index 47bd2aa2c93d..0f73adf2a60b 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/MaxMindIpAsnResolver.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/MaxMindIpAsnResolver.java @@ -18,41 +18,28 @@ package org.graylog.plugins.map.geoip; import com.codahale.metrics.Timer; -import com.maxmind.geoip2.DatabaseReader; import com.maxmind.geoip2.exception.GeoIp2Exception; import com.maxmind.geoip2.model.AsnResponse; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import java.io.File; import java.io.IOException; import java.net.InetAddress; import java.util.Locale; import java.util.Optional; -public class MaxMindIpAsnResolver extends GeoIpResolver { - - private static final Logger LOG = LoggerFactory.getLogger(MaxMindIpAsnResolver.class); +/** + * A {@link GeoIpResolver} to load IP ASN data from {@link org.graylog.plugins.map.config.DatabaseVendorType#MAXMIND}. + */ +public class MaxMindIpAsnResolver extends MaxMindIpResolver { public MaxMindIpAsnResolver(Timer resolveTime, String configPath, boolean enabled) { super(resolveTime, configPath, enabled); } - @Override - DatabaseReader createDataProvider(File configFile) { - try { - return new DatabaseReader.Builder(configFile).build(); - } catch (IOException e) { - String error = String.format(Locale.US, "Error creating '%s'. %s", getClass().getName(), e.getMessage()); - throw new IllegalStateException(error, e); - } - } - @Override protected Optional doGetGeoIpData(InetAddress address) { GeoAsnInformation asn; try { - AsnResponse response = dataProvider.asn(address); + AsnResponse response = databaseReader.asn(address); String number = response.getAutonomousSystemNumber() == null ? "N/A" : response.getAutonomousSystemNumber().toString(); asn = GeoAsnInformation.create(response.getAutonomousSystemOrganization(), "N/A", number); } catch (GeoIp2Exception | IOException | UnsupportedOperationException e) { diff --git a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/MaxMindIpLocationResolver.java b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/MaxMindIpLocationResolver.java index 553dd7087072..71b764e3bdaf 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/MaxMindIpLocationResolver.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/MaxMindIpLocationResolver.java @@ -18,44 +18,29 @@ package org.graylog.plugins.map.geoip; import com.codahale.metrics.Timer; -import com.maxmind.geoip2.DatabaseReader; import com.maxmind.geoip2.model.CityResponse; import com.maxmind.geoip2.record.City; import com.maxmind.geoip2.record.Country; import com.maxmind.geoip2.record.Location; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import java.io.File; -import java.io.IOException; import java.net.InetAddress; -import java.util.Locale; import java.util.Optional; -public class MaxMindIpLocationResolver extends GeoIpResolver { - - private static final Logger LOG = LoggerFactory.getLogger(MaxMindIpLocationResolver.class); +/** + * A {@link GeoIpResolver} to load IP location data from {@link org.graylog.plugins.map.config.DatabaseVendorType#MAXMIND}. + */ +public class MaxMindIpLocationResolver extends MaxMindIpResolver { public MaxMindIpLocationResolver(Timer resolveTime, String configPath, boolean enabled) { super(resolveTime, configPath, enabled); } - @Override - DatabaseReader createDataProvider(File configFile) { - try { - return new DatabaseReader.Builder(configFile).build(); - } catch (IOException e) { - String error = String.format(Locale.US, "Error creating '%s'. %s", getClass().getName(), e.getMessage()); - throw new IllegalStateException(error, e); - } - } - @Override public Optional doGetGeoIpData(InetAddress address) { GeoLocationInformation info; try (Timer.Context ignored = resolveTime.time()) { - final CityResponse response = dataProvider.city(address); + final CityResponse response = databaseReader.city(address); final Location location = response.getLocation(); final Country country = response.getCountry(); final City city = response.getCity(); diff --git a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/MaxMindIpResolver.java b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/MaxMindIpResolver.java new file mode 100644 index 000000000000..14ea1df42f0f --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/MaxMindIpResolver.java @@ -0,0 +1,48 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ + +package org.graylog.plugins.map.geoip; + +import com.codahale.metrics.Timer; +import com.maxmind.geoip2.DatabaseReader; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; + +abstract class MaxMindIpResolver extends GeoIpResolver { + protected static final Logger LOG = LoggerFactory.getLogger(MaxMindIpResolver.class); + + protected DatabaseReader databaseReader; + + MaxMindIpResolver(Timer resolveTime, String configPath, boolean enabled) { + super(resolveTime, configPath, enabled); + } + + @Override + boolean createDataProvider(File configFile) { + try { + databaseReader = new DatabaseReader.Builder(configFile).build(); + } catch (IOException e) { + LOG.warn("Error creating DatabaseReader for '{}' with config file '{}'", getClass().getSimpleName(), configFile); + databaseReader = null; + } + + return databaseReader != null; + } +} diff --git a/graylog2-server/src/main/java/org/graylog2/migrations/MigrationsModule.java b/graylog2-server/src/main/java/org/graylog2/migrations/MigrationsModule.java index 343c6dad35be..e94fb81e3f72 100644 --- a/graylog2-server/src/main/java/org/graylog2/migrations/MigrationsModule.java +++ b/graylog2-server/src/main/java/org/graylog2/migrations/MigrationsModule.java @@ -50,8 +50,9 @@ protected void configure() { addMigration(V20200722110800_AddBuiltinRoles.class); addMigration(GrantsMetaMigration.class); addMigration(V20201103145400_LegacyAuthServiceMigration.class); + addMigration(V20211221144300_GeoIpResolverConfigMigration.class); // Make sure there is always a binder for migration modules - Multibinder.newSetBinder(binder(), V20201103145400_LegacyAuthServiceMigration.MigrationModule.class); + Multibinder.newSetBinder(binder(), V20211221144300_GeoIpResolverConfigMigration.class); } } diff --git a/graylog2-server/src/main/java/org/graylog2/migrations/V20211221144300_GeoIpResolverConfigMigration.java b/graylog2-server/src/main/java/org/graylog2/migrations/V20211221144300_GeoIpResolverConfigMigration.java new file mode 100644 index 000000000000..46c71eb303b8 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog2/migrations/V20211221144300_GeoIpResolverConfigMigration.java @@ -0,0 +1,87 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ + +package org.graylog2.migrations; + +import com.mongodb.client.MongoCollection; +import com.mongodb.client.model.Filters; +import com.mongodb.client.model.Updates; +import com.mongodb.client.result.UpdateResult; +import org.bson.Document; +import org.bson.conversions.Bson; +import org.graylog.plugins.map.config.DatabaseVendorType; +import org.graylog.plugins.map.config.GeoIpResolverConfig; +import org.graylog2.database.MongoConnection; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.inject.Inject; +import java.time.LocalDateTime; +import java.time.Month; +import java.time.ZoneId; +import java.time.ZonedDateTime; + +public class V20211221144300_GeoIpResolverConfigMigration extends Migration { + + private static final Logger LOG = LoggerFactory.getLogger(V20211221144300_GeoIpResolverConfigMigration.class); + private static final String COLLECTION_NAME = "cluster_config"; + public static final String PAYLOAD = "payload"; + private static final String FIELD_DB_VENDOR = PAYLOAD + ".database_vendor_type"; + private static final String FIELD_DB_TYPE = PAYLOAD + ".db_type"; + + private final MongoConnection mongoConnection; + + @Inject + public V20211221144300_GeoIpResolverConfigMigration(MongoConnection mongoConnection) { + this.mongoConnection = mongoConnection; + } + + @Override + public ZonedDateTime createdAt() { + return ZonedDateTime.of(LocalDateTime.of(2021, Month.DECEMBER, 21, 14, 43), ZoneId.systemDefault()); + } + + /** + * This code change modifies {@link GeoIpResolverConfig} by removing the field db_type and adding the field database_vendor_type. + * + *

+ * The objective of this migration is to add the new field (with value {@link DatabaseVendorType#MAXMIND}) if not already present, and to remove the old field. + *

+ */ + @Override + public void upgrade() { + + final MongoCollection collection = mongoConnection.getMongoDatabase().getCollection(COLLECTION_NAME); + LOG.info("Updating '{}' collection.", COLLECTION_NAME); + + Bson geoConfFiler = Filters.eq("type", GeoIpResolverConfig.class.getCanonicalName()); + Bson noColumnFilter = Filters.exists(FIELD_DB_VENDOR, false); + + //Set default db vendor field + Bson setDefaultVendor = Updates.set(FIELD_DB_VENDOR, DatabaseVendorType.MAXMIND.name()); + + //remove vestigial fields + Bson dropAsn = Updates.unset(FIELD_DB_TYPE); + + Bson updates = Updates.combine(setDefaultVendor, dropAsn); + LOG.info("Planned Updates: {}", updates); + final UpdateResult updateResult = collection.updateOne(Filters.and(geoConfFiler, noColumnFilter), updates); + LOG.info("Update Result: {}", updateResult); + + } +} + diff --git a/graylog2-server/src/test/java/org/graylog/plugins/map/geoip/GeoIpResolverEngineTest.java b/graylog2-server/src/test/java/org/graylog/plugins/map/geoip/GeoIpResolverEngineTest.java index e721f047ea06..8a0084f085a4 100644 --- a/graylog2-server/src/test/java/org/graylog/plugins/map/geoip/GeoIpResolverEngineTest.java +++ b/graylog2-server/src/test/java/org/graylog/plugins/map/geoip/GeoIpResolverEngineTest.java @@ -31,6 +31,7 @@ import org.junit.runner.RunWith; import java.net.URISyntaxException; +import java.net.URL; import java.util.Map; import static com.codahale.metrics.MetricRegistry.name; @@ -61,7 +62,11 @@ public void tearDown() { } private String getTestDatabasePath() throws URISyntaxException { - return this.getClass().getResource(GEO_LITE2_CITY_MMDB).toURI().getPath(); + final URL url = getClass().getResource(GEO_LITE2_CITY_MMDB); + if (url == null) { + throw new IllegalStateException("Resource not found. " + GEO_LITE2_CITY_MMDB); + } + return url.toURI().getPath(); } @Test @@ -82,18 +87,6 @@ public void trimFieldValueBeforeLookup() { assertNotNull(resolver.getIpFromFieldValue(ip)); } - @Test - public void extractGeoLocationInformation() { - - final GeoIpResolverEngine resolver = new GeoIpResolverEngine(config, metricRegistry); - - //TODO: update these tests to pass InetAddress--use separate test to test Object-to-InetAddress converter - //assertTrue("Should extract geo location information from public addresses", resolver.extractGeoLocationInformation("1.2.3.4").isPresent()); - //assertFalse("Should not extract geo location information from private addresses", resolver.extractGeoLocationInformation("192.168.0.1").isPresent()); - //assertFalse("Should not extract geo location information numeric fields", resolver.extractGeoLocationInformation(42).isPresent()); - assertTrue("Should extract geo location information IP address fields", resolver.extractGeoLocationInformation(InetAddresses.forString("1.2.3.4")).isPresent()); - } - @Test public void disabledFilterTest() { final GeoIpResolverEngine resolver = new GeoIpResolverEngine(config.toBuilder().enabled(false).build(), metricRegistry); diff --git a/graylog2-server/src/test/java/org/graylog/plugins/map/geoip/GeoIpResolverFactoryTest.java b/graylog2-server/src/test/java/org/graylog/plugins/map/geoip/GeoIpResolverFactoryTest.java new file mode 100644 index 000000000000..41e52e1805ac --- /dev/null +++ b/graylog2-server/src/test/java/org/graylog/plugins/map/geoip/GeoIpResolverFactoryTest.java @@ -0,0 +1,86 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ + +package org.graylog.plugins.map.geoip; + +import com.codahale.metrics.MetricFilter; +import com.codahale.metrics.MetricRegistry; +import com.codahale.metrics.Timer; +import org.graylog.plugins.map.config.DatabaseVendorType; +import org.graylog.plugins.map.config.GeoIpResolverConfig; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Test that the factory creates the appropriate {@link GeoIpResolver}. Each resolver is expected to fail to create a data provider, but should + * succeed in creating a disabled instance. + */ +class GeoIpResolverFactoryTest { + + private MetricRegistry metricRegistry; + private Timer timer; + + @BeforeEach + void setup() { + metricRegistry = new MetricRegistry(); + timer = metricRegistry.timer("ResolverFactoryUnitTest"); + } + + @AfterEach + void tearDown() { + + metricRegistry.removeMatching(MetricFilter.ALL); + metricRegistry = null; + } + + @Test + void testMaxMindVendor() { + + GeoIpResolverConfig config = createConfig(DatabaseVendorType.MAXMIND); + + GeoIpResolverFactory factory = new GeoIpResolverFactory(config, timer); + GeoIpResolver cityResolver = factory.createIpCityResolver(); + Assertions.assertTrue(cityResolver instanceof MaxMindIpLocationResolver); + + GeoIpResolver asnResolver = factory.createIpAsnResolver(); + Assertions.assertTrue(asnResolver instanceof MaxMindIpAsnResolver); + } + + @Test + void testIpInfoVendor() { + + GeoIpResolverConfig config = createConfig(DatabaseVendorType.IPINFO); + + GeoIpResolverFactory factory = new GeoIpResolverFactory(config, timer); + GeoIpResolver cityResolver = factory.createIpCityResolver(); + Assertions.assertTrue(cityResolver instanceof IpInfoLocationResolver); + + GeoIpResolver asnResolver = factory.createIpAsnResolver(); + Assertions.assertTrue(asnResolver instanceof IpInfoIpAsnResolver); + } + + private GeoIpResolverConfig createConfig(DatabaseVendorType vendorType) { + return GeoIpResolverConfig.defaultConfig().toBuilder() + .enabled(true) + .databaseVendorType(vendorType) + .cityDbPath("") + .asnDbPath("") + .build(); + } +} From 76b96cc4a497453237124d95e65e3c9ce8d692c6 Mon Sep 17 00:00:00 2001 From: Roberto Benitez Date: Wed, 22 Dec 2021 12:03:16 -0600 Subject: [PATCH 04/28] Updated ClusterConfigResource to validate GeoIpResolverConfig updates. --- .../system/ClusterConfigResource.java | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/graylog2-server/src/main/java/org/graylog2/rest/resources/system/ClusterConfigResource.java b/graylog2-server/src/main/java/org/graylog2/rest/resources/system/ClusterConfigResource.java index e0e454d9dc99..62ac8865b0b8 100644 --- a/graylog2-server/src/main/java/org/graylog2/rest/resources/system/ClusterConfigResource.java +++ b/graylog2-server/src/main/java/org/graylog2/rest/resources/system/ClusterConfigResource.java @@ -16,6 +16,8 @@ */ package org.graylog2.rest.resources.system; +import com.codahale.metrics.Timer; +import com.codahale.metrics.UniformReservoir; import com.codahale.metrics.annotation.Timed; import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.ObjectMapper; @@ -26,6 +28,9 @@ import io.swagger.annotations.ApiParam; import org.apache.shiro.authz.annotation.RequiresAuthentication; import org.apache.shiro.authz.annotation.RequiresPermissions; +import org.graylog.plugins.map.config.GeoIpResolverConfig; +import org.graylog.plugins.map.geoip.GeoIpResolver; +import org.graylog.plugins.map.geoip.GeoIpResolverFactory; import org.graylog2.audit.AuditEventTypes; import org.graylog2.audit.jersey.AuditEvent; import org.graylog2.plugin.cluster.ClusterConfigService; @@ -55,6 +60,7 @@ import javax.ws.rs.core.Response; import java.io.IOException; import java.io.InputStream; +import java.util.Locale; import java.util.Set; import static java.util.Objects.requireNonNull; @@ -129,6 +135,8 @@ public Response update(@ApiParam(name = "configClass", value = "The name of the throw new BadRequestException(msg); } + validateIfGeoIpResolverConfig(o); + try { clusterConfigService.write(o); } catch (Exception e) { @@ -140,6 +148,29 @@ public Response update(@ApiParam(name = "configClass", value = "The name of the return Response.accepted(o).build(); } + private void validateIfGeoIpResolverConfig(Object o) { + if (o instanceof GeoIpResolverConfig) { + final GeoIpResolverConfig config = (GeoIpResolverConfig) o; + GeoIpResolverFactory factory = new GeoIpResolverFactory(config, new Timer(new UniformReservoir())); + try { + GeoIpResolver cityResolver = factory.createIpCityResolver(); + if (!cityResolver.isEnabled()) { + String msg = String.format(Locale.ENGLISH, "Invalid '%s' City Geo IP database file '%s'. Make sure the file exists and is valid for '%1$s'", config.databaseVendorType(), config.cityDbPath()); + throw new IllegalArgumentException(msg); + } + + GeoIpResolver asnResolver = factory.createIpAsnResolver(); + if (!asnResolver.isEnabled()) { + String msg = String.format(Locale.ENGLISH, "Invalid '%s' ASN database file '%s'. Make sure the file exists and is valid for '%1$s'", config.databaseVendorType(), config.asnDbPath()); + throw new IllegalArgumentException(msg); + } + } catch (IllegalArgumentException e) { + LOG.error(e.getMessage(), e); + throw new BadRequestException(e.getMessage()); + } + } + } + @DELETE @Path("{configClass}") @ApiOperation(value = "Delete configuration settings from database") From a4eaeb0bb5e7294a13175191d967496de70a7247 Mon Sep 17 00:00:00 2001 From: Roberto Benitez Date: Mon, 3 Jan 2022 08:04:31 -0600 Subject: [PATCH 05/28] Updated resolvers to use timer in try-catch as the original --- .../java/org/graylog/plugins/map/geoip/IpInfoIpAsnResolver.java | 2 +- .../org/graylog/plugins/map/geoip/IpInfoLocationResolver.java | 2 +- .../org/graylog/plugins/map/geoip/MaxMindIpAsnResolver.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/IpInfoIpAsnResolver.java b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/IpInfoIpAsnResolver.java index cb0ed061e2a6..0538cb4e31b5 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/IpInfoIpAsnResolver.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/IpInfoIpAsnResolver.java @@ -35,7 +35,7 @@ public IpInfoIpAsnResolver(Timer timer, String configPath, boolean enabled) { protected Optional doGetGeoIpData(InetAddress address) { GeoAsnInformation info; - try { + try (Timer.Context ignored = resolveTime.time()) { final IPinfoASN ipInfoASN = adapter.ipInfoASN(address); info = GeoAsnInformation.create(ipInfoASN.name(), ipInfoASN.type(), ipInfoASN.asn()); } catch (Exception e) { diff --git a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/IpInfoLocationResolver.java b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/IpInfoLocationResolver.java index 312738140b9b..03615f2e525a 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/IpInfoLocationResolver.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/IpInfoLocationResolver.java @@ -36,7 +36,7 @@ public IpInfoLocationResolver(Timer resolveTime, String configPath, boolean enab protected Optional doGetGeoIpData(InetAddress address) { GeoLocationInformation info; - try { + try (Timer.Context ignored = resolveTime.time()) { IPinfoStandardLocation loc = adapter.ipInfoStandardLocation(address); info = GeoLocationInformation.create(loc.latitude(), loc.longitude(), loc.country(), loc.city(), loc.region(), loc.timezone()); diff --git a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/MaxMindIpAsnResolver.java b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/MaxMindIpAsnResolver.java index 0f73adf2a60b..e307701d024c 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/MaxMindIpAsnResolver.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/MaxMindIpAsnResolver.java @@ -38,7 +38,7 @@ public MaxMindIpAsnResolver(Timer resolveTime, String configPath, boolean enable @Override protected Optional doGetGeoIpData(InetAddress address) { GeoAsnInformation asn; - try { + try (Timer.Context ignored = resolveTime.time()) { AsnResponse response = databaseReader.asn(address); String number = response.getAutonomousSystemNumber() == null ? "N/A" : response.getAutonomousSystemNumber().toString(); asn = GeoAsnInformation.create(response.getAutonomousSystemOrganization(), "N/A", number); From f5839e85a95359537ef203bbc44e46193897f475 Mon Sep 17 00:00:00 2001 From: Roberto Benitez Date: Mon, 3 Jan 2022 10:16:11 -0600 Subject: [PATCH 06/28] Started Guice factory conversion for GeoIpResolverFactory --- .../graylog/plugins/map/MapWidgetModule.java | 15 +++++ .../map/geoip/GeoIpResolverEngine.java | 9 ++- .../map/geoip/GeoIpResolverFactory.java | 63 +++---------------- .../map/geoip/GeoIpVendorResolverService.java | 60 ++++++++++++++++++ .../map/geoip/IpInfoIpAsnResolver.java | 2 + .../map/geoip/IpInfoLocationResolver.java | 2 + .../map/geoip/MaxMindIpAsnResolver.java | 2 + .../map/geoip/MaxMindIpLocationResolver.java | 2 + .../map/geoip/processor/GeoIpProcessor.java | 11 +++- .../system/ClusterConfigResource.java | 18 +++--- .../map/geoip/GeoIpResolverEngineTest.java | 16 +++-- .../map/geoip/GeoIpResolverFactoryTest.java | 19 +++--- 12 files changed, 137 insertions(+), 82 deletions(-) create mode 100644 graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpVendorResolverService.java diff --git a/graylog2-server/src/main/java/org/graylog/plugins/map/MapWidgetModule.java b/graylog2-server/src/main/java/org/graylog/plugins/map/MapWidgetModule.java index 81312f1c90d0..7af079cfd390 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/map/MapWidgetModule.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/map/MapWidgetModule.java @@ -16,6 +16,14 @@ */ package org.graylog.plugins.map; +import com.google.inject.assistedinject.FactoryModuleBuilder; +import com.google.inject.name.Names; +import org.graylog.plugins.map.geoip.GeoIpResolver; +import org.graylog.plugins.map.geoip.GeoIpResolverFactory; +import org.graylog.plugins.map.geoip.IpInfoIpAsnResolver; +import org.graylog.plugins.map.geoip.IpInfoLocationResolver; +import org.graylog.plugins.map.geoip.MaxMindIpAsnResolver; +import org.graylog.plugins.map.geoip.MaxMindIpLocationResolver; import org.graylog.plugins.map.geoip.MaxmindDataAdapter; import org.graylog.plugins.map.geoip.processor.GeoIpProcessor; import org.graylog2.plugin.PluginModule; @@ -28,5 +36,12 @@ protected void configure() { MaxmindDataAdapter.class, MaxmindDataAdapter.Factory.class, MaxmindDataAdapter.Config.class); + + install(new FactoryModuleBuilder() + .implement(GeoIpResolver.class, Names.named("MAXMIND_CITY"), MaxMindIpLocationResolver.class) + .implement(GeoIpResolver.class, Names.named("MAXMIND_ASN"), MaxMindIpAsnResolver.class) + .implement(GeoIpResolver.class, Names.named("IPINFO_CITY"), IpInfoLocationResolver.class) + .implement(GeoIpResolver.class, Names.named("IPINFO_ASN"), IpInfoIpAsnResolver.class) + .build(GeoIpResolverFactory.class)); } } diff --git a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpResolverEngine.java b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpResolverEngine.java index 01d5e7c6cd0a..9ffb7c0e002c 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpResolverEngine.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpResolverEngine.java @@ -53,14 +53,13 @@ public class GeoIpResolverEngine { private boolean enabled; - public GeoIpResolverEngine(GeoIpResolverConfig config, MetricRegistry metricRegistry) { + public GeoIpResolverEngine(GeoIpVendorResolverService resolverService, GeoIpResolverConfig config, MetricRegistry metricRegistry) { Timer resolveTime = metricRegistry.timer(name(GeoIpResolverEngine.class, "resolveTime")); - GeoIpResolverFactory resolverFactory = new GeoIpResolverFactory(config, resolveTime); - ipLocationResolver = resolverFactory.createIpCityResolver(); - ipAsnResolver = resolverFactory.createIpAsnResolver(); + ipLocationResolver = resolverService.createCityResolver(config, resolveTime); + ipAsnResolver = resolverService.createAsnResolver(config, resolveTime); - LOG.info("Created Geo IP Resolvers for '{}'", resolverFactory.getDatabaseVendorType()); + LOG.info("Created Geo IP Resolvers for '{}'", config.databaseVendorType()); LOG.info("'{}' Status Enabled: {}", ipLocationResolver.getClass().getSimpleName(), ipLocationResolver.isEnabled()); LOG.info("'{}' Status Enabled: {}", ipAsnResolver.getClass().getSimpleName(), ipAsnResolver.isEnabled()); diff --git a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpResolverFactory.java b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpResolverFactory.java index 1569a26bb03b..61504136acdb 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpResolverFactory.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpResolverFactory.java @@ -18,69 +18,26 @@ package org.graylog.plugins.map.geoip; import com.codahale.metrics.Timer; -import org.graylog.plugins.map.config.DatabaseType; import org.graylog.plugins.map.config.DatabaseVendorType; import org.graylog.plugins.map.config.GeoIpResolverConfig; -import java.util.Locale; -import java.util.Objects; +import javax.inject.Named; /** * A factory to create ASN and Location {@link GeoIpResolver} resolvers based on the {@link DatabaseVendorType} contained in * the current {@link GeoIpResolverConfig}. */ -public class GeoIpResolverFactory { +public interface GeoIpResolverFactory { - private final GeoIpResolverConfig config; - private final Timer resolveTime; + @Named("MAXMIND_CITY") + GeoIpResolver createMaxMindCityResolver(Timer resolveTime, String configPath, boolean enabled); - public GeoIpResolverFactory(GeoIpResolverConfig config, Timer resolveTime) { - this.config = Objects.requireNonNull(config, "Configuration must not be null"); - this.resolveTime = resolveTime; - } + @Named("MAXMIND_ASN") + GeoIpResolver createMaxMindAsnResolver(Timer resolveTime, String configPath, boolean enabled); - public DatabaseVendorType getDatabaseVendorType() { - return config.databaseVendorType(); - } + @Named("IPINFO_CITY") + GeoIpResolver createIpInfoCityResolver(Timer resolveTime, String configPath, boolean enabled); - public GeoIpResolver createIpCityResolver() { - - final GeoIpResolver resolver; - DatabaseType dbType = config.databaseVendorType().getCityDbType(); - switch (dbType) { - case MAXMIND_CITY: - resolver = new MaxMindIpLocationResolver(resolveTime, config.cityDbPath(), config.enabled()); - break; - case IPINFO_STANDARD_LOCATION: - resolver = new IpInfoLocationResolver(resolveTime, config.cityDbPath(), config.enabled()); - break; - default: - String opts = String.join(",", DatabaseType.MAXMIND_CITY.name(), DatabaseType.IPINFO_STANDARD_LOCATION.name()); - String error = String.format(Locale.US, "'%s' is not a valid DatabaseType for a City Geo IP Resolver. Valid options are: %s", dbType, opts); - throw new IllegalArgumentException(error); - } - - return resolver; - } - - public GeoIpResolver createIpAsnResolver() { - - final GeoIpResolver resolver; - - final DatabaseType dbType = config.databaseVendorType().getAsnDbType(); - switch (dbType) { - case IPINFO_ASN: - resolver = new IpInfoIpAsnResolver(resolveTime, config.asnDbPath(), config.enabled()); - break; - case MAXMIND_ASN: - resolver = new MaxMindIpAsnResolver(resolveTime, config.asnDbPath(), config.enabled()); - break; - default: - String opts = String.join(",", DatabaseType.MAXMIND_ASN.name(), DatabaseType.IPINFO_ASN.name()); - String error = String.format(Locale.US, "'%s' is not a valid DatabaseType for an ASN Geo IP Resolver. Valid options are: %s", dbType, opts); - throw new IllegalArgumentException(error); - } - - return resolver; - } + @Named("IPINFO_ASN") + GeoIpResolver createIpInfoAsnResolver(Timer resolveTime, String configPath, boolean enabled); } diff --git a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpVendorResolverService.java b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpVendorResolverService.java new file mode 100644 index 000000000000..3c4cda1e5b8d --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpVendorResolverService.java @@ -0,0 +1,60 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ + +package org.graylog.plugins.map.geoip; + +import com.codahale.metrics.Timer; +import org.graylog.plugins.map.config.DatabaseVendorType; +import org.graylog.plugins.map.config.GeoIpResolverConfig; + +import javax.inject.Inject; + +/** + * A service to create a {@link GeoIpResolver} for a given configuration file and {@link DatabaseVendorType}. + */ +public class GeoIpVendorResolverService { + private final GeoIpResolverFactory resolverFactory; + + @Inject + public GeoIpVendorResolverService(GeoIpResolverFactory resolverFactory) { + this.resolverFactory = resolverFactory; + } + + public GeoIpResolver createCityResolver(GeoIpResolverConfig config, Timer timer) { + + switch (config.databaseVendorType()) { + case IPINFO: + return resolverFactory.createIpInfoCityResolver(timer, config.cityDbPath(), config.enabled()); + case MAXMIND: + return resolverFactory.createMaxMindCityResolver(timer, config.cityDbPath(), config.enabled()); + default: + throw new IllegalArgumentException(config.databaseVendorType().name()); + } + } + + public GeoIpResolver createAsnResolver(GeoIpResolverConfig config, Timer timer) { + + switch (config.databaseVendorType()) { + case IPINFO: + return resolverFactory.createIpInfoAsnResolver(timer, config.asnDbPath(), config.enabled()); + case MAXMIND: + return resolverFactory.createMaxMindAsnResolver(timer, config.asnDbPath(), config.enabled()); + default: + throw new IllegalArgumentException(config.databaseVendorType().name()); + } + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/IpInfoIpAsnResolver.java b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/IpInfoIpAsnResolver.java index 0538cb4e31b5..a41e27e04b3d 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/IpInfoIpAsnResolver.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/IpInfoIpAsnResolver.java @@ -19,6 +19,7 @@ import com.codahale.metrics.Timer; +import javax.inject.Inject; import java.net.InetAddress; import java.util.Optional; @@ -27,6 +28,7 @@ */ public class IpInfoIpAsnResolver extends IpInfoIpResolver { + @Inject public IpInfoIpAsnResolver(Timer timer, String configPath, boolean enabled) { super(timer, configPath, enabled); } diff --git a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/IpInfoLocationResolver.java b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/IpInfoLocationResolver.java index 03615f2e525a..6b2f9f5e9348 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/IpInfoLocationResolver.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/IpInfoLocationResolver.java @@ -19,6 +19,7 @@ import com.codahale.metrics.Timer; +import javax.inject.Inject; import java.net.InetAddress; import java.util.Locale; import java.util.Optional; @@ -28,6 +29,7 @@ */ public class IpInfoLocationResolver extends IpInfoIpResolver { + @Inject public IpInfoLocationResolver(Timer resolveTime, String configPath, boolean enabled) { super(resolveTime, configPath, enabled); } diff --git a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/MaxMindIpAsnResolver.java b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/MaxMindIpAsnResolver.java index e307701d024c..1584b13aef66 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/MaxMindIpAsnResolver.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/MaxMindIpAsnResolver.java @@ -21,6 +21,7 @@ import com.maxmind.geoip2.exception.GeoIp2Exception; import com.maxmind.geoip2.model.AsnResponse; +import javax.inject.Inject; import java.io.IOException; import java.net.InetAddress; import java.util.Locale; @@ -31,6 +32,7 @@ */ public class MaxMindIpAsnResolver extends MaxMindIpResolver { + @Inject public MaxMindIpAsnResolver(Timer resolveTime, String configPath, boolean enabled) { super(resolveTime, configPath, enabled); } diff --git a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/MaxMindIpLocationResolver.java b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/MaxMindIpLocationResolver.java index 71b764e3bdaf..3cd022c805a4 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/MaxMindIpLocationResolver.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/MaxMindIpLocationResolver.java @@ -23,6 +23,7 @@ import com.maxmind.geoip2.record.Country; import com.maxmind.geoip2.record.Location; +import javax.inject.Inject; import java.net.InetAddress; import java.util.Optional; @@ -31,6 +32,7 @@ */ public class MaxMindIpLocationResolver extends MaxMindIpResolver { + @Inject public MaxMindIpLocationResolver(Timer resolveTime, String configPath, boolean enabled) { super(resolveTime, configPath, enabled); } diff --git a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/processor/GeoIpProcessor.java b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/processor/GeoIpProcessor.java index 648af4569d75..cacad4bf6dbf 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/processor/GeoIpProcessor.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/processor/GeoIpProcessor.java @@ -21,6 +21,7 @@ import com.google.common.eventbus.Subscribe; import org.graylog.plugins.map.config.GeoIpResolverConfig; import org.graylog.plugins.map.geoip.GeoIpResolverEngine; +import org.graylog.plugins.map.geoip.GeoIpVendorResolverService; import org.graylog2.cluster.ClusterConfigChangedEvent; import org.graylog2.plugin.Message; import org.graylog2.plugin.Messages; @@ -53,6 +54,7 @@ public String className() { private final ClusterConfigService clusterConfigService; private final ScheduledExecutorService scheduler; private final MetricRegistry metricRegistry; + private final GeoIpVendorResolverService geoIpVendorResolverService; private final AtomicReference config; private final AtomicReference filterEngine; @@ -61,15 +63,18 @@ public String className() { public GeoIpProcessor(ClusterConfigService clusterConfigService, @Named("daemonScheduler") ScheduledExecutorService scheduler, EventBus eventBus, - MetricRegistry metricRegistry) { + MetricRegistry metricRegistry, + GeoIpVendorResolverService geoIpVendorResolverService) { this.clusterConfigService = clusterConfigService; this.scheduler = scheduler; this.metricRegistry = metricRegistry; + this.geoIpVendorResolverService = geoIpVendorResolverService; + final GeoIpResolverConfig config = clusterConfigService.getOrDefault(GeoIpResolverConfig.class, GeoIpResolverConfig.defaultConfig()); this.config = new AtomicReference<>(config); - this.filterEngine = new AtomicReference<>(new GeoIpResolverEngine(config, metricRegistry)); + this.filterEngine = new AtomicReference<>(new GeoIpResolverEngine(geoIpVendorResolverService, config, metricRegistry)); eventBus.register(this); } @@ -99,6 +104,6 @@ private void reload() { LOG.info("Updating GeoIP resolver engine - {}", newConfig); config.set(newConfig); - filterEngine.set(new GeoIpResolverEngine(newConfig, metricRegistry)); + filterEngine.set(new GeoIpResolverEngine(geoIpVendorResolverService, newConfig, metricRegistry)); } } diff --git a/graylog2-server/src/main/java/org/graylog2/rest/resources/system/ClusterConfigResource.java b/graylog2-server/src/main/java/org/graylog2/rest/resources/system/ClusterConfigResource.java index 62ac8865b0b8..5c054ef8db3a 100644 --- a/graylog2-server/src/main/java/org/graylog2/rest/resources/system/ClusterConfigResource.java +++ b/graylog2-server/src/main/java/org/graylog2/rest/resources/system/ClusterConfigResource.java @@ -30,7 +30,7 @@ import org.apache.shiro.authz.annotation.RequiresPermissions; import org.graylog.plugins.map.config.GeoIpResolverConfig; import org.graylog.plugins.map.geoip.GeoIpResolver; -import org.graylog.plugins.map.geoip.GeoIpResolverFactory; +import org.graylog.plugins.map.geoip.GeoIpVendorResolverService; import org.graylog2.audit.AuditEventTypes; import org.graylog2.audit.jersey.AuditEvent; import org.graylog2.plugin.cluster.ClusterConfigService; @@ -75,14 +75,17 @@ public class ClusterConfigResource extends RestResource { private final ClusterConfigService clusterConfigService; private final ChainingClassLoader chainingClassLoader; private final ObjectMapper objectMapper; + private final GeoIpVendorResolverService geoIpVendorResolverService; @Inject public ClusterConfigResource(ClusterConfigService clusterConfigService, ChainingClassLoader chainingClassLoader, - ObjectMapper objectMapper) { + ObjectMapper objectMapper, + GeoIpVendorResolverService geoIpVendorResolverService) { this.clusterConfigService = requireNonNull(clusterConfigService); this.chainingClassLoader = chainingClassLoader; this.objectMapper = objectMapper; + this.geoIpVendorResolverService = geoIpVendorResolverService; } @GET @@ -151,16 +154,17 @@ public Response update(@ApiParam(name = "configClass", value = "The name of the private void validateIfGeoIpResolverConfig(Object o) { if (o instanceof GeoIpResolverConfig) { final GeoIpResolverConfig config = (GeoIpResolverConfig) o; - GeoIpResolverFactory factory = new GeoIpResolverFactory(config, new Timer(new UniformReservoir())); + + Timer timer = new Timer(new UniformReservoir()); try { - GeoIpResolver cityResolver = factory.createIpCityResolver(); - if (!cityResolver.isEnabled()) { + GeoIpResolver cityResolver = geoIpVendorResolverService.createCityResolver(config, timer); + if (config.enabled() && !cityResolver.isEnabled()) { String msg = String.format(Locale.ENGLISH, "Invalid '%s' City Geo IP database file '%s'. Make sure the file exists and is valid for '%1$s'", config.databaseVendorType(), config.cityDbPath()); throw new IllegalArgumentException(msg); } - GeoIpResolver asnResolver = factory.createIpAsnResolver(); - if (!asnResolver.isEnabled()) { + GeoIpResolver asnResolver = geoIpVendorResolverService.createAsnResolver(config, timer); + if (config.enabled() && !asnResolver.isEnabled()) { String msg = String.format(Locale.ENGLISH, "Invalid '%s' ASN database file '%s'. Make sure the file exists and is valid for '%1$s'", config.databaseVendorType(), config.asnDbPath()); throw new IllegalArgumentException(msg); } diff --git a/graylog2-server/src/test/java/org/graylog/plugins/map/geoip/GeoIpResolverEngineTest.java b/graylog2-server/src/test/java/org/graylog/plugins/map/geoip/GeoIpResolverEngineTest.java index 8a0084f085a4..eb8af41a003f 100644 --- a/graylog2-server/src/test/java/org/graylog/plugins/map/geoip/GeoIpResolverEngineTest.java +++ b/graylog2-server/src/test/java/org/graylog/plugins/map/geoip/GeoIpResolverEngineTest.java @@ -29,6 +29,8 @@ import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; import java.net.URISyntaxException; import java.net.URL; @@ -46,11 +48,15 @@ public class GeoIpResolverEngineTest { static final String GEO_LITE2_CITY_MMDB = "/GeoLite2-City.mmdb"; + @Mock + GeoIpVendorResolverService geoIpVendorResolverService; private MetricRegistry metricRegistry; private GeoIpResolverConfig config; @Before - public void setUp() throws URISyntaxException { + public void setUp() throws Exception { + + MockitoAnnotations.openMocks(this).close(); config = GeoIpResolverConfig.defaultConfig().toBuilder().enabled(true).cityDbPath(this.getTestDatabasePath()).build(); metricRegistry = new MetricRegistry(); } @@ -71,7 +77,7 @@ private String getTestDatabasePath() throws URISyntaxException { @Test public void getIpFromFieldValue() { - final GeoIpResolverEngine resolver = new GeoIpResolverEngine(config, metricRegistry); + final GeoIpResolverEngine resolver = new GeoIpResolverEngine(geoIpVendorResolverService, config, metricRegistry); final String ip = "127.0.0.1"; assertEquals(InetAddresses.forString(ip), resolver.getIpFromFieldValue(ip)); @@ -81,7 +87,7 @@ public void getIpFromFieldValue() { @Test public void trimFieldValueBeforeLookup() { - final GeoIpResolverEngine resolver = new GeoIpResolverEngine(config, metricRegistry); + final GeoIpResolverEngine resolver = new GeoIpResolverEngine(geoIpVendorResolverService, config, metricRegistry); final String ip = " 2001:4860:4860::8888\t\n"; assertNotNull(resolver.getIpFromFieldValue(ip)); @@ -89,7 +95,7 @@ public void trimFieldValueBeforeLookup() { @Test public void disabledFilterTest() { - final GeoIpResolverEngine resolver = new GeoIpResolverEngine(config.toBuilder().enabled(false).build(), metricRegistry); + final GeoIpResolverEngine resolver = new GeoIpResolverEngine(geoIpVendorResolverService, config.toBuilder().enabled(false).build(), metricRegistry); final Map messageFields = Maps.newHashMap(); messageFields.put("_id", (new UUID()).toString()); @@ -120,7 +126,7 @@ private void assertFieldResolved(Message message, String fieldName, String error @Test public void filterResolvesIpGeoLocation() { - final GeoIpResolverEngine resolver = new GeoIpResolverEngine(config, metricRegistry); + final GeoIpResolverEngine resolver = new GeoIpResolverEngine(geoIpVendorResolverService, config, metricRegistry); final Map messageFields = Maps.newHashMap(); messageFields.put("_id", (new UUID()).toString()); diff --git a/graylog2-server/src/test/java/org/graylog/plugins/map/geoip/GeoIpResolverFactoryTest.java b/graylog2-server/src/test/java/org/graylog/plugins/map/geoip/GeoIpResolverFactoryTest.java index 41e52e1805ac..c9b935fc6d9d 100644 --- a/graylog2-server/src/test/java/org/graylog/plugins/map/geoip/GeoIpResolverFactoryTest.java +++ b/graylog2-server/src/test/java/org/graylog/plugins/map/geoip/GeoIpResolverFactoryTest.java @@ -25,21 +25,26 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; +import org.mockito.Mock; /** * Test that the factory creates the appropriate {@link GeoIpResolver}. Each resolver is expected to fail to create a data provider, but should * succeed in creating a disabled instance. */ + class GeoIpResolverFactoryTest { + + @Mock private MetricRegistry metricRegistry; private Timer timer; + private GeoIpResolverFactory geoIpResolverFactory; @BeforeEach void setup() { metricRegistry = new MetricRegistry(); timer = metricRegistry.timer("ResolverFactoryUnitTest"); + } @AfterEach @@ -49,29 +54,25 @@ void tearDown() { metricRegistry = null; } - @Test void testMaxMindVendor() { GeoIpResolverConfig config = createConfig(DatabaseVendorType.MAXMIND); - GeoIpResolverFactory factory = new GeoIpResolverFactory(config, timer); - GeoIpResolver cityResolver = factory.createIpCityResolver(); + GeoIpResolver cityResolver = geoIpResolverFactory.createIpInfoCityResolver(timer, config.cityDbPath(), config.enabled()); Assertions.assertTrue(cityResolver instanceof MaxMindIpLocationResolver); - GeoIpResolver asnResolver = factory.createIpAsnResolver(); + GeoIpResolver asnResolver = geoIpResolverFactory.createIpInfoCityResolver(timer, config.asnDbPath(), config.enabled()); Assertions.assertTrue(asnResolver instanceof MaxMindIpAsnResolver); } - @Test void testIpInfoVendor() { GeoIpResolverConfig config = createConfig(DatabaseVendorType.IPINFO); - GeoIpResolverFactory factory = new GeoIpResolverFactory(config, timer); - GeoIpResolver cityResolver = factory.createIpCityResolver(); + GeoIpResolver cityResolver = geoIpResolverFactory.createIpInfoCityResolver(timer, config.cityDbPath(), config.enabled()); Assertions.assertTrue(cityResolver instanceof IpInfoLocationResolver); - GeoIpResolver asnResolver = factory.createIpAsnResolver(); + GeoIpResolver asnResolver = geoIpResolverFactory.createIpInfoCityResolver(timer, config.asnDbPath(), config.enabled()); Assertions.assertTrue(asnResolver instanceof IpInfoIpAsnResolver); } From c33b69d714e1af5f1eba4cf997c8298a54e5018a Mon Sep 17 00:00:00 2001 From: Roberto Benitez Date: Mon, 3 Jan 2022 12:53:56 -0600 Subject: [PATCH 07/28] Updated resolver constructor args with Assisted annotations --- .../org/graylog/plugins/map/geoip/IpInfoIpAsnResolver.java | 5 ++++- .../graylog/plugins/map/geoip/IpInfoLocationResolver.java | 5 ++++- .../org/graylog/plugins/map/geoip/MaxMindIpAsnResolver.java | 5 ++++- .../graylog/plugins/map/geoip/MaxMindIpLocationResolver.java | 5 ++++- 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/IpInfoIpAsnResolver.java b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/IpInfoIpAsnResolver.java index a41e27e04b3d..3f18ff49f02d 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/IpInfoIpAsnResolver.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/IpInfoIpAsnResolver.java @@ -18,6 +18,7 @@ package org.graylog.plugins.map.geoip; import com.codahale.metrics.Timer; +import com.google.inject.assistedinject.Assisted; import javax.inject.Inject; import java.net.InetAddress; @@ -29,7 +30,9 @@ public class IpInfoIpAsnResolver extends IpInfoIpResolver { @Inject - public IpInfoIpAsnResolver(Timer timer, String configPath, boolean enabled) { + public IpInfoIpAsnResolver(@Assisted Timer timer, + @Assisted String configPath, + @Assisted boolean enabled) { super(timer, configPath, enabled); } diff --git a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/IpInfoLocationResolver.java b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/IpInfoLocationResolver.java index 6b2f9f5e9348..dadf64f35fa3 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/IpInfoLocationResolver.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/IpInfoLocationResolver.java @@ -18,6 +18,7 @@ package org.graylog.plugins.map.geoip; import com.codahale.metrics.Timer; +import com.google.inject.assistedinject.Assisted; import javax.inject.Inject; import java.net.InetAddress; @@ -30,7 +31,9 @@ public class IpInfoLocationResolver extends IpInfoIpResolver { @Inject - public IpInfoLocationResolver(Timer resolveTime, String configPath, boolean enabled) { + public IpInfoLocationResolver(@Assisted Timer resolveTime, + @Assisted String configPath, + @Assisted boolean enabled) { super(resolveTime, configPath, enabled); } diff --git a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/MaxMindIpAsnResolver.java b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/MaxMindIpAsnResolver.java index 1584b13aef66..0c512d7433f7 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/MaxMindIpAsnResolver.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/MaxMindIpAsnResolver.java @@ -18,6 +18,7 @@ package org.graylog.plugins.map.geoip; import com.codahale.metrics.Timer; +import com.google.inject.assistedinject.Assisted; import com.maxmind.geoip2.exception.GeoIp2Exception; import com.maxmind.geoip2.model.AsnResponse; @@ -33,7 +34,9 @@ public class MaxMindIpAsnResolver extends MaxMindIpResolver { @Inject - public MaxMindIpAsnResolver(Timer resolveTime, String configPath, boolean enabled) { + public MaxMindIpAsnResolver(@Assisted Timer resolveTime, + @Assisted String configPath, + @Assisted boolean enabled) { super(resolveTime, configPath, enabled); } diff --git a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/MaxMindIpLocationResolver.java b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/MaxMindIpLocationResolver.java index 3cd022c805a4..db28a1347ba9 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/MaxMindIpLocationResolver.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/MaxMindIpLocationResolver.java @@ -18,6 +18,7 @@ package org.graylog.plugins.map.geoip; import com.codahale.metrics.Timer; +import com.google.inject.assistedinject.Assisted; import com.maxmind.geoip2.model.CityResponse; import com.maxmind.geoip2.record.City; import com.maxmind.geoip2.record.Country; @@ -33,7 +34,9 @@ public class MaxMindIpLocationResolver extends MaxMindIpResolver { @Inject - public MaxMindIpLocationResolver(Timer resolveTime, String configPath, boolean enabled) { + public MaxMindIpLocationResolver(@Assisted Timer resolveTime, + @Assisted String configPath, + @Assisted boolean enabled) { super(resolveTime, configPath, enabled); } From 0619080da11d1a7cd0cf909323ec0da467fc5ab1 Mon Sep 17 00:00:00 2001 From: Roberto Benitez Date: Mon, 3 Jan 2022 15:03:32 -0600 Subject: [PATCH 08/28] Updated ClusterConfigResource to not validate ASN DB file path if not provided (i.e. this is optional). Updated GeoIpResolverEngine to check if IP address is reserved --- .../graylog/plugins/map/MapWidgetModule.java | 17 +++-- .../map/geoip/GeoIpResolverEngine.java | 64 +++++++++++++------ .../map/geoip/GeoIpResolverFactory.java | 8 +-- .../map/geoip/GeoIpVendorResolverService.java | 4 +- .../map/geoip/GeoLocationInformation.java | 2 +- .../system/ClusterConfigResource.java | 28 +++++--- .../map/geoip/GeoIpResolverFactoryTest.java | 4 +- 7 files changed, 86 insertions(+), 41 deletions(-) diff --git a/graylog2-server/src/main/java/org/graylog/plugins/map/MapWidgetModule.java b/graylog2-server/src/main/java/org/graylog/plugins/map/MapWidgetModule.java index 7af079cfd390..565bd3ad1855 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/map/MapWidgetModule.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/map/MapWidgetModule.java @@ -16,10 +16,13 @@ */ package org.graylog.plugins.map; +import com.google.inject.TypeLiteral; import com.google.inject.assistedinject.FactoryModuleBuilder; import com.google.inject.name.Names; +import org.graylog.plugins.map.geoip.GeoAsnInformation; import org.graylog.plugins.map.geoip.GeoIpResolver; import org.graylog.plugins.map.geoip.GeoIpResolverFactory; +import org.graylog.plugins.map.geoip.GeoLocationInformation; import org.graylog.plugins.map.geoip.IpInfoIpAsnResolver; import org.graylog.plugins.map.geoip.IpInfoLocationResolver; import org.graylog.plugins.map.geoip.MaxMindIpAsnResolver; @@ -37,11 +40,17 @@ protected void configure() { MaxmindDataAdapter.Factory.class, MaxmindDataAdapter.Config.class); + //Create TypeLiterals to specify method type parameters + TypeLiteral> mmCityTl = new TypeLiteral>() {}; + TypeLiteral> mmAsnTl = new TypeLiteral>() {}; + TypeLiteral> ipinfoCityTl = new TypeLiteral>() {}; + TypeLiteral> ipInfoAsnTl = new TypeLiteral>() {}; + install(new FactoryModuleBuilder() - .implement(GeoIpResolver.class, Names.named("MAXMIND_CITY"), MaxMindIpLocationResolver.class) - .implement(GeoIpResolver.class, Names.named("MAXMIND_ASN"), MaxMindIpAsnResolver.class) - .implement(GeoIpResolver.class, Names.named("IPINFO_CITY"), IpInfoLocationResolver.class) - .implement(GeoIpResolver.class, Names.named("IPINFO_ASN"), IpInfoIpAsnResolver.class) + .implement(mmCityTl, Names.named("MAXMIND_CITY"), MaxMindIpLocationResolver.class) + .implement(mmAsnTl, Names.named("MAXMIND_ASN"), MaxMindIpAsnResolver.class) + .implement(ipinfoCityTl, Names.named("IPINFO_CITY"), IpInfoLocationResolver.class) + .implement(ipInfoAsnTl, Names.named("IPINFO_ASN"), IpInfoIpAsnResolver.class) .build(GeoIpResolverFactory.class)); } } diff --git a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpResolverEngine.java b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpResolverEngine.java index 9ffb7c0e002c..9af26489a360 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpResolverEngine.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpResolverEngine.java @@ -21,8 +21,10 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableMap; import com.google.common.net.InetAddresses; +import org.apache.commons.lang3.StringUtils; import org.graylog.plugins.map.config.GeoIpResolverConfig; import org.graylog2.plugin.Message; +import org.graylog2.utilities.ReservedIpChecker; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -41,6 +43,12 @@ public class GeoIpResolverEngine { /** * A mapping of fields that to search that contain IP addresses. ONLY these fields will be checked * to see if they have valid Geo IP information. + * + *

+ * The mapping is field name -> new message field prefix, where field_name is the field name expected in the message + * which will be searched in the GeoIP database, and the new message field prefix is the prefix for the new field with the GeoIP data + * that will be inserted into the message. + *

*/ private final Map ipAddressFields = new ImmutableMap.Builder() .put("source_ip", "source") @@ -81,33 +89,40 @@ public boolean filter(Message message) { continue; } - //TODO: Tag any reserved IP addresses with 'reserved_ip: true' once ReservedIpChecker is merged - final String prefix = ipAddressFields.get(key); - ipLocationResolver.getGeoIpData(address).ifPresent(locationInformation -> { - message.addField(prefix + "_geo_coordinates", locationInformation.latitude() + "," + locationInformation.longitude()); - message.addField(prefix + "_geo_country", locationInformation.countryIsoCode()); - message.addField(prefix + "_geo_city", locationInformation.cityName()); - message.addField(prefix + "_geo_region", locationInformation.region()); - message.addField(prefix + "_geo_timeZone", locationInformation.timeZone()); - - if (!(locationInformation.countryIsoCode() == null || locationInformation.cityName() == null)) { - String name = String.format(Locale.ENGLISH, "%s, %s", locationInformation.cityName(), locationInformation.countryIsoCode()); - message.addField(prefix + "_geo_name", name); - } - }); - ipAsnResolver.getGeoIpData(address).ifPresent(info -> { - - message.addField(prefix + "_as_organization", info.organization()); - message.addField(prefix + "_as_number", info.asn()); - }); + if (ReservedIpChecker.getInstance().isReservedIpAddress(address.getHostAddress())) { + message.addField(prefix + "_reserved_ip", true); + } else { + addGeoIpDataIfPresent(message, address, prefix); + } } return true; } + private void addGeoIpDataIfPresent(Message message, InetAddress address, String newFieldPrefix) { + ipLocationResolver.getGeoIpData(address).ifPresent(locationInformation -> { + message.addField(newFieldPrefix + "_geo_coordinates", locationInformation.latitude() + "," + locationInformation.longitude()); + message.addField(newFieldPrefix + "_geo_country", locationInformation.countryIsoCode()); + message.addField(newFieldPrefix + "_geo_city", locationInformation.cityName()); + message.addField(newFieldPrefix + "_geo_region", locationInformation.region()); + message.addField(newFieldPrefix + "_geo_timeZone", locationInformation.timeZone()); + + if (areValidGeoNames(locationInformation.cityName(), locationInformation.countryIsoCode())) { + String name = String.format(Locale.ENGLISH, "%s, %s", locationInformation.cityName(), locationInformation.countryIsoCode()); + message.addField(newFieldPrefix + "_geo_name", name); + } + }); + + ipAsnResolver.getGeoIpData(address).ifPresent(info -> { + + message.addField(newFieldPrefix + "_as_organization", info.organization()); + message.addField(newFieldPrefix + "_as_number", info.asn()); + }); + } + private List getIpAddressFields(Message message) { return message.getFieldNames() .stream() @@ -128,6 +143,17 @@ private InetAddress getValidRoutableInetAddress(Object fieldValue) { return ipAddress; } + private boolean areValidGeoNames(String... names) { + + for (String name : names) { + if (StringUtils.isBlank(name) || "N/A".equalsIgnoreCase(name)) { + return false; + } + } + + return true; + } + @Nullable @VisibleForTesting InetAddress getIpFromFieldValue(String fieldValue) { diff --git a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpResolverFactory.java b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpResolverFactory.java index 61504136acdb..06b09850122b 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpResolverFactory.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpResolverFactory.java @@ -30,14 +30,14 @@ public interface GeoIpResolverFactory { @Named("MAXMIND_CITY") - GeoIpResolver createMaxMindCityResolver(Timer resolveTime, String configPath, boolean enabled); + GeoIpResolver createMaxMindCityResolver(Timer resolveTime, String configPath, boolean enabled); @Named("MAXMIND_ASN") - GeoIpResolver createMaxMindAsnResolver(Timer resolveTime, String configPath, boolean enabled); + GeoIpResolver createMaxMindAsnResolver(Timer resolveTime, String configPath, boolean enabled); @Named("IPINFO_CITY") - GeoIpResolver createIpInfoCityResolver(Timer resolveTime, String configPath, boolean enabled); + GeoIpResolver createIpInfoCityResolver(Timer resolveTime, String configPath, boolean enabled); @Named("IPINFO_ASN") - GeoIpResolver createIpInfoAsnResolver(Timer resolveTime, String configPath, boolean enabled); + GeoIpResolver createIpInfoAsnResolver(Timer resolveTime, String configPath, boolean enabled); } diff --git a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpVendorResolverService.java b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpVendorResolverService.java index 3c4cda1e5b8d..6ba1d6745597 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpVendorResolverService.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpVendorResolverService.java @@ -34,7 +34,7 @@ public GeoIpVendorResolverService(GeoIpResolverFactory resolverFactory) { this.resolverFactory = resolverFactory; } - public GeoIpResolver createCityResolver(GeoIpResolverConfig config, Timer timer) { + public GeoIpResolver createCityResolver(GeoIpResolverConfig config, Timer timer) { switch (config.databaseVendorType()) { case IPINFO: @@ -46,7 +46,7 @@ public GeoIpResolver createCityResolver(GeoIpResolverConfig config, Timer timer) } } - public GeoIpResolver createAsnResolver(GeoIpResolverConfig config, Timer timer) { + public GeoIpResolver createAsnResolver(GeoIpResolverConfig config, Timer timer) { switch (config.databaseVendorType()) { case IPINFO: diff --git a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoLocationInformation.java b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoLocationInformation.java index 5f68fca477fb..857732e2e2ee 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoLocationInformation.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoLocationInformation.java @@ -20,7 +20,7 @@ import com.google.auto.value.AutoValue; @AutoValue -abstract class GeoLocationInformation { +public abstract class GeoLocationInformation { public abstract double latitude(); public abstract double longitude(); diff --git a/graylog2-server/src/main/java/org/graylog2/rest/resources/system/ClusterConfigResource.java b/graylog2-server/src/main/java/org/graylog2/rest/resources/system/ClusterConfigResource.java index 5c054ef8db3a..8f250ef63de7 100644 --- a/graylog2-server/src/main/java/org/graylog2/rest/resources/system/ClusterConfigResource.java +++ b/graylog2-server/src/main/java/org/graylog2/rest/resources/system/ClusterConfigResource.java @@ -65,12 +65,13 @@ import static java.util.Objects.requireNonNull; -@Api(value = "System/ClusterConfig", description = "Graylog Cluster Configuration") +@Api(value = "System/ClusterConfig") @RequiresAuthentication @Path("/system/cluster_config") @Produces(MediaType.APPLICATION_JSON) public class ClusterConfigResource extends RestResource { private static final Logger LOG = LoggerFactory.getLogger(ClusterConfigResource.class); + public static final String NO_CLASS_MSG = "Couldn't find configuration class '%s'"; private final ClusterConfigService clusterConfigService; private final ChainingClassLoader chainingClassLoader; @@ -107,7 +108,8 @@ public Object read(@ApiParam(name = "configClass", value = "The name of the clus @PathParam("configClass") @NotBlank String configClass) { final Class cls = classFromName(configClass); if (cls == null) { - throw new NotFoundException("Couldn't find configuration class \"" + configClass + "\""); + String error = createNoClassMsg(configClass); + throw new NotFoundException(error); } return clusterConfigService.get(cls); @@ -126,7 +128,7 @@ public Response update(@ApiParam(name = "configClass", value = "The name of the @NotNull InputStream body) throws IOException { final Class cls = classFromName(configClass); if (cls == null) { - throw new NotFoundException("Couldn't find configuration class \"" + configClass + "\""); + throw new NotFoundException(createNoClassMsg(configClass)); } final Object o; @@ -163,10 +165,13 @@ private void validateIfGeoIpResolverConfig(Object o) { throw new IllegalArgumentException(msg); } - GeoIpResolver asnResolver = geoIpVendorResolverService.createAsnResolver(config, timer); - if (config.enabled() && !asnResolver.isEnabled()) { - String msg = String.format(Locale.ENGLISH, "Invalid '%s' ASN database file '%s'. Make sure the file exists and is valid for '%1$s'", config.databaseVendorType(), config.asnDbPath()); - throw new IllegalArgumentException(msg); + //ASN file is optional--do not validate if not provided. + if (!(config.asnDbPath() == null || config.asnDbPath().trim().isEmpty())) { + GeoIpResolver asnResolver = geoIpVendorResolverService.createAsnResolver(config, timer); + if (config.enabled() && !asnResolver.isEnabled()) { + String msg = String.format(Locale.ENGLISH, "Invalid '%s' ASN database file '%s'. Make sure the file exists and is valid for '%1$s'", config.databaseVendorType(), config.asnDbPath()); + throw new IllegalArgumentException(msg); + } } } catch (IllegalArgumentException e) { LOG.error(e.getMessage(), e); @@ -185,7 +190,7 @@ public void delete(@ApiParam(name = "configClass", value = "The name of the clus @PathParam("configClass") @NotBlank String configClass) { final Class cls = classFromName(configClass); if (cls == null) { - throw new NotFoundException("Couldn't find configuration class \"" + configClass + "\""); + throw new NotFoundException(createNoClassMsg(configClass)); } clusterConfigService.remove(cls); @@ -201,7 +206,7 @@ public JsonSchema schema(@ApiParam(name = "configClass", value = "The name of th @PathParam("configClass") @NotBlank String configClass) { final Class cls = classFromName(configClass); if (cls == null) { - throw new NotFoundException("Couldn't find configuration class \"" + configClass + "\""); + throw new NotFoundException(createNoClassMsg(configClass)); } final SchemaFactoryWrapper visitor = new SchemaFactoryWrapper(); @@ -222,4 +227,9 @@ private Class classFromName(String className) { return null; } } + + private static String createNoClassMsg(String configClass) { + return String.format(Locale.ENGLISH, NO_CLASS_MSG, configClass); + } + } diff --git a/graylog2-server/src/test/java/org/graylog/plugins/map/geoip/GeoIpResolverFactoryTest.java b/graylog2-server/src/test/java/org/graylog/plugins/map/geoip/GeoIpResolverFactoryTest.java index c9b935fc6d9d..4a8059729f56 100644 --- a/graylog2-server/src/test/java/org/graylog/plugins/map/geoip/GeoIpResolverFactoryTest.java +++ b/graylog2-server/src/test/java/org/graylog/plugins/map/geoip/GeoIpResolverFactoryTest.java @@ -61,7 +61,7 @@ void testMaxMindVendor() { GeoIpResolver cityResolver = geoIpResolverFactory.createIpInfoCityResolver(timer, config.cityDbPath(), config.enabled()); Assertions.assertTrue(cityResolver instanceof MaxMindIpLocationResolver); - GeoIpResolver asnResolver = geoIpResolverFactory.createIpInfoCityResolver(timer, config.asnDbPath(), config.enabled()); + GeoIpResolver asnResolver = geoIpResolverFactory.createIpInfoAsnResolver(timer, config.asnDbPath(), config.enabled()); Assertions.assertTrue(asnResolver instanceof MaxMindIpAsnResolver); } @@ -72,7 +72,7 @@ void testIpInfoVendor() { GeoIpResolver cityResolver = geoIpResolverFactory.createIpInfoCityResolver(timer, config.cityDbPath(), config.enabled()); Assertions.assertTrue(cityResolver instanceof IpInfoLocationResolver); - GeoIpResolver asnResolver = geoIpResolverFactory.createIpInfoCityResolver(timer, config.asnDbPath(), config.enabled()); + GeoIpResolver asnResolver = geoIpResolverFactory.createIpInfoAsnResolver(timer, config.asnDbPath(), config.enabled()); Assertions.assertTrue(asnResolver instanceof IpInfoIpAsnResolver); } From 877ef0382690b2d97aa29fd00de3d0f405f1360d Mon Sep 17 00:00:00 2001 From: Roberto Benitez Date: Mon, 3 Jan 2022 15:26:40 -0600 Subject: [PATCH 09/28] Started updates to validate database file (perform actual db query to confirm it is a valid db file). --- .../org/graylog/plugins/map/geoip/GeoIpResolver.java | 3 ++- .../rest/resources/system/ClusterConfigResource.java | 10 +++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpResolver.java b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpResolver.java index ab663120ed71..904b87080289 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpResolver.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpResolver.java @@ -30,6 +30,7 @@ public abstract class GeoIpResolver { private static final Logger LOG = LoggerFactory.getLogger(GeoIpResolver.class); + protected String lastError = null; protected final Timer resolveTime; private final boolean enabled; @@ -56,10 +57,10 @@ public boolean isEnabled() { abstract boolean createDataProvider(File configFile); public Optional getGeoIpData(InetAddress address) { + lastError = null; if (!enabled || address == null) { return Optional.empty(); } - return doGetGeoIpData(address); } diff --git a/graylog2-server/src/main/java/org/graylog2/rest/resources/system/ClusterConfigResource.java b/graylog2-server/src/main/java/org/graylog2/rest/resources/system/ClusterConfigResource.java index 8f250ef63de7..d42037d81c50 100644 --- a/graylog2-server/src/main/java/org/graylog2/rest/resources/system/ClusterConfigResource.java +++ b/graylog2-server/src/main/java/org/graylog2/rest/resources/system/ClusterConfigResource.java @@ -60,6 +60,9 @@ import javax.ws.rs.core.Response; import java.io.IOException; import java.io.InputStream; +import java.net.Inet4Address; +import java.net.InetAddress; +import java.net.UnknownHostException; import java.util.Locale; import java.util.Set; @@ -165,6 +168,10 @@ private void validateIfGeoIpResolverConfig(Object o) { throw new IllegalArgumentException(msg); } + InetAddress address = Inet4Address.getByName("127.0.0.1"); + + cityResolver.getGeoIpData(address); + //ASN file is optional--do not validate if not provided. if (!(config.asnDbPath() == null || config.asnDbPath().trim().isEmpty())) { GeoIpResolver asnResolver = geoIpVendorResolverService.createAsnResolver(config, timer); @@ -172,8 +179,9 @@ private void validateIfGeoIpResolverConfig(Object o) { String msg = String.format(Locale.ENGLISH, "Invalid '%s' ASN database file '%s'. Make sure the file exists and is valid for '%1$s'", config.databaseVendorType(), config.asnDbPath()); throw new IllegalArgumentException(msg); } + asnResolver.getGeoIpData(address); } - } catch (IllegalArgumentException e) { + } catch (UnknownHostException | IllegalArgumentException e) { LOG.error(e.getMessage(), e); throw new BadRequestException(e.getMessage()); } From 78bc2cc7bba4b0c567d5d36a409c87c7ebc24f51 Mon Sep 17 00:00:00 2001 From: Roberto Benitez Date: Mon, 3 Jan 2022 16:37:04 -0600 Subject: [PATCH 10/28] Added last error tracking for resolvers and updated config resource to do query validation for the selected db/resolver. --- .../plugins/map/geoip/GeoIpResolver.java | 9 +++ .../map/geoip/IpInfoIpAsnResolver.java | 10 +++- .../map/geoip/IpInfoLocationResolver.java | 11 +++- .../map/geoip/MaxMindIpAsnResolver.java | 9 ++- .../map/geoip/MaxMindIpLocationResolver.java | 10 +++- .../system/ClusterConfigResource.java | 60 +++++++++++++------ 6 files changed, 82 insertions(+), 27 deletions(-) diff --git a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpResolver.java b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpResolver.java index 904b87080289..630ca52ebd54 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpResolver.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpResolver.java @@ -64,5 +64,14 @@ public Optional getGeoIpData(InetAddress address) { return doGetGeoIpData(address); } + /** + * Get the last error, if any, produced after having called {@link #getGeoIpData(InetAddress)}. + * + * @return optional error message + */ + public Optional getLastError() { + return Optional.ofNullable(lastError); + } + protected abstract Optional doGetGeoIpData(InetAddress address); } diff --git a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/IpInfoIpAsnResolver.java b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/IpInfoIpAsnResolver.java index 3f18ff49f02d..0db87f3f78a5 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/IpInfoIpAsnResolver.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/IpInfoIpAsnResolver.java @@ -19,9 +19,12 @@ import com.codahale.metrics.Timer; import com.google.inject.assistedinject.Assisted; +import com.maxmind.geoip2.exception.AddressNotFoundException; import javax.inject.Inject; +import java.io.IOException; import java.net.InetAddress; +import java.util.Locale; import java.util.Optional; /** @@ -43,8 +46,13 @@ protected Optional doGetGeoIpData(InetAddress address) { try (Timer.Context ignored = resolveTime.time()) { final IPinfoASN ipInfoASN = adapter.ipInfoASN(address); info = GeoAsnInformation.create(ipInfoASN.name(), ipInfoASN.type(), ipInfoASN.asn()); - } catch (Exception e) { + } catch (IOException | AddressNotFoundException | UnsupportedOperationException e) { info = null; + if (e instanceof AddressNotFoundException == false) { + String error = String.format(Locale.US, "Error getting ASN for IP Address '%s'. %s", address, e.getMessage()); + LOG.warn(error, e); + lastError = e.getMessage(); + } } return Optional.ofNullable(info); } diff --git a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/IpInfoLocationResolver.java b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/IpInfoLocationResolver.java index dadf64f35fa3..218ba59af0b1 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/IpInfoLocationResolver.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/IpInfoLocationResolver.java @@ -19,8 +19,10 @@ import com.codahale.metrics.Timer; import com.google.inject.assistedinject.Assisted; +import com.maxmind.geoip2.exception.AddressNotFoundException; import javax.inject.Inject; +import java.io.IOException; import java.net.InetAddress; import java.util.Locale; import java.util.Optional; @@ -46,10 +48,13 @@ protected Optional doGetGeoIpData(InetAddress address) { info = GeoLocationInformation.create(loc.latitude(), loc.longitude(), loc.country(), loc.city(), loc.region(), loc.timezone()); - } catch (Exception e) { - String error = String.format(Locale.US, "Error getting IP location info for '%s'. %s", address, e.getMessage()); - LOG.error(error, e); + } catch (IOException | AddressNotFoundException | UnsupportedOperationException e) { info = null; + if (e instanceof AddressNotFoundException == false) { + String error = String.format(Locale.US, "Error getting IP location info for '%s'. %s", address, e.getMessage()); + LOG.error(error, e); + lastError = e.getMessage(); + } } return Optional.ofNullable(info); } diff --git a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/MaxMindIpAsnResolver.java b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/MaxMindIpAsnResolver.java index 0c512d7433f7..03306dce3d4a 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/MaxMindIpAsnResolver.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/MaxMindIpAsnResolver.java @@ -19,6 +19,7 @@ import com.codahale.metrics.Timer; import com.google.inject.assistedinject.Assisted; +import com.maxmind.geoip2.exception.AddressNotFoundException; import com.maxmind.geoip2.exception.GeoIp2Exception; import com.maxmind.geoip2.model.AsnResponse; @@ -48,9 +49,13 @@ protected Optional doGetGeoIpData(InetAddress address) { String number = response.getAutonomousSystemNumber() == null ? "N/A" : response.getAutonomousSystemNumber().toString(); asn = GeoAsnInformation.create(response.getAutonomousSystemOrganization(), "N/A", number); } catch (GeoIp2Exception | IOException | UnsupportedOperationException e) { - String error = String.format(Locale.US, "Error getting ASN for IP Address '%s'. %s", address, e.getMessage()); - LOG.warn(error, e); asn = null; + + if (e instanceof AddressNotFoundException == false) { + String error = String.format(Locale.US, "Error getting ASN for IP Address '%s'. %s", address, e.getMessage()); + LOG.warn(error, e); + lastError = e.getMessage(); + } } return Optional.ofNullable(asn); diff --git a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/MaxMindIpLocationResolver.java b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/MaxMindIpLocationResolver.java index db28a1347ba9..7c4be4a23a4d 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/MaxMindIpLocationResolver.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/MaxMindIpLocationResolver.java @@ -19,12 +19,15 @@ import com.codahale.metrics.Timer; import com.google.inject.assistedinject.Assisted; +import com.maxmind.geoip2.exception.AddressNotFoundException; +import com.maxmind.geoip2.exception.GeoIp2Exception; import com.maxmind.geoip2.model.CityResponse; import com.maxmind.geoip2.record.City; import com.maxmind.geoip2.record.Country; import com.maxmind.geoip2.record.Location; import javax.inject.Inject; +import java.io.IOException; import java.net.InetAddress; import java.util.Optional; @@ -56,9 +59,12 @@ public Optional doGetGeoIpData(InetAddress address) { city.getGeoNameId() == null ? "N/A" : city.getName(),// calling to .getName() may throw a NPE "N/A", "N/A"); - } catch (Exception e) { - LOG.debug("Could not get location from IP {}", address.getHostAddress(), e); + } catch (IOException | GeoIp2Exception | UnsupportedOperationException e) { info = null; + if (e instanceof AddressNotFoundException == false) { + LOG.debug("Could not get location from IP {}", address.getHostAddress(), e); + lastError = e.getMessage(); + } } return Optional.ofNullable(info); diff --git a/graylog2-server/src/main/java/org/graylog2/rest/resources/system/ClusterConfigResource.java b/graylog2-server/src/main/java/org/graylog2/rest/resources/system/ClusterConfigResource.java index d42037d81c50..de99d8a62664 100644 --- a/graylog2-server/src/main/java/org/graylog2/rest/resources/system/ClusterConfigResource.java +++ b/graylog2-server/src/main/java/org/graylog2/rest/resources/system/ClusterConfigResource.java @@ -29,8 +29,10 @@ import org.apache.shiro.authz.annotation.RequiresAuthentication; import org.apache.shiro.authz.annotation.RequiresPermissions; import org.graylog.plugins.map.config.GeoIpResolverConfig; +import org.graylog.plugins.map.geoip.GeoAsnInformation; import org.graylog.plugins.map.geoip.GeoIpResolver; import org.graylog.plugins.map.geoip.GeoIpVendorResolverService; +import org.graylog.plugins.map.geoip.GeoLocationInformation; import org.graylog2.audit.AuditEventTypes; import org.graylog2.audit.jersey.AuditEvent; import org.graylog2.plugin.cluster.ClusterConfigService; @@ -162,25 +164,14 @@ private void validateIfGeoIpResolverConfig(Object o) { Timer timer = new Timer(new UniformReservoir()); try { - GeoIpResolver cityResolver = geoIpVendorResolverService.createCityResolver(config, timer); - if (config.enabled() && !cityResolver.isEnabled()) { - String msg = String.format(Locale.ENGLISH, "Invalid '%s' City Geo IP database file '%s'. Make sure the file exists and is valid for '%1$s'", config.databaseVendorType(), config.cityDbPath()); - throw new IllegalArgumentException(msg); - } - - InetAddress address = Inet4Address.getByName("127.0.0.1"); - - cityResolver.getGeoIpData(address); - - //ASN file is optional--do not validate if not provided. - if (!(config.asnDbPath() == null || config.asnDbPath().trim().isEmpty())) { - GeoIpResolver asnResolver = geoIpVendorResolverService.createAsnResolver(config, timer); - if (config.enabled() && !asnResolver.isEnabled()) { - String msg = String.format(Locale.ENGLISH, "Invalid '%s' ASN database file '%s'. Make sure the file exists and is valid for '%1$s'", config.databaseVendorType(), config.asnDbPath()); - throw new IllegalArgumentException(msg); - } - asnResolver.getGeoIpData(address); - } + //A test address. This will NOT be in any database, but should only produce an + //AddressNotFoundException. Any other exception suggests an actual error such as + //a database file that does does not belong to the vendor selected + InetAddress testAddress = Inet4Address.getByName("127.0.0.1"); + + validateGeoIpLocationResolver(config, timer, testAddress); + validateGeoIpAsnResolver(config, timer, testAddress); + } catch (UnknownHostException | IllegalArgumentException e) { LOG.error(e.getMessage(), e); throw new BadRequestException(e.getMessage()); @@ -188,6 +179,37 @@ private void validateIfGeoIpResolverConfig(Object o) { } } + private void validateGeoIpAsnResolver(GeoIpResolverConfig config, Timer timer, InetAddress testAddress) { + //ASN file is optional--do not validate if not provided. + if (!(config.asnDbPath() == null || config.asnDbPath().trim().isEmpty())) { + GeoIpResolver asnResolver = geoIpVendorResolverService.createAsnResolver(config, timer); + if (config.enabled() && !asnResolver.isEnabled()) { + String msg = String.format(Locale.ENGLISH, "Invalid '%s' ASN database file '%s'. Make sure the file exists and is valid for '%1$s'", config.databaseVendorType(), config.asnDbPath()); + throw new IllegalArgumentException(msg); + } + asnResolver.getGeoIpData(testAddress); + if (asnResolver.getLastError().isPresent()) { + String error = String.format(Locale.ENGLISH, "Error querying ASN. Make sure you have selected a valid ASN database for '%s'", config.databaseVendorType()); + throw new IllegalStateException(error); + } + } + } + + private void validateGeoIpLocationResolver(GeoIpResolverConfig config, Timer timer, InetAddress testAddress) { + GeoIpResolver cityResolver = geoIpVendorResolverService.createCityResolver(config, timer); + if (config.enabled() && !cityResolver.isEnabled()) { + String msg = String.format(Locale.ENGLISH, "Invalid '%s' City Geo IP database file '%s'. Make sure the file exists and is valid for '%1$s'", config.databaseVendorType(), config.cityDbPath()); + throw new IllegalArgumentException(msg); + } + + + cityResolver.getGeoIpData(testAddress); + if (cityResolver.getLastError().isPresent()) { + String error = String.format(Locale.ENGLISH, "Error querying Geo Location. Make sure you have selected a valid Location database for '%s'", config.databaseVendorType()); + throw new IllegalStateException(error); + } + } + @DELETE @Path("{configClass}") @ApiOperation(value = "Delete configuration settings from database") From 3e203d39f3d4e50b9c7941b615c97deac900b331 Mon Sep 17 00:00:00 2001 From: Roberto Benitez Date: Mon, 3 Jan 2022 16:50:20 -0600 Subject: [PATCH 11/28] Reverted front-end temp changes. --- .../maps/configurations/GeoIpResolverConfig.jsx | 4 +--- graylog2-web-interface/webpack/vendor-module-ids.json | 8 +++----- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/graylog2-web-interface/src/components/maps/configurations/GeoIpResolverConfig.jsx b/graylog2-web-interface/src/components/maps/configurations/GeoIpResolverConfig.jsx index 59694bbafc1d..5a53b13149f2 100644 --- a/graylog2-web-interface/src/components/maps/configurations/GeoIpResolverConfig.jsx +++ b/graylog2-web-interface/src/components/maps/configurations/GeoIpResolverConfig.jsx @@ -100,10 +100,8 @@ const GeoIpResolverConfig = createReactClass({ _availableDatabaseTypes() { // TODO: Support country database as well. - //TODO: Add the rest of the configs--added this one to keep from breaking with backend returns IPINFO return [ - { value: 'MAXMIND_CITY', label: 'MaxMind City database' }, - { value: 'IPINFO_STANDARD_LOCATION', label: 'IPInfo Standard Location' }, + { value: 'MAXMIND_CITY', label: 'City database' }, ]; }, diff --git a/graylog2-web-interface/webpack/vendor-module-ids.json b/graylog2-web-interface/webpack/vendor-module-ids.json index b47222f93498..121d36eca69a 100644 --- a/graylog2-web-interface/webpack/vendor-module-ids.json +++ b/graylog2-web-interface/webpack/vendor-module-ids.json @@ -372,12 +372,10 @@ "./node_modules/react-router-bootstrap/lib/IndexLinkContainer.js": 199, "./node_modules/react-router-bootstrap/lib/LinkContainer.js": 3610, "./node_modules/react-router-bootstrap/lib/index.js": 915, - "./node_modules/react-router-dom/esm/react-router-dom.js": 727, "./node_modules/react-router-dom/esm/react-router-dom.js|31d5e9d85c289ae7bc5f646483e4d91b": 9924, "./node_modules/react-router-dom/esm/react-router-dom.js|f87c77e6daa53d57f46e9108aab19cf8": 76, "./node_modules/react-router/esm/react-router.js": 6550, "./node_modules/react-router/esm/react-router.js|32a177570954fe8aee09cce6f8c45fd9": 20, - "./node_modules/react-router/esm/react-router.js|59da0427b285e9b4bbe0ccc5e82a35cf": 977, "./node_modules/react-router/node_modules/path-to-regexp/index.js": 658, "./node_modules/react-transition-group/Transition.js": 644, "./node_modules/react-transition-group/utils/PropTypes.js": 4726, @@ -1784,10 +1782,12 @@ "usedIds": [ 5, 8, + 20, 36, 54, 68, 71, + 76, 94, 101, 124, @@ -1829,18 +1829,17 @@ 670, 699, 702, - 727, 826, 837, 867, 874, 877, 894, + 905, 915, 934, 950, 964, - 977, 1083, 1104, 1106, @@ -1888,7 +1887,6 @@ 2138, 2175, 2177, - 2240, 2243, 2288, 2328, From 18bf9edca66f3403383d67dddbbaf31330d0e9c7 Mon Sep 17 00:00:00 2001 From: Roberto Benitez Date: Tue, 4 Jan 2022 10:23:41 -0600 Subject: [PATCH 12/28] Code cleanup --- .../rest/resources/system/ClusterConfigResource.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/graylog2-server/src/main/java/org/graylog2/rest/resources/system/ClusterConfigResource.java b/graylog2-server/src/main/java/org/graylog2/rest/resources/system/ClusterConfigResource.java index de99d8a62664..98ccb8912657 100644 --- a/graylog2-server/src/main/java/org/graylog2/rest/resources/system/ClusterConfigResource.java +++ b/graylog2-server/src/main/java/org/graylog2/rest/resources/system/ClusterConfigResource.java @@ -26,6 +26,7 @@ import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; +import org.apache.commons.lang3.StringUtils; import org.apache.shiro.authz.annotation.RequiresAuthentication; import org.apache.shiro.authz.annotation.RequiresPermissions; import org.graylog.plugins.map.config.GeoIpResolverConfig; @@ -70,7 +71,7 @@ import static java.util.Objects.requireNonNull; -@Api(value = "System/ClusterConfig") +@Api(value = "System/ClusterConfig", description = "Graylog Cluster Configuration") @RequiresAuthentication @Path("/system/cluster_config") @Produces(MediaType.APPLICATION_JSON) @@ -181,7 +182,7 @@ private void validateIfGeoIpResolverConfig(Object o) { private void validateGeoIpAsnResolver(GeoIpResolverConfig config, Timer timer, InetAddress testAddress) { //ASN file is optional--do not validate if not provided. - if (!(config.asnDbPath() == null || config.asnDbPath().trim().isEmpty())) { + if (config.enabled() && StringUtils.isNotBlank(config.asnDbPath())) { GeoIpResolver asnResolver = geoIpVendorResolverService.createAsnResolver(config, timer); if (config.enabled() && !asnResolver.isEnabled()) { String msg = String.format(Locale.ENGLISH, "Invalid '%s' ASN database file '%s'. Make sure the file exists and is valid for '%1$s'", config.databaseVendorType(), config.asnDbPath()); From 101336bec3202c79bedf217520624d76a2b3dfb1 Mon Sep 17 00:00:00 2001 From: Roberto Benitez Date: Tue, 4 Jan 2022 13:53:23 -0600 Subject: [PATCH 13/28] Started extraction of cluster config validation --- .../java/org/graylog2/commands/Server.java | 4 +- .../system/ClusterConfigResource.java | 98 +++++-------------- .../validate/ClusterConfigValidator.java | 23 +++++ .../ClusterConfigValidatorModule.java | 38 +++++++ .../ClusterConfigValidatorService.java | 46 +++++++++ .../validate/ConfigValidationException.java | 24 +++++ .../GeoIpResolverConfigValidator.java | 97 ++++++++++++++++++ 7 files changed, 257 insertions(+), 73 deletions(-) create mode 100644 graylog2-server/src/main/java/org/graylog2/rest/resources/system/validate/ClusterConfigValidator.java create mode 100644 graylog2-server/src/main/java/org/graylog2/rest/resources/system/validate/ClusterConfigValidatorModule.java create mode 100644 graylog2-server/src/main/java/org/graylog2/rest/resources/system/validate/ClusterConfigValidatorService.java create mode 100644 graylog2-server/src/main/java/org/graylog2/rest/resources/system/validate/ConfigValidationException.java create mode 100644 graylog2-server/src/main/java/org/graylog2/rest/resources/system/validate/GeoIpResolverConfigValidator.java diff --git a/graylog2-server/src/main/java/org/graylog2/commands/Server.java b/graylog2-server/src/main/java/org/graylog2/commands/Server.java index 63dc1652da00..4236ebfabae7 100644 --- a/graylog2-server/src/main/java/org/graylog2/commands/Server.java +++ b/graylog2-server/src/main/java/org/graylog2/commands/Server.java @@ -82,6 +82,7 @@ import org.graylog2.plugin.ServerStatus; import org.graylog2.plugin.Tools; import org.graylog2.plugin.system.NodeId; +import org.graylog2.rest.resources.system.validate.ClusterConfigValidatorModule; import org.graylog2.shared.UI; import org.graylog2.shared.bindings.MessageInputBindings; import org.graylog2.shared.bindings.ObjectMapperModule; @@ -179,7 +180,8 @@ protected List getCommandBindings() { new FreeEnterpriseModule(), new GRNTypesModule(), new SecurityModule(), - new PrometheusMetricsModule() + new PrometheusMetricsModule(), + new ClusterConfigValidatorModule() ); if (!isMigrationCommand()) { diff --git a/graylog2-server/src/main/java/org/graylog2/rest/resources/system/ClusterConfigResource.java b/graylog2-server/src/main/java/org/graylog2/rest/resources/system/ClusterConfigResource.java index 98ccb8912657..5d148f61c60e 100644 --- a/graylog2-server/src/main/java/org/graylog2/rest/resources/system/ClusterConfigResource.java +++ b/graylog2-server/src/main/java/org/graylog2/rest/resources/system/ClusterConfigResource.java @@ -16,8 +16,6 @@ */ package org.graylog2.rest.resources.system; -import com.codahale.metrics.Timer; -import com.codahale.metrics.UniformReservoir; import com.codahale.metrics.annotation.Timed; import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.ObjectMapper; @@ -26,19 +24,15 @@ import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; -import org.apache.commons.lang3.StringUtils; import org.apache.shiro.authz.annotation.RequiresAuthentication; import org.apache.shiro.authz.annotation.RequiresPermissions; -import org.graylog.plugins.map.config.GeoIpResolverConfig; -import org.graylog.plugins.map.geoip.GeoAsnInformation; -import org.graylog.plugins.map.geoip.GeoIpResolver; -import org.graylog.plugins.map.geoip.GeoIpVendorResolverService; -import org.graylog.plugins.map.geoip.GeoLocationInformation; import org.graylog2.audit.AuditEventTypes; import org.graylog2.audit.jersey.AuditEvent; import org.graylog2.plugin.cluster.ClusterConfigService; import org.graylog2.rest.MoreMediaTypes; import org.graylog2.rest.models.system.config.ClusterConfigList; +import org.graylog2.rest.resources.system.validate.ClusterConfigValidatorService; +import org.graylog2.rest.resources.system.validate.ConfigValidationException; import org.graylog2.shared.plugins.ChainingClassLoader; import org.graylog2.shared.rest.resources.RestResource; import org.graylog2.shared.security.RestPermissions; @@ -63,9 +57,6 @@ import javax.ws.rs.core.Response; import java.io.IOException; import java.io.InputStream; -import java.net.Inet4Address; -import java.net.InetAddress; -import java.net.UnknownHostException; import java.util.Locale; import java.util.Set; @@ -82,17 +73,17 @@ public class ClusterConfigResource extends RestResource { private final ClusterConfigService clusterConfigService; private final ChainingClassLoader chainingClassLoader; private final ObjectMapper objectMapper; - private final GeoIpVendorResolverService geoIpVendorResolverService; + private final ClusterConfigValidatorService clusterConfigValidatorService; @Inject public ClusterConfigResource(ClusterConfigService clusterConfigService, ChainingClassLoader chainingClassLoader, ObjectMapper objectMapper, - GeoIpVendorResolverService geoIpVendorResolverService) { + ClusterConfigValidatorService clusterConfigValidatorService) { this.clusterConfigService = requireNonNull(clusterConfigService); this.chainingClassLoader = chainingClassLoader; this.objectMapper = objectMapper; - this.geoIpVendorResolverService = geoIpVendorResolverService; + this.clusterConfigValidatorService = clusterConfigValidatorService; } @GET @@ -137,78 +128,41 @@ public Response update(@ApiParam(name = "configClass", value = "The name of the throw new NotFoundException(createNoClassMsg(configClass)); } - final Object o; - try { - o = objectMapper.readValue(body, cls); - } catch (Exception e) { - final String msg = "Couldn't parse cluster configuration \"" + configClass + "\"."; - LOG.error(msg, e); - throw new BadRequestException(msg); - } + final Object configObject = parseConfigObject(configClass, body, cls); + validateConfigObject(configObject); + writeConfigObject(configClass, configObject); - validateIfGeoIpResolverConfig(o); + return Response.accepted(configObject).build(); + } + private void writeConfigObject(String configClass, Object configObject) { try { - clusterConfigService.write(o); + clusterConfigService.write(configObject); } catch (Exception e) { final String msg = "Couldn't write cluster config \"" + configClass + "\"."; LOG.error(msg, e); throw new InternalServerErrorException(msg, e); } - - return Response.accepted(o).build(); } - private void validateIfGeoIpResolverConfig(Object o) { - if (o instanceof GeoIpResolverConfig) { - final GeoIpResolverConfig config = (GeoIpResolverConfig) o; - - Timer timer = new Timer(new UniformReservoir()); - try { - //A test address. This will NOT be in any database, but should only produce an - //AddressNotFoundException. Any other exception suggests an actual error such as - //a database file that does does not belong to the vendor selected - InetAddress testAddress = Inet4Address.getByName("127.0.0.1"); - - validateGeoIpLocationResolver(config, timer, testAddress); - validateGeoIpAsnResolver(config, timer, testAddress); - - } catch (UnknownHostException | IllegalArgumentException e) { - LOG.error(e.getMessage(), e); - throw new BadRequestException(e.getMessage()); - } - } - } - - private void validateGeoIpAsnResolver(GeoIpResolverConfig config, Timer timer, InetAddress testAddress) { - //ASN file is optional--do not validate if not provided. - if (config.enabled() && StringUtils.isNotBlank(config.asnDbPath())) { - GeoIpResolver asnResolver = geoIpVendorResolverService.createAsnResolver(config, timer); - if (config.enabled() && !asnResolver.isEnabled()) { - String msg = String.format(Locale.ENGLISH, "Invalid '%s' ASN database file '%s'. Make sure the file exists and is valid for '%1$s'", config.databaseVendorType(), config.asnDbPath()); - throw new IllegalArgumentException(msg); - } - asnResolver.getGeoIpData(testAddress); - if (asnResolver.getLastError().isPresent()) { - String error = String.format(Locale.ENGLISH, "Error querying ASN. Make sure you have selected a valid ASN database for '%s'", config.databaseVendorType()); - throw new IllegalStateException(error); - } + private void validateConfigObject(Object configObject) { + try { + clusterConfigValidatorService.validate(configObject); + } catch (ConfigValidationException e) { + throw new BadRequestException(e.getMessage(), e); } } - private void validateGeoIpLocationResolver(GeoIpResolverConfig config, Timer timer, InetAddress testAddress) { - GeoIpResolver cityResolver = geoIpVendorResolverService.createCityResolver(config, timer); - if (config.enabled() && !cityResolver.isEnabled()) { - String msg = String.format(Locale.ENGLISH, "Invalid '%s' City Geo IP database file '%s'. Make sure the file exists and is valid for '%1$s'", config.databaseVendorType(), config.cityDbPath()); - throw new IllegalArgumentException(msg); - } - - - cityResolver.getGeoIpData(testAddress); - if (cityResolver.getLastError().isPresent()) { - String error = String.format(Locale.ENGLISH, "Error querying Geo Location. Make sure you have selected a valid Location database for '%s'", config.databaseVendorType()); - throw new IllegalStateException(error); + private Object parseConfigObject(String configClass, InputStream body, Class cls) { + final Object object; + try { + object = objectMapper.readValue(body, cls); + } catch (Exception e) { + final String msg = "Couldn't parse cluster configuration \"" + configClass + "\"."; + LOG.error(msg, e); + throw new BadRequestException(msg); } + return object; } @DELETE diff --git a/graylog2-server/src/main/java/org/graylog2/rest/resources/system/validate/ClusterConfigValidator.java b/graylog2-server/src/main/java/org/graylog2/rest/resources/system/validate/ClusterConfigValidator.java new file mode 100644 index 000000000000..3db70cdc4543 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog2/rest/resources/system/validate/ClusterConfigValidator.java @@ -0,0 +1,23 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ + +package org.graylog2.rest.resources.system.validate; + +public interface ClusterConfigValidator { + + void validate(Object configObject) throws ConfigValidationException; +} diff --git a/graylog2-server/src/main/java/org/graylog2/rest/resources/system/validate/ClusterConfigValidatorModule.java b/graylog2-server/src/main/java/org/graylog2/rest/resources/system/validate/ClusterConfigValidatorModule.java new file mode 100644 index 000000000000..04bddf48e68e --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog2/rest/resources/system/validate/ClusterConfigValidatorModule.java @@ -0,0 +1,38 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ + +package org.graylog2.rest.resources.system.validate; + +import com.google.inject.multibindings.MapBinder; +import org.graylog.plugins.map.config.GeoIpResolverConfig; +import org.graylog2.plugin.PluginModule; + +public class ClusterConfigValidatorModule extends PluginModule { + + @Override + public void configure() { + addClusterConfigValidator(GeoIpResolverConfig.class, GeoIpResolverConfigValidator.class); + } + + private void addClusterConfigValidator(Class configClass, Class configValidatorClass) { + + MapBinder mapBinder = MapBinder.newMapBinder(binder(), Class.class, ClusterConfigValidator.class); + + mapBinder.addBinding(configClass).to(configValidatorClass); + + } +} diff --git a/graylog2-server/src/main/java/org/graylog2/rest/resources/system/validate/ClusterConfigValidatorService.java b/graylog2-server/src/main/java/org/graylog2/rest/resources/system/validate/ClusterConfigValidatorService.java new file mode 100644 index 000000000000..6085b0b0e573 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog2/rest/resources/system/validate/ClusterConfigValidatorService.java @@ -0,0 +1,46 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ + +package org.graylog2.rest.resources.system.validate; + +import javax.inject.Inject; +import java.util.Map; + +public class ClusterConfigValidatorService { + + private final Map validators; + + @Inject + public ClusterConfigValidatorService(Map validators) { + this.validators = validators; + } + + public void validate(Object configObject) throws ConfigValidationException { + if (configObject == null) { + return; + } + + try { + Class zclass = configObject.getClass(); + ClusterConfigValidator validator = validators.getOrDefault(zclass, object -> {}); + validator.validate(configObject); + + } catch (RuntimeException e) { + throw new ConfigValidationException(e.getMessage()); + } + } +} diff --git a/graylog2-server/src/main/java/org/graylog2/rest/resources/system/validate/ConfigValidationException.java b/graylog2-server/src/main/java/org/graylog2/rest/resources/system/validate/ConfigValidationException.java new file mode 100644 index 000000000000..dd7ef7fbe519 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog2/rest/resources/system/validate/ConfigValidationException.java @@ -0,0 +1,24 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ + +package org.graylog2.rest.resources.system.validate; + +public class ConfigValidationException extends Exception { + public ConfigValidationException(String message) { + super(message); + } +} diff --git a/graylog2-server/src/main/java/org/graylog2/rest/resources/system/validate/GeoIpResolverConfigValidator.java b/graylog2-server/src/main/java/org/graylog2/rest/resources/system/validate/GeoIpResolverConfigValidator.java new file mode 100644 index 000000000000..cb9b4b65a66f --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog2/rest/resources/system/validate/GeoIpResolverConfigValidator.java @@ -0,0 +1,97 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ + +package org.graylog2.rest.resources.system.validate; + +import com.codahale.metrics.Timer; +import com.codahale.metrics.UniformReservoir; +import org.apache.commons.lang3.StringUtils; +import org.graylog.plugins.map.config.GeoIpResolverConfig; +import org.graylog.plugins.map.geoip.GeoAsnInformation; +import org.graylog.plugins.map.geoip.GeoIpResolver; +import org.graylog.plugins.map.geoip.GeoIpVendorResolverService; +import org.graylog.plugins.map.geoip.GeoLocationInformation; + +import javax.inject.Inject; +import java.net.Inet4Address; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.Locale; + +public class GeoIpResolverConfigValidator implements ClusterConfigValidator { + + private final GeoIpVendorResolverService geoIpVendorResolverService; + + @Inject + public GeoIpResolverConfigValidator(GeoIpVendorResolverService geoIpVendorResolverService) { + this.geoIpVendorResolverService = geoIpVendorResolverService; + } + + @Override + public void validate(Object configObject) throws ConfigValidationException { + + if (configObject instanceof GeoIpResolverConfig) { + final GeoIpResolverConfig config = (GeoIpResolverConfig) configObject; + + Timer timer = new Timer(new UniformReservoir()); + try { + //A test address. This will NOT be in any database, but should only produce an + //AddressNotFoundException. Any other exception suggests an actual error such as + //a database file that does does not belong to the vendor selected + InetAddress testAddress = Inet4Address.getByName("127.0.0.1"); + + validateGeoIpLocationResolver(config, timer, testAddress); + validateGeoIpAsnResolver(config, timer, testAddress); + + } catch (UnknownHostException | IllegalArgumentException e) { + throw new ConfigValidationException(e.getMessage()); + } + } + } + + private void validateGeoIpAsnResolver(GeoIpResolverConfig config, Timer timer, InetAddress testAddress) { + //ASN file is optional--do not validate if not provided. + if (config.enabled() && StringUtils.isNotBlank(config.asnDbPath())) { + GeoIpResolver asnResolver = geoIpVendorResolverService.createAsnResolver(config, timer); + if (config.enabled() && !asnResolver.isEnabled()) { + String msg = String.format(Locale.ENGLISH, "Invalid '%s' ASN database file '%s'. Make sure the file exists and is valid for '%1$s'", config.databaseVendorType(), config.asnDbPath()); + throw new IllegalArgumentException(msg); + } + asnResolver.getGeoIpData(testAddress); + if (asnResolver.getLastError().isPresent()) { + String error = String.format(Locale.ENGLISH, "Error querying ASN. Make sure you have selected a valid ASN database for '%s'", config.databaseVendorType()); + throw new IllegalStateException(error); + } + } + } + + private void validateGeoIpLocationResolver(GeoIpResolverConfig config, Timer timer, InetAddress testAddress) { + GeoIpResolver cityResolver = geoIpVendorResolverService.createCityResolver(config, timer); + if (config.enabled() && !cityResolver.isEnabled()) { + String msg = String.format(Locale.ENGLISH, "Invalid '%s' City Geo IP database file '%s'. Make sure the file exists and is valid for '%1$s'", config.databaseVendorType(), config.cityDbPath()); + throw new IllegalArgumentException(msg); + } + + + cityResolver.getGeoIpData(testAddress); + if (cityResolver.getLastError().isPresent()) { + String error = String.format(Locale.ENGLISH, "Error querying Geo Location. Make sure you have selected a valid Location database for '%s'", config.databaseVendorType()); + throw new IllegalStateException(error); + } + } + +} From 4efdb5dd06bd4500c9a5873794225759b182a45e Mon Sep 17 00:00:00 2001 From: Roberto Benitez Date: Wed, 5 Jan 2022 14:15:45 -0600 Subject: [PATCH 14/28] Cleaned up cluster validation and db migration --- .../map/config/GeoIpResolverConfig.java | 8 ++-- ...21144300_GeoIpResolverConfigMigration.java | 19 +++++---- .../validate/ClusterConfigValidator.java | 3 ++ .../ClusterConfigValidatorModule.java | 11 +++-- .../ClusterConfigValidatorService.java | 12 ++++-- .../GeoIpResolverConfigValidator.java | 40 +++++++++++++------ 6 files changed, 61 insertions(+), 32 deletions(-) diff --git a/graylog2-server/src/main/java/org/graylog/plugins/map/config/GeoIpResolverConfig.java b/graylog2-server/src/main/java/org/graylog/plugins/map/config/GeoIpResolverConfig.java index 4a0cf6b61af9..26799491ae19 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/map/config/GeoIpResolverConfig.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/map/config/GeoIpResolverConfig.java @@ -30,10 +30,10 @@ public abstract class GeoIpResolverConfig { @JsonProperty("enabled") public abstract boolean enabled(); - @JsonProperty("database_vendor_type") + @JsonProperty("db_vendor_type") public abstract DatabaseVendorType databaseVendorType(); - @JsonProperty("db_path") + @JsonProperty("city_db_path") public abstract String cityDbPath(); @JsonProperty("asn_db_path") @@ -41,8 +41,8 @@ public abstract class GeoIpResolverConfig { @JsonCreator public static GeoIpResolverConfig create(@JsonProperty("enabled") boolean cityEnabled, - @JsonProperty("database_vendor_type") DatabaseVendorType databaseVendorType, - @JsonProperty("db_path") String cityDbPath, + @JsonProperty("db_vendor_type") DatabaseVendorType databaseVendorType, + @JsonProperty("city_db_path") String cityDbPath, @JsonProperty("asn_db_path") String asnDbPath) { return builder() .enabled(cityEnabled) diff --git a/graylog2-server/src/main/java/org/graylog2/migrations/V20211221144300_GeoIpResolverConfigMigration.java b/graylog2-server/src/main/java/org/graylog2/migrations/V20211221144300_GeoIpResolverConfigMigration.java index 46c71eb303b8..b655fa714c8d 100644 --- a/graylog2-server/src/main/java/org/graylog2/migrations/V20211221144300_GeoIpResolverConfigMigration.java +++ b/graylog2-server/src/main/java/org/graylog2/migrations/V20211221144300_GeoIpResolverConfigMigration.java @@ -30,9 +30,6 @@ import org.slf4j.LoggerFactory; import javax.inject.Inject; -import java.time.LocalDateTime; -import java.time.Month; -import java.time.ZoneId; import java.time.ZonedDateTime; public class V20211221144300_GeoIpResolverConfigMigration extends Migration { @@ -40,8 +37,10 @@ public class V20211221144300_GeoIpResolverConfigMigration extends Migration { private static final Logger LOG = LoggerFactory.getLogger(V20211221144300_GeoIpResolverConfigMigration.class); private static final String COLLECTION_NAME = "cluster_config"; public static final String PAYLOAD = "payload"; - private static final String FIELD_DB_VENDOR = PAYLOAD + ".database_vendor_type"; + private static final String FIELD_DB_VENDOR = PAYLOAD + ".db_vendor_type"; private static final String FIELD_DB_TYPE = PAYLOAD + ".db_type"; + private static final String FIELD_DB_PATH = PAYLOAD + ".db_path"; + private static final String FIELD_CITY_DB_PATH = PAYLOAD + ".city_db_path"; private final MongoConnection mongoConnection; @@ -52,7 +51,7 @@ public V20211221144300_GeoIpResolverConfigMigration(MongoConnection mongoConnect @Override public ZonedDateTime createdAt() { - return ZonedDateTime.of(LocalDateTime.of(2021, Month.DECEMBER, 21, 14, 43), ZoneId.systemDefault()); + return ZonedDateTime.parse("2021-12-21t14:43Z"); } /** @@ -71,13 +70,13 @@ public void upgrade() { Bson geoConfFiler = Filters.eq("type", GeoIpResolverConfig.class.getCanonicalName()); Bson noColumnFilter = Filters.exists(FIELD_DB_VENDOR, false); - //Set default db vendor field - Bson setDefaultVendor = Updates.set(FIELD_DB_VENDOR, DatabaseVendorType.MAXMIND.name()); + //rename db type field to db vendor type + Bson renameDbTypeToVendor = Updates.rename(FIELD_DB_TYPE, FIELD_DB_VENDOR); - //remove vestigial fields - Bson dropAsn = Updates.unset(FIELD_DB_TYPE); + //rename existing db_path field to city_db_path + Bson renameDbPath = Updates.rename(FIELD_DB_PATH, FIELD_CITY_DB_PATH); - Bson updates = Updates.combine(setDefaultVendor, dropAsn); + Bson updates = Updates.combine(renameDbTypeToVendor, renameDbPath); LOG.info("Planned Updates: {}", updates); final UpdateResult updateResult = collection.updateOne(Filters.and(geoConfFiler, noColumnFilter), updates); LOG.info("Update Result: {}", updateResult); diff --git a/graylog2-server/src/main/java/org/graylog2/rest/resources/system/validate/ClusterConfigValidator.java b/graylog2-server/src/main/java/org/graylog2/rest/resources/system/validate/ClusterConfigValidator.java index 3db70cdc4543..83ff4561ac15 100644 --- a/graylog2-server/src/main/java/org/graylog2/rest/resources/system/validate/ClusterConfigValidator.java +++ b/graylog2-server/src/main/java/org/graylog2/rest/resources/system/validate/ClusterConfigValidator.java @@ -17,6 +17,9 @@ package org.graylog2.rest.resources.system.validate; +/** + * Specification for a Cluster Configuration object. + **/ public interface ClusterConfigValidator { void validate(Object configObject) throws ConfigValidationException; diff --git a/graylog2-server/src/main/java/org/graylog2/rest/resources/system/validate/ClusterConfigValidatorModule.java b/graylog2-server/src/main/java/org/graylog2/rest/resources/system/validate/ClusterConfigValidatorModule.java index 04bddf48e68e..8c17e203ccd9 100644 --- a/graylog2-server/src/main/java/org/graylog2/rest/resources/system/validate/ClusterConfigValidatorModule.java +++ b/graylog2-server/src/main/java/org/graylog2/rest/resources/system/validate/ClusterConfigValidatorModule.java @@ -17,22 +17,27 @@ package org.graylog2.rest.resources.system.validate; +import com.google.inject.TypeLiteral; import com.google.inject.multibindings.MapBinder; import org.graylog.plugins.map.config.GeoIpResolverConfig; import org.graylog2.plugin.PluginModule; public class ClusterConfigValidatorModule extends PluginModule { - @Override public void configure() { + addClusterConfigValidator(GeoIpResolverConfig.class, GeoIpResolverConfigValidator.class); } private void addClusterConfigValidator(Class configClass, Class configValidatorClass) { - MapBinder mapBinder = MapBinder.newMapBinder(binder(), Class.class, ClusterConfigValidator.class); + mapBinder().addBinding(configClass).to(configValidatorClass); - mapBinder.addBinding(configClass).to(configValidatorClass); + } + private MapBinder, ClusterConfigValidator> mapBinder() { + TypeLiteral> keyType = new TypeLiteral>() {}; + TypeLiteral valueType = new TypeLiteral() {}; + return MapBinder.newMapBinder(binder(), keyType, valueType); } } diff --git a/graylog2-server/src/main/java/org/graylog2/rest/resources/system/validate/ClusterConfigValidatorService.java b/graylog2-server/src/main/java/org/graylog2/rest/resources/system/validate/ClusterConfigValidatorService.java index 6085b0b0e573..b368af308d29 100644 --- a/graylog2-server/src/main/java/org/graylog2/rest/resources/system/validate/ClusterConfigValidatorService.java +++ b/graylog2-server/src/main/java/org/graylog2/rest/resources/system/validate/ClusterConfigValidatorService.java @@ -22,10 +22,10 @@ public class ClusterConfigValidatorService { - private final Map validators; + private final Map, ClusterConfigValidator> validators; @Inject - public ClusterConfigValidatorService(Map validators) { + public ClusterConfigValidatorService(Map, ClusterConfigValidator> validators) { this.validators = validators; } @@ -36,7 +36,13 @@ public void validate(Object configObject) throws ConfigValidationException { try { Class zclass = configObject.getClass(); - ClusterConfigValidator validator = validators.getOrDefault(zclass, object -> {}); + ClusterConfigValidator validator = validators.get(zclass); + if (validator == null) { + //try parent class--the config object is likely an AutoValue generated class which extends the + //registered class. + Class zParent = zclass.getSuperclass(); + validator = validators.getOrDefault(zParent, obj -> {}); + } validator.validate(configObject); } catch (RuntimeException e) { diff --git a/graylog2-server/src/main/java/org/graylog2/rest/resources/system/validate/GeoIpResolverConfigValidator.java b/graylog2-server/src/main/java/org/graylog2/rest/resources/system/validate/GeoIpResolverConfigValidator.java index cb9b4b65a66f..a552f7eb31cf 100644 --- a/graylog2-server/src/main/java/org/graylog2/rest/resources/system/validate/GeoIpResolverConfigValidator.java +++ b/graylog2-server/src/main/java/org/graylog2/rest/resources/system/validate/GeoIpResolverConfigValidator.java @@ -25,15 +25,21 @@ import org.graylog.plugins.map.geoip.GeoIpResolver; import org.graylog.plugins.map.geoip.GeoIpVendorResolverService; import org.graylog.plugins.map.geoip.GeoLocationInformation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import javax.inject.Inject; -import java.net.Inet4Address; import java.net.InetAddress; import java.net.UnknownHostException; import java.util.Locale; +/** + * A {@link ClusterConfigValidator} to validate configuration objects of type {@link GeoIpResolverConfig}. + */ public class GeoIpResolverConfigValidator implements ClusterConfigValidator { + private static final Logger LOG = LoggerFactory.getLogger(GeoIpResolverConfigValidator.class); + private final GeoIpVendorResolverService geoIpVendorResolverService; @Inject @@ -47,19 +53,29 @@ public void validate(Object configObject) throws ConfigValidationException { if (configObject instanceof GeoIpResolverConfig) { final GeoIpResolverConfig config = (GeoIpResolverConfig) configObject; - Timer timer = new Timer(new UniformReservoir()); - try { - //A test address. This will NOT be in any database, but should only produce an - //AddressNotFoundException. Any other exception suggests an actual error such as - //a database file that does does not belong to the vendor selected - InetAddress testAddress = Inet4Address.getByName("127.0.0.1"); + if (config.enabled()) { + validateConfig(config); + } else { + LOG.debug("'{}' is disabled. Skipping validation", config); + } + } else { + LOG.warn("'{}' cannot be validated with '{}'. Validator may have been registered incorrectly.", configObject, getClass()); + } + } - validateGeoIpLocationResolver(config, timer, testAddress); - validateGeoIpAsnResolver(config, timer, testAddress); + private void validateConfig(GeoIpResolverConfig config) throws ConfigValidationException { + Timer timer = new Timer(new UniformReservoir()); + try { + //A test address. This will NOT be in any database, but should only produce an + //AddressNotFoundException. Any other exception suggests an actual error such as + //a database file that does does not belong to the vendor selected + InetAddress testAddress = InetAddress.getByName("127.0.0.1"); - } catch (UnknownHostException | IllegalArgumentException e) { - throw new ConfigValidationException(e.getMessage()); - } + validateGeoIpLocationResolver(config, timer, testAddress); + validateGeoIpAsnResolver(config, timer, testAddress); + + } catch (UnknownHostException | IllegalArgumentException e) { + throw new ConfigValidationException(e.getMessage()); } } From f3110aed73c4030a31314eea102f9dab671b46a0 Mon Sep 17 00:00:00 2001 From: Roberto Benitez Date: Thu, 6 Jan 2022 10:55:52 -0600 Subject: [PATCH 15/28] Updated message fields. Add config field to optionally enforce the Graylog Schema--if true, IP address fields to search will be limited to the preset list. --- .../map/config/GeoIpResolverConfig.java | 8 +++++ .../map/geoip/GeoIpResolverEngine.java | 32 +++++++++++++++---- .../map/geoip/GeoLocationInformation.java | 6 ++-- .../map/geoip/IpInfoLocationResolver.java | 6 ++-- .../map/geoip/MaxMindIpLocationResolver.java | 3 +- .../GeoIpResolverConfigValidator.java | 4 +-- 6 files changed, 45 insertions(+), 14 deletions(-) diff --git a/graylog2-server/src/main/java/org/graylog/plugins/map/config/GeoIpResolverConfig.java b/graylog2-server/src/main/java/org/graylog/plugins/map/config/GeoIpResolverConfig.java index 26799491ae19..96d1a7e602f5 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/map/config/GeoIpResolverConfig.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/map/config/GeoIpResolverConfig.java @@ -30,6 +30,9 @@ public abstract class GeoIpResolverConfig { @JsonProperty("enabled") public abstract boolean enabled(); + @JsonProperty("enforce_graylog_schema") + public abstract boolean enforceGraylogSchema(); + @JsonProperty("db_vendor_type") public abstract DatabaseVendorType databaseVendorType(); @@ -41,11 +44,13 @@ public abstract class GeoIpResolverConfig { @JsonCreator public static GeoIpResolverConfig create(@JsonProperty("enabled") boolean cityEnabled, + @JsonProperty("enforce_graylog_schema") boolean enforceGraylogSchema, @JsonProperty("db_vendor_type") DatabaseVendorType databaseVendorType, @JsonProperty("city_db_path") String cityDbPath, @JsonProperty("asn_db_path") String asnDbPath) { return builder() .enabled(cityEnabled) + .enforceGraylogSchema(enforceGraylogSchema) .databaseVendorType(databaseVendorType == null ? DatabaseVendorType.MAXMIND : databaseVendorType) .cityDbPath(cityDbPath) .asnDbPath(asnDbPath) @@ -56,6 +61,7 @@ public static GeoIpResolverConfig defaultConfig() { return builder() .enabled(false) .databaseVendorType(DatabaseVendorType.MAXMIND) + .enforceGraylogSchema(false) .cityDbPath("/etc/graylog/server/GeoLite2-City.mmdb") .asnDbPath("/etc/graylog/server/GeoLite2-ASN.mmdb") .build(); @@ -71,6 +77,8 @@ public static Builder builder() { public abstract static class Builder { public abstract Builder enabled(boolean enabled); + public abstract Builder enforceGraylogSchema(boolean enforce); + public abstract Builder databaseVendorType(DatabaseVendorType type); public abstract Builder cityDbPath(String dbPath); diff --git a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpResolverEngine.java b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpResolverEngine.java index 9af26489a360..c04da2e64b6c 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpResolverEngine.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpResolverEngine.java @@ -41,7 +41,8 @@ public class GeoIpResolverEngine { private static final Logger LOG = LoggerFactory.getLogger(GeoIpResolverEngine.class); /** - * A mapping of fields that to search that contain IP addresses. ONLY these fields will be checked + * A mapping of fields (per the Graylog Schema) that to search that contain IP addresses. When the user opts to + * enforce the Graylog Schema ONLY these fields will be checked; otherwise, all message fields will be checked. * to see if they have valid Geo IP information. * *

@@ -58,12 +59,14 @@ public class GeoIpResolverEngine { private final GeoIpResolver ipLocationResolver; private final GeoIpResolver ipAsnResolver; - private boolean enabled; + private final boolean enabled; + private final boolean enforceGraylogSchema; public GeoIpResolverEngine(GeoIpVendorResolverService resolverService, GeoIpResolverConfig config, MetricRegistry metricRegistry) { Timer resolveTime = metricRegistry.timer(name(GeoIpResolverEngine.class, "resolveTime")); + enforceGraylogSchema = config.enforceGraylogSchema(); ipLocationResolver = resolverService.createCityResolver(config, resolveTime); ipAsnResolver = resolverService.createAsnResolver(config, resolveTime); @@ -89,7 +92,9 @@ public boolean filter(Message message) { continue; } - final String prefix = ipAddressFields.get(key); + //IF the user has opted NOT to enforce the Graylog Schema, the key will likely not + //be in the field map--in such cases use the key (full field name) as the prefix + final String prefix = ipAddressFields.getOrDefault(key, key); if (ReservedIpChecker.getInstance().isReservedIpAddress(address.getHostAddress())) { message.addField(prefix + "_reserved_ip", true); @@ -105,10 +110,14 @@ public boolean filter(Message message) { private void addGeoIpDataIfPresent(Message message, InetAddress address, String newFieldPrefix) { ipLocationResolver.getGeoIpData(address).ifPresent(locationInformation -> { message.addField(newFieldPrefix + "_geo_coordinates", locationInformation.latitude() + "," + locationInformation.longitude()); - message.addField(newFieldPrefix + "_geo_country", locationInformation.countryIsoCode()); + message.addField(newFieldPrefix + "_geo_country_iso", locationInformation.countryIsoCode()); message.addField(newFieldPrefix + "_geo_city", locationInformation.cityName()); message.addField(newFieldPrefix + "_geo_region", locationInformation.region()); - message.addField(newFieldPrefix + "_geo_timeZone", locationInformation.timeZone()); + message.addField(newFieldPrefix + "_geo_timezone", locationInformation.timeZone()); + + if (areValidGeoNames(locationInformation.countryName())) { + message.addField(newFieldPrefix + "_geo_country", locationInformation.countryName()); + } if (areValidGeoNames(locationInformation.cityName(), locationInformation.countryIsoCode())) { String name = String.format(Locale.ENGLISH, "%s, %s", locationInformation.cityName(), locationInformation.countryIsoCode()); @@ -123,10 +132,21 @@ private void addGeoIpDataIfPresent(Message message, InetAddress address, String }); } + /** + * Get the message fields that will be checked for IP addresses. + * + *

+ * If the user has chosen NOT to enforce the Graylog Schema, then all fields will be checked as any field could + * have an IP address. + *

+ * + * @param message message + * @return a list of field that may have an IP address + */ private List getIpAddressFields(Message message) { return message.getFieldNames() .stream() - .filter(e -> ipAddressFields.containsKey(e) + .filter(e -> (!enforceGraylogSchema || ipAddressFields.containsKey(e)) && !e.startsWith(Message.INTERNAL_FIELD_PREFIX)) .collect(Collectors.toList()); } diff --git a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoLocationInformation.java b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoLocationInformation.java index 857732e2e2ee..3f238d0af25f 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoLocationInformation.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoLocationInformation.java @@ -27,15 +27,17 @@ public abstract class GeoLocationInformation { public abstract String countryIsoCode(); + public abstract String countryName(); + public abstract String cityName(); public abstract String region(); public abstract String timeZone(); - public static GeoLocationInformation create(double latitude, double longitude, String countryIsoCode, String cityName, + public static GeoLocationInformation create(double latitude, double longitude, String countryIsoCode, String countryName, String cityName, String region, String timeZone) { - return new AutoValue_GeoLocationInformation(latitude, longitude, countryIsoCode, cityName, + return new AutoValue_GeoLocationInformation(latitude, longitude, countryIsoCode, countryName, cityName, region, timeZone); } } diff --git a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/IpInfoLocationResolver.java b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/IpInfoLocationResolver.java index 218ba59af0b1..235b15ca7482 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/IpInfoLocationResolver.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/IpInfoLocationResolver.java @@ -45,12 +45,12 @@ protected Optional doGetGeoIpData(InetAddress address) { try (Timer.Context ignored = resolveTime.time()) { IPinfoStandardLocation loc = adapter.ipInfoStandardLocation(address); - info = GeoLocationInformation.create(loc.latitude(), loc.longitude(), loc.country(), + info = GeoLocationInformation.create(loc.latitude(), loc.longitude(), loc.country(), "N/A", loc.city(), loc.region(), loc.timezone()); - } catch (IOException | AddressNotFoundException | UnsupportedOperationException e) { + } catch (NullPointerException | IOException | AddressNotFoundException | UnsupportedOperationException e) { info = null; - if (e instanceof AddressNotFoundException == false) { + if (e instanceof AddressNotFoundException) { String error = String.format(Locale.US, "Error getting IP location info for '%s'. %s", address, e.getMessage()); LOG.error(error, e); lastError = e.getMessage(); diff --git a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/MaxMindIpLocationResolver.java b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/MaxMindIpLocationResolver.java index 7c4be4a23a4d..f8e621274acc 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/MaxMindIpLocationResolver.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/MaxMindIpLocationResolver.java @@ -56,12 +56,13 @@ public Optional doGetGeoIpData(InetAddress address) { info = GeoLocationInformation.create( location.getLatitude(), location.getLongitude(), country.getGeoNameId() == null ? "N/A" : country.getIsoCode(), + country.getGeoNameId() == null ? "N/A" : country.getName(), city.getGeoNameId() == null ? "N/A" : city.getName(),// calling to .getName() may throw a NPE "N/A", "N/A"); } catch (IOException | GeoIp2Exception | UnsupportedOperationException e) { info = null; - if (e instanceof AddressNotFoundException == false) { + if (e instanceof AddressNotFoundException) { LOG.debug("Could not get location from IP {}", address.getHostAddress(), e); lastError = e.getMessage(); } diff --git a/graylog2-server/src/main/java/org/graylog2/rest/resources/system/validate/GeoIpResolverConfigValidator.java b/graylog2-server/src/main/java/org/graylog2/rest/resources/system/validate/GeoIpResolverConfigValidator.java index a552f7eb31cf..3dcfbdb82c21 100644 --- a/graylog2-server/src/main/java/org/graylog2/rest/resources/system/validate/GeoIpResolverConfigValidator.java +++ b/graylog2-server/src/main/java/org/graylog2/rest/resources/system/validate/GeoIpResolverConfigValidator.java @@ -89,7 +89,7 @@ private void validateGeoIpAsnResolver(GeoIpResolverConfig config, Timer timer, I } asnResolver.getGeoIpData(testAddress); if (asnResolver.getLastError().isPresent()) { - String error = String.format(Locale.ENGLISH, "Error querying ASN. Make sure you have selected a valid ASN database for '%s'", config.databaseVendorType()); + String error = String.format(Locale.ENGLISH, "Error querying ASN. Make sure you have selected a valid ASN database type for '%s'", config.databaseVendorType()); throw new IllegalStateException(error); } } @@ -105,7 +105,7 @@ private void validateGeoIpLocationResolver(GeoIpResolverConfig config, Timer tim cityResolver.getGeoIpData(testAddress); if (cityResolver.getLastError().isPresent()) { - String error = String.format(Locale.ENGLISH, "Error querying Geo Location. Make sure you have selected a valid Location database for '%s'", config.databaseVendorType()); + String error = String.format(Locale.ENGLISH, "Error querying Geo Location. Make sure you have selected a valid database type for '%s'", config.databaseVendorType()); throw new IllegalStateException(error); } } From 46de5e68557a7e1ec042cade085a03280a5c7e02 Mon Sep 17 00:00:00 2001 From: Roberto Benitez Date: Thu, 6 Jan 2022 13:20:17 -0600 Subject: [PATCH 16/28] updated db migration to explicitly set all defaults. --- .../V20211221144300_GeoIpResolverConfigMigration.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/graylog2-server/src/main/java/org/graylog2/migrations/V20211221144300_GeoIpResolverConfigMigration.java b/graylog2-server/src/main/java/org/graylog2/migrations/V20211221144300_GeoIpResolverConfigMigration.java index b655fa714c8d..2c95fa7d4579 100644 --- a/graylog2-server/src/main/java/org/graylog2/migrations/V20211221144300_GeoIpResolverConfigMigration.java +++ b/graylog2-server/src/main/java/org/graylog2/migrations/V20211221144300_GeoIpResolverConfigMigration.java @@ -41,6 +41,8 @@ public class V20211221144300_GeoIpResolverConfigMigration extends Migration { private static final String FIELD_DB_TYPE = PAYLOAD + ".db_type"; private static final String FIELD_DB_PATH = PAYLOAD + ".db_path"; private static final String FIELD_CITY_DB_PATH = PAYLOAD + ".city_db_path"; + private static final String FIELD_ASN_DB_PATH = PAYLOAD + ".asn_db_path"; + private static final String FIELD_ENFORCE = PAYLOAD + ".enforce_graylog_schema"; private final MongoConnection mongoConnection; @@ -70,13 +72,19 @@ public void upgrade() { Bson geoConfFiler = Filters.eq("type", GeoIpResolverConfig.class.getCanonicalName()); Bson noColumnFilter = Filters.exists(FIELD_DB_VENDOR, false); + //set default value for 'enforce_graylog_schema' + Bson setEnforceSchema = Updates.set(FIELD_ENFORCE, false); + + //set blank asn db path + Bson setAsnPath = Updates.set(FIELD_ASN_DB_PATH, ""); + //rename db type field to db vendor type Bson renameDbTypeToVendor = Updates.rename(FIELD_DB_TYPE, FIELD_DB_VENDOR); //rename existing db_path field to city_db_path Bson renameDbPath = Updates.rename(FIELD_DB_PATH, FIELD_CITY_DB_PATH); - Bson updates = Updates.combine(renameDbTypeToVendor, renameDbPath); + Bson updates = Updates.combine(setEnforceSchema, renameDbTypeToVendor, renameDbPath, setAsnPath); LOG.info("Planned Updates: {}", updates); final UpdateResult updateResult = collection.updateOne(Filters.and(geoConfFiler, noColumnFilter), updates); LOG.info("Update Result: {}", updateResult); From c900c5b096a8b337081802302bcd3d91c3b82ce6 Mon Sep 17 00:00:00 2001 From: Mike Klein Date: Thu, 6 Jan 2022 15:20:53 -0500 Subject: [PATCH 17/28] Frontend functionality for the GeoIpResolverConfiguration - Updated GeoIpResolverConfiguration.jsx to include the new fields and have logical defaults for them in the ui. - When moving from disasbled to enabled it applies the logical defaults the location fields. - The refactoring of this component to bring it up to current standards will come as a seperate PR. --- .../configurations/GeoIpResolverConfig.jsx | 119 +++++++++++++----- 1 file changed, 88 insertions(+), 31 deletions(-) diff --git a/graylog2-web-interface/src/components/maps/configurations/GeoIpResolverConfig.jsx b/graylog2-web-interface/src/components/maps/configurations/GeoIpResolverConfig.jsx index 5a53b13149f2..4cb8c57aa153 100644 --- a/graylog2-web-interface/src/components/maps/configurations/GeoIpResolverConfig.jsx +++ b/graylog2-web-interface/src/components/maps/configurations/GeoIpResolverConfig.jsx @@ -19,10 +19,19 @@ import React from 'react'; import createReactClass from 'create-react-class'; import { IfPermitted, Select } from 'components/common'; -import { Button, BootstrapModalForm, Input } from 'components/bootstrap'; +import { Button, BootstrapModalForm, Col, Input, Row } from 'components/bootstrap'; import { DocumentationLink } from 'components/support'; import ObjectUtils from 'util/ObjectUtils'; +const defaultConfig = { + enabled: false, + enforce_graylog_schema: true, + db_vendor_type: 'MAXMIND', + city_db_path: '/etc/graylog/server/GeoLite2-City.mmdb', + asn_db_path: '/etc/graylog/server/asn.mmdb', + run_before_extractors: false, +}; + const GeoIpResolverConfig = createReactClass({ displayName: 'GeoIpResolverConfig', @@ -34,10 +43,7 @@ const GeoIpResolverConfig = createReactClass({ getDefaultProps() { return { config: { - enabled: false, - db_type: 'MAXMIND_CITY', - db_path: '/etc/graylog/server/GeoLite2-City.mmdb', - run_before_extractors: false, + ...defaultConfig, }, }; }, @@ -58,9 +64,16 @@ const GeoIpResolverConfig = createReactClass({ _updateConfigField(field, value) { const { config } = this.state; - const update = ObjectUtils.clone(config); + let update = ObjectUtils.clone(config); + + if (field === 'enabled' && value && config.city_db_path === '' && config.asn_db_path === '') { + update = { + ...defaultConfig, + }; + } update[field] = value; + this.setState({ config: update }); }, @@ -93,15 +106,22 @@ const GeoIpResolverConfig = createReactClass({ const { updateConfig } = this.props; const { config } = this.state; - updateConfig(config).then(() => { + const updatedConfig = { ...config }; + + if (!updatedConfig.enabled) { + updatedConfig.asn_db_path = ''; + updatedConfig.city_db_path = ''; + } + + updateConfig(updatedConfig).then(() => { this._closeModal(); }); }, _availableDatabaseTypes() { - // TODO: Support country database as well. return [ - { value: 'MAXMIND_CITY', label: 'City database' }, + { value: 'MAXMIND', label: 'MaxMind City Database' }, + { value: 'IPINFO', label: 'IPInfo Standard Location' }, ]; }, @@ -109,6 +129,14 @@ const GeoIpResolverConfig = createReactClass({ return this._availableDatabaseTypes().filter((t) => t.value === type)[0].label; }, + _onDbTypeSelect(value) { + const { config } = this.state; + const update = ObjectUtils.clone(config); + + update.db_vendor_type = value; + this.setState({ config: update }); + }, + render() { const { config } = this.state; @@ -124,11 +152,19 @@ const GeoIpResolverConfig = createReactClass({
Enabled:
-
{config.enabled === true ? 'yes' : 'no'}
-
Database type:
-
{this._activeDatabaseType(config.db_type)}
-
Database path:
-
{config.db_path}
+
{config.enabled === true ? 'Yes' : 'No'}
+ {config.enabled && ( + <> +
Default Graylog schema:
+
{config.enforce_graylog_schema === true ? 'Yes' : 'No'}
+
Database type:
+
{this._activeDatabaseType(config.db_vendor_type)}
+
Database path:
+
{config.city_db_path}
+
ASN database path:
+
{config.asn_db_path}
+ + )}
@@ -141,30 +177,51 @@ const GeoIpResolverConfig = createReactClass({ onModalClose={this._resetConfig} submitButtonText="Save">
- { this.inputs.configEnabled = elem; }} - label="Enable Geo-Location processor" - name="enabled" - checked={config.enabled} - onChange={this._onCheckboxClick('enabled', 'configEnabled')} /> + + + { this.inputs.configEnabled = elem; }} + label="Enable Geo-Location processor" + name="enabled" + checked={config.enabled} + onChange={this._onCheckboxClick('enabled', 'configEnabled')} /> + + + { this.inputs.enforceEnabled = elem; }} + label="Enforce default Graylog schema" + name="enforce_graylog_schema" + checked={config.enforce_graylog_schema} + onChange={this._onCheckboxClick('enforce_graylog_schema', 'enforceEnabled')} /> + + - - + You can download a free version of the database from MaxMind.} - name="db_path" - value={config.db_path} - onChange={this._onUpdate('db_path')} /> + disabled={!config.enabled} + label="Path to the ASN database" + name="asn_db_path" + value={config.asn_db_path} + onChange={this._onUpdate('asn_db_path')} />
From 4e2afe6fad69c2be63a10d0247e283517ea8e254 Mon Sep 17 00:00:00 2001 From: Roberto Benitez Date: Thu, 6 Jan 2022 14:46:53 -0600 Subject: [PATCH 18/28] Updated resolver issue that caused false positive during validation --- .../java/org/graylog/plugins/map/geoip/IpInfoIpAsnResolver.java | 2 +- .../org/graylog/plugins/map/geoip/IpInfoLocationResolver.java | 2 +- .../org/graylog/plugins/map/geoip/MaxMindIpAsnResolver.java | 2 +- .../graylog/plugins/map/geoip/MaxMindIpLocationResolver.java | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/IpInfoIpAsnResolver.java b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/IpInfoIpAsnResolver.java index 0db87f3f78a5..8200182f497d 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/IpInfoIpAsnResolver.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/IpInfoIpAsnResolver.java @@ -48,7 +48,7 @@ protected Optional doGetGeoIpData(InetAddress address) { info = GeoAsnInformation.create(ipInfoASN.name(), ipInfoASN.type(), ipInfoASN.asn()); } catch (IOException | AddressNotFoundException | UnsupportedOperationException e) { info = null; - if (e instanceof AddressNotFoundException == false) { + if (!(e instanceof AddressNotFoundException)) { String error = String.format(Locale.US, "Error getting ASN for IP Address '%s'. %s", address, e.getMessage()); LOG.warn(error, e); lastError = e.getMessage(); diff --git a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/IpInfoLocationResolver.java b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/IpInfoLocationResolver.java index 235b15ca7482..bef169e27991 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/IpInfoLocationResolver.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/IpInfoLocationResolver.java @@ -50,7 +50,7 @@ protected Optional doGetGeoIpData(InetAddress address) { } catch (NullPointerException | IOException | AddressNotFoundException | UnsupportedOperationException e) { info = null; - if (e instanceof AddressNotFoundException) { + if (!(e instanceof AddressNotFoundException)) { String error = String.format(Locale.US, "Error getting IP location info for '%s'. %s", address, e.getMessage()); LOG.error(error, e); lastError = e.getMessage(); diff --git a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/MaxMindIpAsnResolver.java b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/MaxMindIpAsnResolver.java index 03306dce3d4a..4fcab3f39298 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/MaxMindIpAsnResolver.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/MaxMindIpAsnResolver.java @@ -51,7 +51,7 @@ protected Optional doGetGeoIpData(InetAddress address) { } catch (GeoIp2Exception | IOException | UnsupportedOperationException e) { asn = null; - if (e instanceof AddressNotFoundException == false) { + if (!(e instanceof AddressNotFoundException)) { String error = String.format(Locale.US, "Error getting ASN for IP Address '%s'. %s", address, e.getMessage()); LOG.warn(error, e); lastError = e.getMessage(); diff --git a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/MaxMindIpLocationResolver.java b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/MaxMindIpLocationResolver.java index f8e621274acc..4fc9ebbd4368 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/MaxMindIpLocationResolver.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/MaxMindIpLocationResolver.java @@ -62,7 +62,7 @@ public Optional doGetGeoIpData(InetAddress address) { "N/A"); } catch (IOException | GeoIp2Exception | UnsupportedOperationException e) { info = null; - if (e instanceof AddressNotFoundException) { + if (!(e instanceof AddressNotFoundException)) { LOG.debug("Could not get location from IP {}", address.getHostAddress(), e); lastError = e.getMessage(); } From e1b667b3c7177524ab3727e73de7a36df1db604c Mon Sep 17 00:00:00 2001 From: Roberto Benitez Date: Mon, 10 Jan 2022 08:34:27 -0600 Subject: [PATCH 19/28] Updated GeoIpResolverEngineTest. Moved Cluster Validation to a more generic package for use elsewhere. --- .../map/geoip/GeoIpResolverEngine.java | 3 +- .../java/org/graylog2/commands/Server.java | 2 +- .../org/graylog2/plugin/PluginModule.java | 9 +- .../plugin/inject/Graylog2Module.java | 8 +- .../validate/ClusterConfigValidator.java | 2 +- .../ClusterConfigValidatorService.java | 2 +- .../validate/ConfigValidationException.java | 2 +- .../system/ClusterConfigResource.java | 4 +- .../ClusterConfigValidatorModule.java | 15 +- .../GeoIpResolverConfigValidator.java | 4 +- .../map/geoip/GeoIpResolverEngineTest.java | 243 ++++++++++++++---- .../map/geoip/GeoIpResolverFactoryTest.java | 87 ------- 12 files changed, 214 insertions(+), 167 deletions(-) rename graylog2-server/src/main/java/org/graylog2/{rest/resources/system => plugin}/validate/ClusterConfigValidator.java (94%) rename graylog2-server/src/main/java/org/graylog2/{rest/resources/system => plugin}/validate/ClusterConfigValidatorService.java (97%) rename graylog2-server/src/main/java/org/graylog2/{rest/resources/system => plugin}/validate/ConfigValidationException.java (93%) rename graylog2-server/src/main/java/org/graylog2/rest/resources/system/{validate => }/ClusterConfigValidatorModule.java (59%) rename graylog2-server/src/main/java/org/graylog2/rest/resources/system/{validate => }/GeoIpResolverConfigValidator.java (96%) delete mode 100644 graylog2-server/src/test/java/org/graylog/plugins/map/geoip/GeoIpResolverFactoryTest.java diff --git a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpResolverEngine.java b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpResolverEngine.java index c04da2e64b6c..c0aba08004c4 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpResolverEngine.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpResolverEngine.java @@ -143,7 +143,8 @@ private void addGeoIpDataIfPresent(Message message, InetAddress address, String * @param message message * @return a list of field that may have an IP address */ - private List getIpAddressFields(Message message) { + @VisibleForTesting + List getIpAddressFields(Message message) { return message.getFieldNames() .stream() .filter(e -> (!enforceGraylogSchema || ipAddressFields.containsKey(e)) diff --git a/graylog2-server/src/main/java/org/graylog2/commands/Server.java b/graylog2-server/src/main/java/org/graylog2/commands/Server.java index 4236ebfabae7..9ff878541d17 100644 --- a/graylog2-server/src/main/java/org/graylog2/commands/Server.java +++ b/graylog2-server/src/main/java/org/graylog2/commands/Server.java @@ -82,7 +82,7 @@ import org.graylog2.plugin.ServerStatus; import org.graylog2.plugin.Tools; import org.graylog2.plugin.system.NodeId; -import org.graylog2.rest.resources.system.validate.ClusterConfigValidatorModule; +import org.graylog2.rest.resources.system.ClusterConfigValidatorModule; import org.graylog2.shared.UI; import org.graylog2.shared.bindings.MessageInputBindings; import org.graylog2.shared.bindings.ObjectMapperModule; diff --git a/graylog2-server/src/main/java/org/graylog2/plugin/PluginModule.java b/graylog2-server/src/main/java/org/graylog2/plugin/PluginModule.java index 1205aa406db5..635bd69b15d0 100644 --- a/graylog2-server/src/main/java/org/graylog2/plugin/PluginModule.java +++ b/graylog2-server/src/main/java/org/graylog2/plugin/PluginModule.java @@ -42,7 +42,6 @@ import org.graylog2.audit.PluginAuditEventTypes; import org.graylog2.audit.formatter.AuditEventFormatter; import org.graylog2.contentpacks.constraints.ConstraintChecker; -import org.graylog2.contentpacks.facades.EntityFacade; import org.graylog2.contentpacks.facades.EntityWithExcerptFacade; import org.graylog2.contentpacks.model.ModelType; import org.graylog2.migrations.Migration; @@ -61,6 +60,7 @@ import org.graylog2.plugin.rest.PluginRestResource; import org.graylog2.plugin.security.PasswordAlgorithm; import org.graylog2.plugin.security.PluginPermissions; +import org.graylog2.plugin.validate.ClusterConfigValidator; import org.graylog2.shared.messageq.MessageQueueAcknowledger; import org.graylog2.shared.messageq.MessageQueueReader; import org.graylog2.shared.messageq.MessageQueueWriter; @@ -384,4 +384,11 @@ protected void bindMessageQueueImplementation(Class serviceBinder().addBinding().to((Class) service).in(Scopes.SINGLETON)); } + + + protected void addClusterConfigValidator(Class configClass, Class configValidatorClass) { + + mapBinder().addBinding(configClass).to(configValidatorClass); + + } } diff --git a/graylog2-server/src/main/java/org/graylog2/plugin/inject/Graylog2Module.java b/graylog2-server/src/main/java/org/graylog2/plugin/inject/Graylog2Module.java index 96bf86c251b5..3b682b95743e 100644 --- a/graylog2-server/src/main/java/org/graylog2/plugin/inject/Graylog2Module.java +++ b/graylog2-server/src/main/java/org/graylog2/plugin/inject/Graylog2Module.java @@ -33,7 +33,6 @@ import org.graylog2.audit.PluginAuditEventTypes; import org.graylog2.audit.formatter.AuditEventFormatter; import org.graylog2.contentpacks.constraints.ConstraintChecker; -import org.graylog2.contentpacks.facades.EntityFacade; import org.graylog2.contentpacks.facades.EntityWithExcerptFacade; import org.graylog2.contentpacks.model.ModelType; import org.graylog2.migrations.Migration; @@ -53,6 +52,7 @@ import org.graylog2.plugin.outputs.MessageOutput; import org.graylog2.plugin.security.PasswordAlgorithm; import org.graylog2.plugin.security.PluginPermissions; +import org.graylog2.plugin.validate.ClusterConfigValidator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -494,4 +494,10 @@ private Multibinder> systemRestResourceBinder() { Names.named(SYSTEM_REST_RESOURCES) ); } + + protected MapBinder, ClusterConfigValidator> mapBinder() { + TypeLiteral> keyType = new TypeLiteral>() {}; + TypeLiteral valueType = new TypeLiteral() {}; + return MapBinder.newMapBinder(binder(), keyType, valueType); + } } diff --git a/graylog2-server/src/main/java/org/graylog2/rest/resources/system/validate/ClusterConfigValidator.java b/graylog2-server/src/main/java/org/graylog2/plugin/validate/ClusterConfigValidator.java similarity index 94% rename from graylog2-server/src/main/java/org/graylog2/rest/resources/system/validate/ClusterConfigValidator.java rename to graylog2-server/src/main/java/org/graylog2/plugin/validate/ClusterConfigValidator.java index 83ff4561ac15..49d76ddfdbca 100644 --- a/graylog2-server/src/main/java/org/graylog2/rest/resources/system/validate/ClusterConfigValidator.java +++ b/graylog2-server/src/main/java/org/graylog2/plugin/validate/ClusterConfigValidator.java @@ -15,7 +15,7 @@ * . */ -package org.graylog2.rest.resources.system.validate; +package org.graylog2.plugin.validate; /** * Specification for a Cluster Configuration object. diff --git a/graylog2-server/src/main/java/org/graylog2/rest/resources/system/validate/ClusterConfigValidatorService.java b/graylog2-server/src/main/java/org/graylog2/plugin/validate/ClusterConfigValidatorService.java similarity index 97% rename from graylog2-server/src/main/java/org/graylog2/rest/resources/system/validate/ClusterConfigValidatorService.java rename to graylog2-server/src/main/java/org/graylog2/plugin/validate/ClusterConfigValidatorService.java index b368af308d29..4a19415a0fdb 100644 --- a/graylog2-server/src/main/java/org/graylog2/rest/resources/system/validate/ClusterConfigValidatorService.java +++ b/graylog2-server/src/main/java/org/graylog2/plugin/validate/ClusterConfigValidatorService.java @@ -15,7 +15,7 @@ * . */ -package org.graylog2.rest.resources.system.validate; +package org.graylog2.plugin.validate; import javax.inject.Inject; import java.util.Map; diff --git a/graylog2-server/src/main/java/org/graylog2/rest/resources/system/validate/ConfigValidationException.java b/graylog2-server/src/main/java/org/graylog2/plugin/validate/ConfigValidationException.java similarity index 93% rename from graylog2-server/src/main/java/org/graylog2/rest/resources/system/validate/ConfigValidationException.java rename to graylog2-server/src/main/java/org/graylog2/plugin/validate/ConfigValidationException.java index dd7ef7fbe519..a13ab2254da4 100644 --- a/graylog2-server/src/main/java/org/graylog2/rest/resources/system/validate/ConfigValidationException.java +++ b/graylog2-server/src/main/java/org/graylog2/plugin/validate/ConfigValidationException.java @@ -15,7 +15,7 @@ * . */ -package org.graylog2.rest.resources.system.validate; +package org.graylog2.plugin.validate; public class ConfigValidationException extends Exception { public ConfigValidationException(String message) { diff --git a/graylog2-server/src/main/java/org/graylog2/rest/resources/system/ClusterConfigResource.java b/graylog2-server/src/main/java/org/graylog2/rest/resources/system/ClusterConfigResource.java index 5d148f61c60e..813bf771127c 100644 --- a/graylog2-server/src/main/java/org/graylog2/rest/resources/system/ClusterConfigResource.java +++ b/graylog2-server/src/main/java/org/graylog2/rest/resources/system/ClusterConfigResource.java @@ -29,10 +29,10 @@ import org.graylog2.audit.AuditEventTypes; import org.graylog2.audit.jersey.AuditEvent; import org.graylog2.plugin.cluster.ClusterConfigService; +import org.graylog2.plugin.validate.ClusterConfigValidatorService; +import org.graylog2.plugin.validate.ConfigValidationException; import org.graylog2.rest.MoreMediaTypes; import org.graylog2.rest.models.system.config.ClusterConfigList; -import org.graylog2.rest.resources.system.validate.ClusterConfigValidatorService; -import org.graylog2.rest.resources.system.validate.ConfigValidationException; import org.graylog2.shared.plugins.ChainingClassLoader; import org.graylog2.shared.rest.resources.RestResource; import org.graylog2.shared.security.RestPermissions; diff --git a/graylog2-server/src/main/java/org/graylog2/rest/resources/system/validate/ClusterConfigValidatorModule.java b/graylog2-server/src/main/java/org/graylog2/rest/resources/system/ClusterConfigValidatorModule.java similarity index 59% rename from graylog2-server/src/main/java/org/graylog2/rest/resources/system/validate/ClusterConfigValidatorModule.java rename to graylog2-server/src/main/java/org/graylog2/rest/resources/system/ClusterConfigValidatorModule.java index 8c17e203ccd9..b35e4fa975c2 100644 --- a/graylog2-server/src/main/java/org/graylog2/rest/resources/system/validate/ClusterConfigValidatorModule.java +++ b/graylog2-server/src/main/java/org/graylog2/rest/resources/system/ClusterConfigValidatorModule.java @@ -15,10 +15,8 @@ * . */ -package org.graylog2.rest.resources.system.validate; +package org.graylog2.rest.resources.system; -import com.google.inject.TypeLiteral; -import com.google.inject.multibindings.MapBinder; import org.graylog.plugins.map.config.GeoIpResolverConfig; import org.graylog2.plugin.PluginModule; @@ -29,15 +27,4 @@ public void configure() { addClusterConfigValidator(GeoIpResolverConfig.class, GeoIpResolverConfigValidator.class); } - private void addClusterConfigValidator(Class configClass, Class configValidatorClass) { - - mapBinder().addBinding(configClass).to(configValidatorClass); - - } - - private MapBinder, ClusterConfigValidator> mapBinder() { - TypeLiteral> keyType = new TypeLiteral>() {}; - TypeLiteral valueType = new TypeLiteral() {}; - return MapBinder.newMapBinder(binder(), keyType, valueType); - } } diff --git a/graylog2-server/src/main/java/org/graylog2/rest/resources/system/validate/GeoIpResolverConfigValidator.java b/graylog2-server/src/main/java/org/graylog2/rest/resources/system/GeoIpResolverConfigValidator.java similarity index 96% rename from graylog2-server/src/main/java/org/graylog2/rest/resources/system/validate/GeoIpResolverConfigValidator.java rename to graylog2-server/src/main/java/org/graylog2/rest/resources/system/GeoIpResolverConfigValidator.java index 3dcfbdb82c21..6d1ec28e9830 100644 --- a/graylog2-server/src/main/java/org/graylog2/rest/resources/system/validate/GeoIpResolverConfigValidator.java +++ b/graylog2-server/src/main/java/org/graylog2/rest/resources/system/GeoIpResolverConfigValidator.java @@ -15,7 +15,7 @@ * . */ -package org.graylog2.rest.resources.system.validate; +package org.graylog2.rest.resources.system; import com.codahale.metrics.Timer; import com.codahale.metrics.UniformReservoir; @@ -25,6 +25,8 @@ import org.graylog.plugins.map.geoip.GeoIpResolver; import org.graylog.plugins.map.geoip.GeoIpVendorResolverService; import org.graylog.plugins.map.geoip.GeoLocationInformation; +import org.graylog2.plugin.validate.ClusterConfigValidator; +import org.graylog2.plugin.validate.ConfigValidationException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/graylog2-server/src/test/java/org/graylog/plugins/map/geoip/GeoIpResolverEngineTest.java b/graylog2-server/src/test/java/org/graylog/plugins/map/geoip/GeoIpResolverEngineTest.java index eb8af41a003f..b3d9ad7278f0 100644 --- a/graylog2-server/src/test/java/org/graylog/plugins/map/geoip/GeoIpResolverEngineTest.java +++ b/graylog2-server/src/test/java/org/graylog/plugins/map/geoip/GeoIpResolverEngineTest.java @@ -18,46 +18,88 @@ import com.codahale.metrics.MetricFilter; import com.codahale.metrics.MetricRegistry; +import com.codahale.metrics.Timer; import com.eaio.uuid.UUID; import com.google.common.collect.Maps; import com.google.common.net.InetAddresses; -import org.graylog.plugins.map.ConditionalRunner; -import org.graylog.plugins.map.ResourceExistsCondition; +import org.graylog.plugins.map.config.DatabaseVendorType; import org.graylog.plugins.map.config.GeoIpResolverConfig; import org.graylog2.plugin.Message; import org.junit.After; import org.junit.Before; import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.Assertions; import org.mockito.Mock; import org.mockito.MockitoAnnotations; -import java.net.URISyntaxException; -import java.net.URL; +import java.net.InetAddress; +import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.Optional; -import static com.codahale.metrics.MetricRegistry.name; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; -@RunWith(ConditionalRunner.class) -@ResourceExistsCondition(GeoIpResolverEngineTest.GEO_LITE2_CITY_MMDB) public class GeoIpResolverEngineTest { - static final String GEO_LITE2_CITY_MMDB = "/GeoLite2-City.mmdb"; + + @Mock + private GeoIpResolver maxMindCityResolver; + + @Mock + private GeoIpResolver maxMindAsnResolver; + + @Mock + private GeoIpResolver ipInfoCityResolver; + + @Mock + private GeoIpResolver ipInfoAsnResolver; @Mock GeoIpVendorResolverService geoIpVendorResolverService; + + private final GeoLocationInformation maxMindLocationInfo = GeoLocationInformation.create(1, 2, "US", "USA", "Houston", "Texas", "America/Chicago"); + private final GeoLocationInformation ipInfoLocationInfo = GeoLocationInformation.create(1, 2, "US", "N/A", "Houston", "Texas", "America/Chicago"); + private final GeoAsnInformation maxMindAsnInfo = GeoAsnInformation.create("Scamcast", "", "1000"); + private final GeoAsnInformation ipInfoAsnInfo = GeoAsnInformation.create("Scamcast", "001", "1000"); + private MetricRegistry metricRegistry; private GeoIpResolverConfig config; + private InetAddress reservedIp; + private InetAddress publicIp; + @Before public void setUp() throws Exception { + reservedIp = InetAddress.getByName("127.0.0.1"); + publicIp = InetAddress.getByName("96.110.152.253"); + MockitoAnnotations.openMocks(this).close(); - config = GeoIpResolverConfig.defaultConfig().toBuilder().enabled(true).cityDbPath(this.getTestDatabasePath()).build(); + + when(maxMindAsnResolver.isEnabled()).thenReturn(true); + when(maxMindCityResolver.getGeoIpData(publicIp)) + .thenReturn(Optional.of(maxMindLocationInfo)); + when(maxMindAsnResolver.getGeoIpData(publicIp)) + .thenReturn(Optional.of(maxMindAsnInfo)); + when(maxMindCityResolver.isEnabled()).thenReturn(true); + + when(geoIpVendorResolverService.createCityResolver(any(GeoIpResolverConfig.class), any(Timer.class))) + .thenReturn(maxMindCityResolver); + when(geoIpVendorResolverService.createAsnResolver(any(GeoIpResolverConfig.class), any(Timer.class))) + .thenReturn(maxMindAsnResolver); + + + config = GeoIpResolverConfig.defaultConfig().toBuilder() + .enforceGraylogSchema(true) + .enabled(true) + .cityDbPath("") + .build(); + metricRegistry = new MetricRegistry(); } @@ -67,84 +109,173 @@ public void tearDown() { metricRegistry = null; } - private String getTestDatabasePath() throws URISyntaxException { - final URL url = getClass().getResource(GEO_LITE2_CITY_MMDB); - if (url == null) { - throw new IllegalStateException("Resource not found. " + GEO_LITE2_CITY_MMDB); - } - return url.toURI().getPath(); + @Test + public void testGetIpAddressFieldsEnforceGraylogSchema() { + + GeoIpResolverConfig conf = config.toBuilder().enforceGraylogSchema(true).build(); + final GeoIpResolverEngine engine = new GeoIpResolverEngine(geoIpVendorResolverService, conf, metricRegistry); + + Map fields = new HashMap<>(); + fields.put("_id", java.util.UUID.randomUUID().toString()); + fields.put("source_ip", "127.0.0.1"); + fields.put("src_ip", "127.0.0.1"); + fields.put("destination_ip", "127.0.0.1"); + fields.put("dest_ip", "127.0.0.1"); + fields.put("gl2_test", "127.0.0.1"); + + + Message message = new Message(fields); + List ipFields = engine.getIpAddressFields(message); + + //with the Graylog Schema enforced, only the source_ip and destination_ip should be returned + Assertions.assertEquals(2, ipFields.size()); + Assertions.assertTrue(ipFields.contains("source_ip")); + Assertions.assertTrue(ipFields.contains("destination_ip")); + } @Test - public void getIpFromFieldValue() { - final GeoIpResolverEngine resolver = new GeoIpResolverEngine(geoIpVendorResolverService, config, metricRegistry); - final String ip = "127.0.0.1"; + public void testGetIpAddressFieldsEnforceGraylogSchemaFalse() { + + GeoIpResolverConfig conf = config.toBuilder().enforceGraylogSchema(false).build(); + final GeoIpResolverEngine engine = new GeoIpResolverEngine(geoIpVendorResolverService, conf, metricRegistry); + + Map fields = new HashMap<>(); + fields.put("_id", java.util.UUID.randomUUID().toString()); + fields.put("source_ip", "127.0.0.1"); + fields.put("src_ip", "127.0.0.1"); + fields.put("destination_ip", "127.0.0.1"); + fields.put("dest_ip", "127.0.0.1"); + fields.put("gl2_test", "127.0.0.1"); + + + Message message = new Message(fields); + List ipFields = engine.getIpAddressFields(message); + + //without enforcing the Graylog Schema, all but the gl2_* fields should be returned. + Assertions.assertEquals(5, ipFields.size()); - assertEquals(InetAddresses.forString(ip), resolver.getIpFromFieldValue(ip)); - assertNull(resolver.getIpFromFieldValue("Message from \"127.0.0.1\"")); - assertNull(resolver.getIpFromFieldValue("Test message with no IP")); } @Test - public void trimFieldValueBeforeLookup() { - final GeoIpResolverEngine resolver = new GeoIpResolverEngine(geoIpVendorResolverService, config, metricRegistry); - final String ip = " 2001:4860:4860::8888\t\n"; + public void testFilterMaxMind() { + + final GeoIpResolverEngine engine = new GeoIpResolverEngine(geoIpVendorResolverService, config, metricRegistry); + + Map fields = new HashMap<>(); + fields.put("_id", java.util.UUID.randomUUID().toString()); + fields.put("source_ip", publicIp.getHostAddress()); + + Message message = new Message(fields); + engine.filter(message); + + String expectedGeoName = maxMindLocationInfo.cityName() + ", " + maxMindLocationInfo.countryIsoCode(); + Assertions.assertEquals(expectedGeoName, message.getField("source_geo_name")); + Assertions.assertEquals(maxMindLocationInfo.region(), message.getField("source_geo_region")); + Assertions.assertEquals(maxMindLocationInfo.cityName(), message.getField("source_geo_city")); + Assertions.assertEquals(maxMindLocationInfo.timeZone(), message.getField("source_geo_timezone")); + Assertions.assertEquals(maxMindLocationInfo.countryName(), message.getField("source_geo_country")); + Assertions.assertEquals(maxMindAsnInfo.organization(), message.getField("source_as_organization")); + Assertions.assertEquals(maxMindAsnInfo.asn(), message.getField("source_as_number")); - assertNotNull(resolver.getIpFromFieldValue(ip)); } @Test - public void disabledFilterTest() { - final GeoIpResolverEngine resolver = new GeoIpResolverEngine(geoIpVendorResolverService, config.toBuilder().enabled(false).build(), metricRegistry); + public void testFilterIpInfo() { - final Map messageFields = Maps.newHashMap(); - messageFields.put("_id", (new UUID()).toString()); - messageFields.put("source", "192.168.0.1"); - messageFields.put("message", "Hello from 1.2.3.4"); - messageFields.put("extracted_ip", "1.2.3.4"); - messageFields.put("ipv6", "2001:4860:4860::8888"); + when(ipInfoAsnResolver.isEnabled()).thenReturn(true); + when(ipInfoAsnResolver.getGeoIpData(publicIp)).thenReturn(Optional.of(ipInfoAsnInfo)); + when(ipInfoCityResolver.isEnabled()).thenReturn(true); + when(ipInfoCityResolver.getGeoIpData(publicIp)).thenReturn(Optional.of(ipInfoLocationInfo)); - final Message message = new Message(messageFields); - final boolean filtered = resolver.filter(message); + when(geoIpVendorResolverService.createCityResolver(any(GeoIpResolverConfig.class), any(Timer.class))) + .thenReturn(ipInfoCityResolver); + when(geoIpVendorResolverService.createAsnResolver(any(GeoIpResolverConfig.class), any(Timer.class))) + .thenReturn(ipInfoAsnResolver); - assertFalse("Message should not be filtered out", filtered); - assertEquals("Filter should not add new message fields", messageFields.size(), message.getFields().size()); + GeoIpResolverConfig conf = config.toBuilder() + .databaseVendorType(DatabaseVendorType.IPINFO) + .build(); + + final GeoIpResolverEngine engine = new GeoIpResolverEngine(geoIpVendorResolverService, conf, metricRegistry); + + Map fields = new HashMap<>(); + fields.put("_id", java.util.UUID.randomUUID().toString()); + fields.put("source_ip", publicIp.getHostAddress()); + + Message message = new Message(fields); + engine.filter(message); + + String expectedGeoName = ipInfoLocationInfo.cityName() + ", " + ipInfoLocationInfo.countryIsoCode(); + Assertions.assertEquals(expectedGeoName, message.getField("source_geo_name")); + Assertions.assertEquals(ipInfoLocationInfo.region(), message.getField("source_geo_region")); + Assertions.assertEquals(ipInfoLocationInfo.cityName(), message.getField("source_geo_city")); + Assertions.assertEquals(ipInfoLocationInfo.timeZone(), message.getField("source_geo_timezone")); + Assertions.assertFalse(message.hasField("source_geo_country")); + Assertions.assertEquals(ipInfoLocationInfo.countryIsoCode(), message.getField("source_geo_country_iso")); + Assertions.assertEquals(ipInfoAsnInfo.organization(), message.getField("source_as_organization")); + Assertions.assertEquals(ipInfoAsnInfo.asn(), message.getField("source_as_number")); } - private void assertFieldNotResolved(Message message, String fieldName, String errorMessage) { - assertNull(errorMessage + " coordinates in " + fieldName, message.getField(fieldName + "_geolocation")); - assertNull(errorMessage + " country in " + fieldName, message.getField(fieldName + "_country_code")); - assertNull(errorMessage + " city in " + fieldName, message.getField(fieldName + "_city_name")); + @Test + public void testFilterWithReservedIpAddress() { + + final GeoIpResolverEngine engine = new GeoIpResolverEngine(geoIpVendorResolverService, config, metricRegistry); + + Map fields = new HashMap<>(); + fields.put("_id", java.util.UUID.randomUUID().toString()); + fields.put("source_ip", "127.0.0.1"); + + Message message = new Message(fields); + engine.filter(message); + Assertions.assertTrue(message.hasField("source_reserved_ip")); + } - private void assertFieldResolved(Message message, String fieldName, String errorMessage) { - assertNotNull(errorMessage + " coordinates in " + fieldName, message.getField(fieldName + "_geolocation")); - assertNotNull(errorMessage + " country in " + fieldName, message.getField(fieldName + "_country_code")); - assertNotNull(errorMessage + " city in " + fieldName, message.getField(fieldName + "_city_name")); - assertTrue("Location coordinates for " + fieldName + " should include a comma", ((String) message.getField(fieldName + "_geolocation")).contains(",")); + @Test + public void getIpFromFieldValue() { + + when(geoIpVendorResolverService.createCityResolver(any(GeoIpResolverConfig.class), any(Timer.class))) + .thenReturn(maxMindCityResolver); + + when(geoIpVendorResolverService.createAsnResolver(any(GeoIpResolverConfig.class), any(Timer.class))) + .thenReturn(maxMindAsnResolver); + + final GeoIpResolverEngine engine = new GeoIpResolverEngine(geoIpVendorResolverService, config, metricRegistry); + final String ip = "127.0.0.1"; + + assertEquals(InetAddresses.forString(ip), engine.getIpFromFieldValue(ip)); + assertNull(engine.getIpFromFieldValue("Message from \"127.0.0.1\"")); + assertNull(engine.getIpFromFieldValue("Test message with no IP")); } @Test - public void filterResolvesIpGeoLocation() { + public void trimFieldValueBeforeLookup() { final GeoIpResolverEngine resolver = new GeoIpResolverEngine(geoIpVendorResolverService, config, metricRegistry); + final String ip = " 2001:4860:4860::8888\t\n"; + + assertNotNull(resolver.getIpFromFieldValue(ip)); + } + + @Test + public void disabledFilterTest() { + + when(maxMindCityResolver.isEnabled()).thenReturn(false); + when(maxMindAsnResolver.isEnabled()).thenReturn(false); + + final GeoIpResolverEngine resolver = new GeoIpResolverEngine(geoIpVendorResolverService, config.toBuilder().enabled(false).build(), metricRegistry); final Map messageFields = Maps.newHashMap(); messageFields.put("_id", (new UUID()).toString()); messageFields.put("source", "192.168.0.1"); messageFields.put("message", "Hello from 1.2.3.4"); messageFields.put("extracted_ip", "1.2.3.4"); - messageFields.put(Message.FIELD_GL2_REMOTE_IP, "1.2.3.4"); messageFields.put("ipv6", "2001:4860:4860::8888"); final Message message = new Message(messageFields); final boolean filtered = resolver.filter(message); assertFalse("Message should not be filtered out", filtered); - assertEquals("Should have looked up three IPs", 3, metricRegistry.timer(name(GeoIpResolverEngine.class, "resolveTime")).getCount()); - assertFieldNotResolved(message, "source", "Should not have resolved private IP"); - assertFieldNotResolved(message, "message", "Should have resolved public IP"); - assertFieldNotResolved(message, "gl2_remote_ip", "Should not have resolved text with an IP"); - assertFieldResolved(message, "extracted_ip", "Should have resolved public IP"); - assertFieldResolved(message, "ipv6", "Should have resolved public IPv6"); + assertEquals("Filter should not add new message fields", messageFields.size(), message.getFields().size()); } } diff --git a/graylog2-server/src/test/java/org/graylog/plugins/map/geoip/GeoIpResolverFactoryTest.java b/graylog2-server/src/test/java/org/graylog/plugins/map/geoip/GeoIpResolverFactoryTest.java deleted file mode 100644 index 4a8059729f56..000000000000 --- a/graylog2-server/src/test/java/org/graylog/plugins/map/geoip/GeoIpResolverFactoryTest.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright (C) 2020 Graylog, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * . - */ - -package org.graylog.plugins.map.geoip; - -import com.codahale.metrics.MetricFilter; -import com.codahale.metrics.MetricRegistry; -import com.codahale.metrics.Timer; -import org.graylog.plugins.map.config.DatabaseVendorType; -import org.graylog.plugins.map.config.GeoIpResolverConfig; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.mockito.Mock; - -/** - * Test that the factory creates the appropriate {@link GeoIpResolver}. Each resolver is expected to fail to create a data provider, but should - * succeed in creating a disabled instance. - */ - -class GeoIpResolverFactoryTest { - - - @Mock - private MetricRegistry metricRegistry; - private Timer timer; - private GeoIpResolverFactory geoIpResolverFactory; - - @BeforeEach - void setup() { - metricRegistry = new MetricRegistry(); - timer = metricRegistry.timer("ResolverFactoryUnitTest"); - - } - - @AfterEach - void tearDown() { - - metricRegistry.removeMatching(MetricFilter.ALL); - metricRegistry = null; - } - - void testMaxMindVendor() { - - GeoIpResolverConfig config = createConfig(DatabaseVendorType.MAXMIND); - - GeoIpResolver cityResolver = geoIpResolverFactory.createIpInfoCityResolver(timer, config.cityDbPath(), config.enabled()); - Assertions.assertTrue(cityResolver instanceof MaxMindIpLocationResolver); - - GeoIpResolver asnResolver = geoIpResolverFactory.createIpInfoAsnResolver(timer, config.asnDbPath(), config.enabled()); - Assertions.assertTrue(asnResolver instanceof MaxMindIpAsnResolver); - } - - void testIpInfoVendor() { - - GeoIpResolverConfig config = createConfig(DatabaseVendorType.IPINFO); - - GeoIpResolver cityResolver = geoIpResolverFactory.createIpInfoCityResolver(timer, config.cityDbPath(), config.enabled()); - Assertions.assertTrue(cityResolver instanceof IpInfoLocationResolver); - - GeoIpResolver asnResolver = geoIpResolverFactory.createIpInfoAsnResolver(timer, config.asnDbPath(), config.enabled()); - Assertions.assertTrue(asnResolver instanceof IpInfoIpAsnResolver); - } - - private GeoIpResolverConfig createConfig(DatabaseVendorType vendorType) { - return GeoIpResolverConfig.defaultConfig().toBuilder() - .enabled(true) - .databaseVendorType(vendorType) - .cityDbPath("") - .asnDbPath("") - .build(); - } -} From e6fd2481792f784456e5d4ae52e12c1ff9ed2d94 Mon Sep 17 00:00:00 2001 From: Roberto Benitez Date: Mon, 10 Jan 2022 09:35:16 -0600 Subject: [PATCH 20/28] renamed method Graylog2Module::mapBinder to clusterConfigMapBinder --- .../src/main/java/org/graylog2/plugin/PluginModule.java | 2 +- .../main/java/org/graylog2/plugin/inject/Graylog2Module.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/graylog2-server/src/main/java/org/graylog2/plugin/PluginModule.java b/graylog2-server/src/main/java/org/graylog2/plugin/PluginModule.java index 635bd69b15d0..36c22b7cc475 100644 --- a/graylog2-server/src/main/java/org/graylog2/plugin/PluginModule.java +++ b/graylog2-server/src/main/java/org/graylog2/plugin/PluginModule.java @@ -388,7 +388,7 @@ protected void bindMessageQueueImplementation(Class configClass, Class configValidatorClass) { - mapBinder().addBinding(configClass).to(configValidatorClass); + clusterConfigMapBinder().addBinding(configClass).to(configValidatorClass); } } diff --git a/graylog2-server/src/main/java/org/graylog2/plugin/inject/Graylog2Module.java b/graylog2-server/src/main/java/org/graylog2/plugin/inject/Graylog2Module.java index 3b682b95743e..6e1478c18736 100644 --- a/graylog2-server/src/main/java/org/graylog2/plugin/inject/Graylog2Module.java +++ b/graylog2-server/src/main/java/org/graylog2/plugin/inject/Graylog2Module.java @@ -495,7 +495,7 @@ private Multibinder> systemRestResourceBinder() { ); } - protected MapBinder, ClusterConfigValidator> mapBinder() { + protected MapBinder, ClusterConfigValidator> clusterConfigMapBinder() { TypeLiteral> keyType = new TypeLiteral>() {}; TypeLiteral valueType = new TypeLiteral() {}; return MapBinder.newMapBinder(binder(), keyType, valueType); From 33f1829c2e331dfcd2cccdbf2d06d0cfcd5aaf36 Mon Sep 17 00:00:00 2001 From: Roberto Benitez Date: Tue, 11 Jan 2022 08:15:04 -0600 Subject: [PATCH 21/28] Removed unused dependency --- graylog2-server/pom.xml | 5 ----- pom.xml | 1 - 2 files changed, 6 deletions(-) diff --git a/graylog2-server/pom.xml b/graylog2-server/pom.xml index 83d27b4885ec..071b7e740b88 100644 --- a/graylog2-server/pom.xml +++ b/graylog2-server/pom.xml @@ -564,11 +564,6 @@ com.maxmind.geoip2 geoip2 - - io.ipinfo - ipinfo-api - ${ipinfo.version} - org.graylog.cef cef-parser diff --git a/pom.xml b/pom.xml index c17d4feb482f..8397d73f325a 100644 --- a/pom.xml +++ b/pom.xml @@ -114,7 +114,6 @@ 2.1.12 6.1.2.Final 2.6.1 - 2.1 2.9.10.20200411 0.13.0 0.9.0 From 355c2c4405e3d36c7355183a891f6faa2ae623fa Mon Sep 17 00:00:00 2001 From: Roberto Benitez Date: Wed, 12 Jan 2022 15:13:28 -0600 Subject: [PATCH 22/28] Updated database/vendor type labels --- .../maps/configurations/GeoIpResolverConfig.jsx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/graylog2-web-interface/src/components/maps/configurations/GeoIpResolverConfig.jsx b/graylog2-web-interface/src/components/maps/configurations/GeoIpResolverConfig.jsx index 52f82e88b1e3..ffa931cf2be1 100644 --- a/graylog2-web-interface/src/components/maps/configurations/GeoIpResolverConfig.jsx +++ b/graylog2-web-interface/src/components/maps/configurations/GeoIpResolverConfig.jsx @@ -120,8 +120,8 @@ const GeoIpResolverConfig = createReactClass({ _availableDatabaseTypes() { return [ - { value: 'MAXMIND', label: 'MaxMind City Database' }, - { value: 'IPINFO', label: 'IPInfo Standard Location' }, + { value: 'MAXMIND', label: 'MaxMind' }, + { value: 'IPINFO', label: 'IPInfo' }, ]; }, @@ -157,9 +157,9 @@ const GeoIpResolverConfig = createReactClass({ <>
Default Graylog schema:
{config.enforce_graylog_schema === true ? 'Yes' : 'No'}
-
Database type:
+
Database vendor type:
{this._activeDatabaseType(config.db_vendor_type)}
-
Database path:
+
City database path:
{config.city_db_path}
ASN database path:
{config.asn_db_path}
From 578b45d2e25ad7624c6479300727f866cad516c3 Mon Sep 17 00:00:00 2001 From: Roberto Benitez Date: Thu, 13 Jan 2022 13:02:26 -0600 Subject: [PATCH 23/28] Updated defaultConfig on UI side --- .../src/components/maps/configurations/GeoIpResolverConfig.jsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/graylog2-web-interface/src/components/maps/configurations/GeoIpResolverConfig.jsx b/graylog2-web-interface/src/components/maps/configurations/GeoIpResolverConfig.jsx index ffa931cf2be1..7a5ee0eeff08 100644 --- a/graylog2-web-interface/src/components/maps/configurations/GeoIpResolverConfig.jsx +++ b/graylog2-web-interface/src/components/maps/configurations/GeoIpResolverConfig.jsx @@ -28,7 +28,7 @@ const defaultConfig = { enforce_graylog_schema: true, db_vendor_type: 'MAXMIND', city_db_path: '/etc/graylog/server/GeoLite2-City.mmdb', - asn_db_path: '/etc/graylog/server/asn.mmdb', + asn_db_path: '/etc/graylog/server/GeoLite2-ASN.mmdb', run_before_extractors: false, }; From 193f991d343b89640ab8fce15e500ada66a8a795 Mon Sep 17 00:00:00 2001 From: Roberto Benitez Date: Thu, 13 Jan 2022 15:19:05 -0600 Subject: [PATCH 24/28] Updated DB migration V20211221144300_GeoIpResolverConfigMigration to set default vendor type after renaming old column --- .../V20211221144300_GeoIpResolverConfigMigration.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/graylog2-server/src/main/java/org/graylog2/migrations/V20211221144300_GeoIpResolverConfigMigration.java b/graylog2-server/src/main/java/org/graylog2/migrations/V20211221144300_GeoIpResolverConfigMigration.java index 2c95fa7d4579..e3e7c58175de 100644 --- a/graylog2-server/src/main/java/org/graylog2/migrations/V20211221144300_GeoIpResolverConfigMigration.java +++ b/graylog2-server/src/main/java/org/graylog2/migrations/V20211221144300_GeoIpResolverConfigMigration.java @@ -81,10 +81,12 @@ public void upgrade() { //rename db type field to db vendor type Bson renameDbTypeToVendor = Updates.rename(FIELD_DB_TYPE, FIELD_DB_VENDOR); + Bson setDefaultVendor = Updates.set(FIELD_DB_VENDOR, DatabaseVendorType.MAXMIND); + //rename existing db_path field to city_db_path Bson renameDbPath = Updates.rename(FIELD_DB_PATH, FIELD_CITY_DB_PATH); - Bson updates = Updates.combine(setEnforceSchema, renameDbTypeToVendor, renameDbPath, setAsnPath); + Bson updates = Updates.combine(setEnforceSchema, renameDbTypeToVendor, setDefaultVendor, renameDbPath, setAsnPath); LOG.info("Planned Updates: {}", updates); final UpdateResult updateResult = collection.updateOne(Filters.and(geoConfFiler, noColumnFilter), updates); LOG.info("Update Result: {}", updateResult); From 4c6efcd1fcc2844ebceb790d47f782fbf318a01c Mon Sep 17 00:00:00 2001 From: Roberto Benitez Date: Thu, 13 Jan 2022 15:37:13 -0600 Subject: [PATCH 25/28] Updated DB migration V20211221144300_GeoIpResolverConfigMigration to set default vendor after having renamed field in separate update. --- ...20211221144300_GeoIpResolverConfigMigration.java | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/graylog2-server/src/main/java/org/graylog2/migrations/V20211221144300_GeoIpResolverConfigMigration.java b/graylog2-server/src/main/java/org/graylog2/migrations/V20211221144300_GeoIpResolverConfigMigration.java index e3e7c58175de..be2ca63db44d 100644 --- a/graylog2-server/src/main/java/org/graylog2/migrations/V20211221144300_GeoIpResolverConfigMigration.java +++ b/graylog2-server/src/main/java/org/graylog2/migrations/V20211221144300_GeoIpResolverConfigMigration.java @@ -69,7 +69,7 @@ public void upgrade() { final MongoCollection collection = mongoConnection.getMongoDatabase().getCollection(COLLECTION_NAME); LOG.info("Updating '{}' collection.", COLLECTION_NAME); - Bson geoConfFiler = Filters.eq("type", GeoIpResolverConfig.class.getCanonicalName()); + Bson geoConfFilter = Filters.eq("type", GeoIpResolverConfig.class.getCanonicalName()); Bson noColumnFilter = Filters.exists(FIELD_DB_VENDOR, false); //set default value for 'enforce_graylog_schema' @@ -81,16 +81,19 @@ public void upgrade() { //rename db type field to db vendor type Bson renameDbTypeToVendor = Updates.rename(FIELD_DB_TYPE, FIELD_DB_VENDOR); - Bson setDefaultVendor = Updates.set(FIELD_DB_VENDOR, DatabaseVendorType.MAXMIND); - //rename existing db_path field to city_db_path Bson renameDbPath = Updates.rename(FIELD_DB_PATH, FIELD_CITY_DB_PATH); - Bson updates = Updates.combine(setEnforceSchema, renameDbTypeToVendor, setDefaultVendor, renameDbPath, setAsnPath); + Bson updates = Updates.combine(setEnforceSchema, renameDbTypeToVendor, renameDbPath, setAsnPath); LOG.info("Planned Updates: {}", updates); - final UpdateResult updateResult = collection.updateOne(Filters.and(geoConfFiler, noColumnFilter), updates); + final UpdateResult updateResult = collection.updateOne(Filters.and(geoConfFilter, noColumnFilter), updates); LOG.info("Update Result: {}", updateResult); + Bson setDefaultVendor = Updates.set(FIELD_DB_VENDOR, DatabaseVendorType.MAXMIND.name()); + LOG.info("Setting default vendor: " + setDefaultVendor); + final UpdateResult updateVendorResult = collection.updateOne(geoConfFilter, setDefaultVendor); + LOG.info("Default Vendor Update Result: " + updateVendorResult); + } } From 599ddad919bf131ac2b30614fbfba91412350f77 Mon Sep 17 00:00:00 2001 From: Roberto Benitez Date: Thu, 13 Jan 2022 15:38:05 -0600 Subject: [PATCH 26/28] Code Cleanup --- .../V20211221144300_GeoIpResolverConfigMigration.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/graylog2-server/src/main/java/org/graylog2/migrations/V20211221144300_GeoIpResolverConfigMigration.java b/graylog2-server/src/main/java/org/graylog2/migrations/V20211221144300_GeoIpResolverConfigMigration.java index be2ca63db44d..3011b262fc52 100644 --- a/graylog2-server/src/main/java/org/graylog2/migrations/V20211221144300_GeoIpResolverConfigMigration.java +++ b/graylog2-server/src/main/java/org/graylog2/migrations/V20211221144300_GeoIpResolverConfigMigration.java @@ -90,9 +90,9 @@ public void upgrade() { LOG.info("Update Result: {}", updateResult); Bson setDefaultVendor = Updates.set(FIELD_DB_VENDOR, DatabaseVendorType.MAXMIND.name()); - LOG.info("Setting default vendor: " + setDefaultVendor); + LOG.info("Setting default vendor: {}", setDefaultVendor); final UpdateResult updateVendorResult = collection.updateOne(geoConfFilter, setDefaultVendor); - LOG.info("Default Vendor Update Result: " + updateVendorResult); + LOG.info("Default Vendor Update Result: {}", updateVendorResult); } } From 4b365c0a4b9acdc313854dab5ce7f1c09e235afb Mon Sep 17 00:00:00 2001 From: Roberto Benitez Date: Mon, 24 Jan 2022 13:31:30 -0600 Subject: [PATCH 27/28] Updated GeoIpProcessor to defer filter engine creation to the first time the process(...) message is called. --- .../map/geoip/processor/GeoIpProcessor.java | 30 ++++++++++++------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/processor/GeoIpProcessor.java b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/processor/GeoIpProcessor.java index cacad4bf6dbf..c709bbb1442a 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/processor/GeoIpProcessor.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/processor/GeoIpProcessor.java @@ -25,6 +25,7 @@ import org.graylog2.cluster.ClusterConfigChangedEvent; import org.graylog2.plugin.Message; import org.graylog2.plugin.Messages; +import org.graylog2.plugin.ServerStatus; import org.graylog2.plugin.cluster.ClusterConfigService; import org.graylog2.plugin.messageprocessors.MessageProcessor; import org.slf4j.Logger; @@ -55,32 +56,40 @@ public String className() { private final ScheduledExecutorService scheduler; private final MetricRegistry metricRegistry; private final GeoIpVendorResolverService geoIpVendorResolverService; + private final ServerStatus serverStatus; - private final AtomicReference config; - private final AtomicReference filterEngine; + private final AtomicReference filterEngine = new AtomicReference<>(null); @Inject public GeoIpProcessor(ClusterConfigService clusterConfigService, @Named("daemonScheduler") ScheduledExecutorService scheduler, EventBus eventBus, MetricRegistry metricRegistry, - GeoIpVendorResolverService geoIpVendorResolverService) { + GeoIpVendorResolverService geoIpVendorResolverService, + ServerStatus serverStatus) { this.clusterConfigService = clusterConfigService; this.scheduler = scheduler; this.metricRegistry = metricRegistry; this.geoIpVendorResolverService = geoIpVendorResolverService; - - final GeoIpResolverConfig config = clusterConfigService.getOrDefault(GeoIpResolverConfig.class, - GeoIpResolverConfig.defaultConfig()); - - this.config = new AtomicReference<>(config); - this.filterEngine = new AtomicReference<>(new GeoIpResolverEngine(geoIpVendorResolverService, config, metricRegistry)); + this.serverStatus = serverStatus; eventBus.register(this); } @Override public Messages process(Messages messages) { + + if (filterEngine.get() == null) { + try { + serverStatus.awaitRunning(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + LOG.error("The GeoIpProcessor was interrupted while waiting for the Server to start up."); + return messages; + } + reload(); + } + for (Message message : messages) { filterEngine.get().filter(message); } @@ -95,7 +104,7 @@ public void updateConfig(ClusterConfigChangedEvent event) { return; } - scheduler.schedule((Runnable) this::reload, 0, TimeUnit.SECONDS); + scheduler.schedule(this::reload, 0, TimeUnit.SECONDS); } private void reload() { @@ -103,7 +112,6 @@ private void reload() { GeoIpResolverConfig.defaultConfig()); LOG.info("Updating GeoIP resolver engine - {}", newConfig); - config.set(newConfig); filterEngine.set(new GeoIpResolverEngine(geoIpVendorResolverService, newConfig, metricRegistry)); } } From f8327c0aaf228ff8563f7fc74ba1e8172fe6fdbf Mon Sep 17 00:00:00 2001 From: Roberto Benitez Date: Wed, 26 Jan 2022 10:02:00 -0600 Subject: [PATCH 28/28] Updated repetitive log messages to debug in GeoIpResolverEngine & GeoIpResolverEngine --- .../org/graylog/plugins/map/geoip/GeoIpResolverEngine.java | 6 +++--- .../graylog/plugins/map/geoip/processor/GeoIpProcessor.java | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpResolverEngine.java b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpResolverEngine.java index c0aba08004c4..7718e3916559 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpResolverEngine.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/GeoIpResolverEngine.java @@ -70,9 +70,9 @@ public GeoIpResolverEngine(GeoIpVendorResolverService resolverService, GeoIpReso ipLocationResolver = resolverService.createCityResolver(config, resolveTime); ipAsnResolver = resolverService.createAsnResolver(config, resolveTime); - LOG.info("Created Geo IP Resolvers for '{}'", config.databaseVendorType()); - LOG.info("'{}' Status Enabled: {}", ipLocationResolver.getClass().getSimpleName(), ipLocationResolver.isEnabled()); - LOG.info("'{}' Status Enabled: {}", ipAsnResolver.getClass().getSimpleName(), ipAsnResolver.isEnabled()); + LOG.debug("Created Geo IP Resolvers for '{}'", config.databaseVendorType()); + LOG.debug("'{}' Status Enabled: {}", ipLocationResolver.getClass().getSimpleName(), ipLocationResolver.isEnabled()); + LOG.debug("'{}' Status Enabled: {}", ipAsnResolver.getClass().getSimpleName(), ipAsnResolver.isEnabled()); this.enabled = ipLocationResolver.isEnabled() || ipAsnResolver.isEnabled(); diff --git a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/processor/GeoIpProcessor.java b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/processor/GeoIpProcessor.java index c709bbb1442a..453dda5652fd 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/processor/GeoIpProcessor.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/map/geoip/processor/GeoIpProcessor.java @@ -111,7 +111,7 @@ private void reload() { final GeoIpResolverConfig newConfig = clusterConfigService.getOrDefault(GeoIpResolverConfig.class, GeoIpResolverConfig.defaultConfig()); - LOG.info("Updating GeoIP resolver engine - {}", newConfig); + LOG.debug("Updating GeoIP resolver engine - {}", newConfig); filterEngine.set(new GeoIpResolverEngine(geoIpVendorResolverService, newConfig, metricRegistry)); } }