Skip to content

Commit

Permalink
Patch config fetch (#1647)
Browse files Browse the repository at this point in the history
* remove extended logging

* refactor AgentConfigurationReloadTask

* add docsObjectsTask test

* optimize task and fix test

* optimize cached agent mappings

* remove redundant logs

* update naming convention

* add requested changes

* refactor supplier

* refactor AgentConfiguration

* remove static supplier

* rename method

* move AgentDocumentation to configdocsgenerator

* add tests

* refactor tests

* remove deprecated test

* make documentation volatile
  • Loading branch information
EddeCCC authored Feb 27, 2024
1 parent 4d05537 commit a51aa93
Show file tree
Hide file tree
Showing 23 changed files with 779 additions and 505 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -55,10 +55,10 @@ public class ConfigDocsGenerator {
Map<String, String> specialInputDescriptionsRegex = Collections.singletonMap("_arg\\d", "The _argN-th argument with which the instrumented method was called within which this action is getting executed.");

/**
* Map of files, which contain a set of defined objects like actions, scopes, rules & metrics
* Set of documentations, which contain a set of documentable objects like actions, scopes, rules & metrics for every file
*/
@Setter
private Map<String, Set<String>> docsObjectsByFile = new HashMap<>();
private Set<AgentDocumentation> agentDocumentations = new HashSet<>();

/**
* Generates a ConfigDocumentation from a YAML String describing an {@link InspectitConfig}.
Expand Down Expand Up @@ -233,11 +233,11 @@ private List<ActionDocs> generateActionDocs(Map<String, GenericActionSettings> a
* @return a set of files, which contain the provided object
*/
private Set<String> findFiles(String name) {
Set<String> files = docsObjectsByFile.entrySet().stream()
.filter(entry -> entry.getValue().contains(name))
.map(Map.Entry::getKey)
Set<String> files = agentDocumentations.stream()
.filter(documentation -> documentation.getObjects().contains(name))
.map(AgentDocumentation::getFilePath)
.collect(Collectors.toSet());
if(files.isEmpty()) log.warn("No file found with definition of " + name);
if(files.isEmpty()) log.debug("No file found with definition of " + name);

return files;
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
package inspectit.ocelot.configdocsgenerator.model;

import lombok.Value;

import java.util.Set;

/**
* Model, to store documentable objects of a specific yaml file.
* Documentable objects can be actions, scopes, rules & metrics.
*
*/
@Value
public class AgentDocumentation {

/**
* file, which contains the documentable objects
*/
private String filePath;

/**
* documentable objects of the file
*/
private Set<String> objects;
}
Original file line number Diff line number Diff line change
Expand Up @@ -371,8 +371,10 @@ void verifyFindFiles() throws IOException {
docsObjects.add(ruleDocParent.getName());
docsObjects.add(metricDoc.getName());

Map<String, Set<String>> docsObjectsByFile = Collections.singletonMap(file, docsObjects);
configDocsGenerator.setDocsObjectsByFile(docsObjectsByFile);
AgentDocumentation documentation = new AgentDocumentation(file, docsObjects);
Set<AgentDocumentation> agentDocumentations = Collections.singleton(documentation);

configDocsGenerator.setAgentDocumentations(agentDocumentations);
when(actionWithDocInYaml.getFiles()).thenReturn(Collections.singleton(file));
when(actionWithoutDocInYaml.getFiles()).thenReturn(Collections.singleton(file));
when(scopeDoc.getFiles()).thenReturn(Collections.singleton(file));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -293,19 +293,10 @@ class StatusTable extends React.Component {

agentHealthTemplate = (rowData) => {
const { onShowHealthStateDialog } = this.props;
const { onShowDownloadDialog } = this.props;
const { metaInformation } = rowData;
const { healthState, metaInformation } = rowData;
const { health } = healthState;
const { agentId } = metaInformation;
// TODO Remove console.error() after agent health issues are solved
let health;
let healthState;
try {
healthState = rowData['healthState'];
health = healthState['health'];
} catch (error) {
console.error(`Could not read agent health from ${agentId}`, error);
console.error(rowData);
}
const { onShowDownloadDialog } = this.props;

let { agentCommandsEnabled, supportArchiveAvailable } = this.resolveServiceAvailability(metaInformation);

Expand Down
Original file line number Diff line number Diff line change
@@ -1,33 +1,50 @@
package rocks.inspectit.ocelot.agentconfiguration;

import lombok.Builder;
import inspectit.ocelot.configdocsgenerator.model.AgentDocumentation;
import lombok.Value;
import lombok.extern.slf4j.Slf4j;
import org.springframework.util.DigestUtils;
import org.yaml.snakeyaml.Yaml;
import rocks.inspectit.ocelot.file.FileInfo;
import rocks.inspectit.ocelot.file.accessor.AbstractFileAccessor;
import rocks.inspectit.ocelot.mappings.model.AgentMapping;

import java.nio.charset.Charset;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.*;
import java.util.function.Predicate;
import java.util.stream.Collectors;

/**
* An {@link AgentMapping} which has its configuration loaded in-memory.
* In addition, a cryptographic hash is computed to detect changes of configurations.
*/
@Value
@Slf4j
public class AgentConfiguration {

/**
* Used as maker in {@link AgentConfigurationManager#attributesToConfigurationCache} to mark attribute-maps
* for which no mapping matches.
*/
public static AgentConfiguration NO_MATCHING_MAPPING = createDefault();

/**
* Predicate to check if a given file path ends with .yml or .yaml
*/
private static final Predicate<String> HAS_YAML_ENDING = filePath -> filePath.toLowerCase()
.endsWith(".yml") || filePath.toLowerCase().endsWith(".yaml");

/**
* The agent mapping for which this instance represents the loaded configuration.
*/
private AgentMapping mapping;

/**
* The set of defined documentable objects in this configuration for each file. <br>
* - Key: the file path <br>
* - Value: the set of objects, like actions, scopes, rules & metrics
* The set of suppliers for documentable objects.
* There is a set of defined documentable objects in this configuration for each file. <br>
* The documentable objects will be loaded lazy.
*/
private Map<String, Set<String>> docsObjectsByFile;
private Set<AgentDocumentationSupplier> documentationSuppliers;

/**
* The merged YAML configuration for the given mapping.
Expand All @@ -39,11 +56,127 @@ public class AgentConfiguration {
*/
private String hash;

@Builder
private AgentConfiguration(AgentMapping mapping, Map<String, Set<String>> docsObjectsByFile, String configYaml) {
private AgentConfiguration(AgentMapping mapping, Set<AgentDocumentationSupplier> documentationSuppliers, String configYaml, String hash) {
this.mapping = mapping;
this.docsObjectsByFile = docsObjectsByFile;
this.documentationSuppliers = documentationSuppliers;
this.configYaml = configYaml;
hash = DigestUtils.md5DigestAsHex(configYaml.getBytes(Charset.defaultCharset()));
this.hash = hash;
}

/**
* Factory method to create AgentConfigurations. Also creates a cryptographic hash.
*
* @param mapping The agent mapping for which this instance represents the loaded configuration
* @param fileAccessor The accessor to use for reading the files
*
* @return Created AgentConfiguration
*/
public static AgentConfiguration create(AgentMapping mapping, AbstractFileAccessor fileAccessor) {
Set<AgentDocumentationSupplier> documentationSuppliers = new HashSet<>();
Object yamlResult = null;

LinkedHashSet<String> allYamlFiles = getAllYamlFilesForMapping(fileAccessor, mapping);
for (String path : allYamlFiles) {
String src = fileAccessor.readConfigurationFile(path).orElse("");
yamlResult = ObjectStructureMerger.loadAndMergeYaml(src, yamlResult, path);

AgentDocumentationSupplier supplier = new AgentDocumentationSupplier(() -> loadDocumentation(path, src));
documentationSuppliers.add(supplier);
}

String configYaml = yamlResult == null ? "" : new Yaml().dump(yamlResult);
String hash = DigestUtils.md5DigestAsHex(configYaml.getBytes(Charset.defaultCharset()));

return new AgentConfiguration(mapping, documentationSuppliers, configYaml, hash);
}

/**
* Factory method to create a default AgentConfiguration.
*
* @return Created default AgentConfiguration
*/
private static AgentConfiguration createDefault() {
String configYaml = "";
String hash = DigestUtils.md5DigestAsHex(configYaml.getBytes(Charset.defaultCharset()));
return new AgentConfiguration(null, new HashSet<>(), configYaml, hash);
}

/**
* Get the current agent documentations. The documentations will be loaded lazy.
*
* @return The loaded agent documentations
*/
public synchronized Set<AgentDocumentation> getDocumentations() {
if (documentationSuppliers == null) return Collections.emptySet();
return documentationSuppliers.stream().map(AgentDocumentationSupplier::get).collect(Collectors.toSet());
}

/**
* Returns the set of yaml files, which is defined in the agent mapping sources.
*
* @param fileAccessor the accessor to use for reading the file
* @param mapping the mapping, which contains a list of source file paths
*
* @return the set of yaml file paths for the provided mapping
* If this task has been canceled, null is returned.
*/
private static LinkedHashSet<String> getAllYamlFilesForMapping(AbstractFileAccessor fileAccessor, AgentMapping mapping) {
LinkedHashSet<String> allYamlFiles = new LinkedHashSet<>();
for (String path : mapping.sources()) {
List<String> yamlFiles = getAllYamlFiles(fileAccessor, path);
allYamlFiles.addAll(yamlFiles);
}
return allYamlFiles;
}

/**
* If the given path is a yaml file, a list containing only it is returned.
* If the path is a directory, the absolute path of all contained yaml files is returned in alphabetical order.
* If it is neither, an empty list is returned.
*
* @param path the path to check for yaml files, can start with a slash which will be ignored
*
* @return a list of absolute paths of contained YAML files
*/
private static List<String> getAllYamlFiles(AbstractFileAccessor fileAccessor, String path) {
String cleanedPath;
if (path.startsWith("/")) {
cleanedPath = path.substring(1);
} else {
cleanedPath = path;
}

if (fileAccessor.configurationFileExists(cleanedPath)) {
if (fileAccessor.configurationFileIsDirectory(cleanedPath)) {
List<FileInfo> fileInfos = fileAccessor.listConfigurationFiles(cleanedPath);

return fileInfos.stream()
.flatMap(file -> file.getAbsoluteFilePaths(cleanedPath))
.filter(HAS_YAML_ENDING)
.sorted()
.collect(Collectors.toList());
} else if (HAS_YAML_ENDING.test(cleanedPath)) {
return Collections.singletonList(cleanedPath);
}
}
return Collections.emptyList();
}

/**
* Loads all documentable objects of the yaml source string for the provided file.
*
* @param filePath the path to the yaml file
* @param src the yaml string
*
* @return the set of documentable objects for the provided file
*/
private static AgentDocumentation loadDocumentation(String filePath, String src) {
Set<String> objects = Collections.emptySet();
try {
objects = DocsObjectsLoader.loadObjects(src);
} catch (Exception e) {
log.warn("Could not parse configuration: {}", filePath, e);
}
return new AgentDocumentation(filePath, objects);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -24,18 +24,15 @@
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;

import static rocks.inspectit.ocelot.agentconfiguration.AgentConfiguration.NO_MATCHING_MAPPING;

/**
* Manager responsible for serving the agent configuration based on the set of {@link AgentMapping}s.
*/
@Component
@Slf4j
public class AgentConfigurationManager {

/**
* Used as maker in {@link #attributesToConfigurationCache} to mark attribute-maps for which no mapping matches.
*/
private static final AgentConfiguration NO_MATCHING_MAPPING = AgentConfiguration.builder().configYaml("").build();

@Autowired
@VisibleForTesting
InspectitServerSettings config;
Expand Down Expand Up @@ -120,7 +117,7 @@ private synchronized void replaceConfigurations(List<AgentConfiguration> newConf
attributesToConfigurationCache = CacheBuilder.newBuilder()
.maximumSize(config.getMaxAgents())
.expireAfterAccess(config.getAgentEvictionDelay().toMillis(), TimeUnit.MILLISECONDS)
.build(new CacheLoader<Map<String, String>, AgentConfiguration>() {
.build(new CacheLoader<>() {
@Override
public AgentConfiguration load(Map<String, String> agentAttributes) {
return newConfigurations.stream()
Expand Down
Loading

0 comments on commit a51aa93

Please sign in to comment.