diff --git a/10-observability/observability-models-openai/README.md b/10-observability/observability-models-openai/README.md new file mode 100644 index 0000000..c18c78b --- /dev/null +++ b/10-observability/observability-models-openai/README.md @@ -0,0 +1,121 @@ +# LLM Observability: OpenAI + +LLM Observability for OpenAI. + +## Running the application + +The application relies on an OpenAI API for providing LLMs. This example guides you to use either the OpenAI Platform +or Ollama via the OpenAI-compatible API. The application also relies on Testcontainers to provision automatically +a Grafana LGTM observability stack. + +### When using OpenAI + +First, make sure you have an OpenAI account. +Then, define an environment variable with the OpenAI API Key associated to your OpenAI account as the value. + +```shell +export SPRING_AI_OPENAI_API_KEY= +``` + +Finally, run the Spring Boot application. + +```shell +./gradlew bootTestRun +``` + +### When using Ollama as a native application + +First, make sure you have [Ollama](https://ollama.ai) installed on your laptop. +Then, use Ollama to run the _mistral_ and _nomic-embed-text_ models. Those are the ones we'll use in this example. + +```shell +ollama run mistral +ollama run nomic-embed-text +``` + +Finally, run the Spring Boot application. + +```shell +./gradlew bootTestRun --args='--spring.profiles.active=ollama' +``` + +## Observability Platform + +Grafana is listening to port 3000. Check your container runtime to find the port to which is exposed to your localhost +and access Grafana from http://localhost:. The credentials are `admin`/`admin`. + +The application is automatically configured to export metrics and traces to the Grafana LGTM stack via OpenTelemetry. +In Grafana, you can query the traces from the "Explore" page, selecting the "Tempo" data source. You can also visualize metrics in "Explore > Metrics". + +## Calling the application + +You can now call the application to perform generative AI operations. +This example uses [httpie](https://httpie.io) to send HTTP requests. + +### Chat + +```shell +http :8080/chat +``` + +Try passing your custom prompt and check the result. + +```shell +http :8080/chat message=="What is the capital of Italy?" +``` + +The next request is configured with a custom temperature value to obtain a more creative, yet less precise answer. + +```shell +http :8080/chat/generic-options message=="Why is a raven like a writing desk? Give a short answer." +``` + +The next request is configured with Open AI-specific customizations. + +```shell +http :8080/chat/openai-options message=="What can you see beyond what you can see? Give a short answer." +``` + +Finally, try a request which uses function calling. + +```shell +http :8080/chat/functions authorName=="Philip Pullman" +``` + +### Embedding + +```shell +http :8080/embed +``` + +Try passing your custom prompt and check the result. + +```shell +http :8080/embed message=="The capital of Italy is Rome" +``` + +The next request is configured with OpenAI-specific customizations. + +```shell +http :8080/embed/openai-options message=="The capital of Italy is Rome" +``` + +### Image + +_If you're using the Ollama OpenAI API compatibility, the image use case is not supported._ + +```shell +http :8080/image +``` + +Try passing your custom prompt and check the result. + +```shell +http :8080/image message=="Yellow Submarine" +``` + +The next request is configured with Open AI-specific customizations. + +```shell +http :8080/image/openai-options message=="Here comes the sun" +``` diff --git a/10-observability/observability-models-openai/build.gradle b/10-observability/observability-models-openai/build.gradle new file mode 100644 index 0000000..517734e --- /dev/null +++ b/10-observability/observability-models-openai/build.gradle @@ -0,0 +1,43 @@ +plugins { + id 'java' + id 'org.springframework.boot' + id 'io.spring.dependency-management' +} + +group = 'com.thomasvitale' +version = '0.0.1-SNAPSHOT' + +java { + toolchain { + languageVersion = JavaLanguageVersion.of(22) + } +} + +repositories { + mavenCentral() + maven { url 'https://repo.spring.io/milestone' } + maven { url 'https://repo.spring.io/snapshot' } +} + +dependencies { + implementation platform("org.springframework.ai:spring-ai-bom:${springAiVersion}") + + implementation 'org.springframework.boot:spring-boot-starter-actuator' + implementation 'org.springframework.boot:spring-boot-starter-web' + implementation 'org.springframework.ai:spring-ai-openai-spring-boot-starter' + + implementation 'io.micrometer:micrometer-tracing-bridge-otel' + implementation 'io.opentelemetry:opentelemetry-exporter-otlp' + implementation 'io.micrometer:micrometer-registry-otlp' + + testAndDevelopmentOnly 'org.springframework.boot:spring-boot-devtools' + + testImplementation 'org.springframework.boot:spring-boot-starter-test' + testImplementation 'org.springframework.boot:spring-boot-testcontainers' + testImplementation 'org.testcontainers:junit-jupiter' + testRuntimeOnly 'org.junit.platform:junit-platform-launcher' +} + +tasks.named('test') { + useJUnitPlatform() +} diff --git a/10-observability/observability-models-openai/src/main/java/com/thomasvitale/ai/spring/BookService.java b/10-observability/observability-models-openai/src/main/java/com/thomasvitale/ai/spring/BookService.java new file mode 100644 index 0000000..4ea663e --- /dev/null +++ b/10-observability/observability-models-openai/src/main/java/com/thomasvitale/ai/spring/BookService.java @@ -0,0 +1,40 @@ +package com.thomasvitale.ai.spring; + +import org.springframework.stereotype.Service; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +@Service +public class BookService { + + private static final Map books = new ConcurrentHashMap<>(); + + static { + books.put(1, new Book("His Dark Materials", "Philip Pullman")); + books.put(2, new Book("Narnia", "C.S. Lewis")); + books.put(3, new Book("The Hobbit", "J.R.R. Tolkien")); + books.put(4, new Book("The Lord of The Rings", "J.R.R. Tolkien")); + books.put(5, new Book("The Silmarillion", "J.R.R. Tolkien")); + } + + List getBooksByAuthor(Author author) { + return books.values().stream() + .filter(book -> author.name().equals(book.author())) + .toList(); + } + + Book getBestsellerByAuthor(Author author) { + return switch (author.name()) { + case "J.R.R. Tolkien" -> books.get(4); + case "C.S. Lewis" -> books.get(2); + case "Philip Pullman" -> books.get(1); + default -> null; + }; + } + + public record Book(String title, String author) {} + public record Author(String name) {} + +} diff --git a/10-observability/observability-models-openai/src/main/java/com/thomasvitale/ai/spring/ChatController.java b/10-observability/observability-models-openai/src/main/java/com/thomasvitale/ai/spring/ChatController.java new file mode 100644 index 0000000..8e36054 --- /dev/null +++ b/10-observability/observability-models-openai/src/main/java/com/thomasvitale/ai/spring/ChatController.java @@ -0,0 +1,65 @@ +package com.thomasvitale.ai.spring; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.ai.chat.model.ChatModel; +import org.springframework.ai.chat.prompt.ChatOptionsBuilder; +import org.springframework.ai.chat.prompt.Prompt; +import org.springframework.ai.openai.OpenAiChatOptions; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import java.util.List; +import java.util.Set; + +@RestController +class ChatController { + + private final Logger logger = LoggerFactory.getLogger(ChatController.class); + + private final ChatModel chatModel; + + ChatController(ChatModel chatModel) { + this.chatModel = chatModel; + } + + @GetMapping("/chat") + String chat(@RequestParam(defaultValue = "What did Gandalf say to the Balrog?") String message) { + logger.info(message); + return chatModel.call(message); + } + + @GetMapping("/chat/generic-options") + String chatWithGenericOptions(@RequestParam(defaultValue = "What did Gandalf say to the Balrog?") String message) { + return chatModel.call(new Prompt(message, ChatOptionsBuilder.builder() + .withTemperature(1.3f) + .build())) + .getResult().getOutput().getContent(); + } + + @GetMapping("/chat/openai-options") + String chatWithOpenAiOptions(@RequestParam(defaultValue = "What did Gandalf say to the Balrog?") String message) { + return chatModel.call(new Prompt(message, OpenAiChatOptions.builder() + .withFrequencyPenalty(1.3f) + .withMaxTokens(1500) + .withPresencePenalty(1.0f) + .withStop(List.of("this-is-the-end", "addio")) + .withTemperature(0.7f) + .withTopP(0f) + .withUser("jon.snow") + .build())) + .getResult().getOutput().getContent(); + } + + @GetMapping("/chat/functions") + String chatWithFunctions(@RequestParam(defaultValue = "Philip Pullman") String author) { + return chatModel.call(new Prompt("What books written by %s are available to read and what is their bestseller?".formatted(author), + OpenAiChatOptions.builder() + .withTemperature(0.3f) + .withFunctions(Set.of("booksByAuthor", "bestsellerBookByAuthor")) + .build())) + .getResult().getOutput().getContent(); + } + +} diff --git a/10-observability/observability-models-openai/src/main/java/com/thomasvitale/ai/spring/EmbeddingController.java b/10-observability/observability-models-openai/src/main/java/com/thomasvitale/ai/spring/EmbeddingController.java new file mode 100644 index 0000000..5094ebf --- /dev/null +++ b/10-observability/observability-models-openai/src/main/java/com/thomasvitale/ai/spring/EmbeddingController.java @@ -0,0 +1,37 @@ +package com.thomasvitale.ai.spring; + +import org.springframework.ai.embedding.EmbeddingModel; +import org.springframework.ai.embedding.EmbeddingRequest; +import org.springframework.ai.openai.OpenAiEmbeddingOptions; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import java.util.List; + +@RestController +class EmbeddingController { + + private final EmbeddingModel embeddingModel; + + EmbeddingController(EmbeddingModel embeddingModel) { + this.embeddingModel = embeddingModel; + } + + @GetMapping("/embed") + String embed(@RequestParam(defaultValue = "And Gandalf yelled: 'You shall not pass!'") String message) { + var embeddings = embeddingModel.embed(message); + return "Size of the embedding vector: " + embeddings.size(); + } + + @GetMapping("/embed/openai-options") + String embedWithOpenAiOptions(@RequestParam(defaultValue = "And Gandalf yelled: 'You shall not pass!'") String message) { + var embeddings = embeddingModel.call(new EmbeddingRequest(List.of(message), OpenAiEmbeddingOptions.builder() + .withDimensions(1536) + .withEncodingFormat("float") + .build())) + .getResult().getOutput(); + return "Size of the embedding vector: " + embeddings.size(); + } + +} diff --git a/10-observability/observability-models-openai/src/main/java/com/thomasvitale/ai/spring/Functions.java b/10-observability/observability-models-openai/src/main/java/com/thomasvitale/ai/spring/Functions.java new file mode 100644 index 0000000..da3d930 --- /dev/null +++ b/10-observability/observability-models-openai/src/main/java/com/thomasvitale/ai/spring/Functions.java @@ -0,0 +1,25 @@ +package com.thomasvitale.ai.spring; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Description; + +import java.util.List; +import java.util.function.Function; + +@Configuration(proxyBeanMethods = false) +public class Functions { + + @Bean + @Description("Get the list of available books written by the given author") + public Function> booksByAuthor(BookService bookService) { + return bookService::getBooksByAuthor; + } + + @Bean + @Description("Get the bestseller book written by the given author") + public Function bestsellerBookByAuthor(BookService bookService) { + return bookService::getBestsellerByAuthor; + } + +} diff --git a/10-observability/observability-models-openai/src/main/java/com/thomasvitale/ai/spring/HttpClientConfig.java b/10-observability/observability-models-openai/src/main/java/com/thomasvitale/ai/spring/HttpClientConfig.java new file mode 100644 index 0000000..71e3f35 --- /dev/null +++ b/10-observability/observability-models-openai/src/main/java/com/thomasvitale/ai/spring/HttpClientConfig.java @@ -0,0 +1,27 @@ +package com.thomasvitale.ai.spring; + +import org.springframework.boot.web.client.ClientHttpRequestFactories; +import org.springframework.boot.web.client.ClientHttpRequestFactorySettings; +import org.springframework.boot.web.client.RestClientCustomizer; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.http.client.BufferingClientHttpRequestFactory; + +import java.time.Duration; + +@Configuration(proxyBeanMethods = false) +public class HttpClientConfig { + + @Bean + RestClientCustomizer restClientCustomizer() { + return restClientBuilder -> { + restClientBuilder + .requestFactory(new BufferingClientHttpRequestFactory( + ClientHttpRequestFactories.get(ClientHttpRequestFactorySettings.DEFAULTS + .withConnectTimeout(Duration.ofSeconds(60)) + .withReadTimeout(Duration.ofSeconds(60)) + ))); + }; + } + +} diff --git a/10-observability/observability-models-openai/src/main/java/com/thomasvitale/ai/spring/ImageController.java b/10-observability/observability-models-openai/src/main/java/com/thomasvitale/ai/spring/ImageController.java new file mode 100644 index 0000000..4d0845f --- /dev/null +++ b/10-observability/observability-models-openai/src/main/java/com/thomasvitale/ai/spring/ImageController.java @@ -0,0 +1,38 @@ +package com.thomasvitale.ai.spring; + +import org.springframework.ai.image.ImageModel; +import org.springframework.ai.image.ImagePrompt; +import org.springframework.ai.openai.OpenAiImageOptions; +import org.springframework.ai.openai.api.OpenAiImageApi; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +@RestController +class ImageController { + + private final ImageModel imageModel; + + ImageController(ImageModel imageModel) { + this.imageModel = imageModel; + } + + @GetMapping("/image") + String image(@RequestParam(defaultValue = "Here comes the sun") String message) { + return imageModel.call(new ImagePrompt(message)).getResult().getOutput().getUrl(); + } + + @GetMapping("/image/openai-options") + String imageWithOpenAiOptions(@RequestParam(defaultValue = "Here comes the sun") String message) { + var imageResponse = imageModel.call(new ImagePrompt(message, OpenAiImageOptions.builder() + .withQuality("standard") + .withN(1) + .withHeight(1024) + .withWidth(1024) + .withModel(OpenAiImageApi.ImageModel.DALL_E_3.getValue()) + .withResponseFormat("url") + .build())); + return imageResponse.getResult().getOutput().getUrl(); + } + +} diff --git a/10-observability/observability-models-openai/src/main/java/com/thomasvitale/ai/spring/ObservabilityModelsOpenAiApplication.java b/10-observability/observability-models-openai/src/main/java/com/thomasvitale/ai/spring/ObservabilityModelsOpenAiApplication.java new file mode 100644 index 0000000..dd9c532 --- /dev/null +++ b/10-observability/observability-models-openai/src/main/java/com/thomasvitale/ai/spring/ObservabilityModelsOpenAiApplication.java @@ -0,0 +1,13 @@ +package com.thomasvitale.ai.spring; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +@SpringBootApplication +public class ObservabilityModelsOpenAiApplication { + + public static void main(String[] args) { + SpringApplication.run(ObservabilityModelsOpenAiApplication.class, args); + } + +} diff --git a/10-observability/observability-models-openai/src/main/resources/application.yml b/10-observability/observability-models-openai/src/main/resources/application.yml new file mode 100644 index 0000000..770b66e --- /dev/null +++ b/10-observability/observability-models-openai/src/main/resources/application.yml @@ -0,0 +1,55 @@ +spring: + application: + name: observability-models-openai + ai: + chat: + observations: + include-completion: true + include-prompt: true + image: + observations: + include-prompt: true + openai: + api-key: ${OPENAI_API_KEY} + chat: + options: + model: gpt-4o-mini + temperature: 0.7 + embedding: + options: + model: text-embedding-3-small + image: + options: + model: dall-e-3 + +management: + endpoints: + web: + exposure: + include: "*" + metrics: + tags: + service.name: ${spring.application.name} + tracing: + sampling: + probability: 1.0 + otlp: + tracing: + endpoint: http://localhost:4318/v1/traces + +--- + +spring: + config: + activate: + on-profile: ollama + ai: + openai: + base-url: http://localhost:11434 + chat: + options: + model: mistral + temperature: 0.7 + embedding: + options: + model: nomic-embed-text diff --git a/10-observability/observability-models-openai/src/test/java/com/thomasvitale/ai/spring/ObservabilityModelsOpenAiApplicationTests.java b/10-observability/observability-models-openai/src/test/java/com/thomasvitale/ai/spring/ObservabilityModelsOpenAiApplicationTests.java new file mode 100644 index 0000000..3965233 --- /dev/null +++ b/10-observability/observability-models-openai/src/test/java/com/thomasvitale/ai/spring/ObservabilityModelsOpenAiApplicationTests.java @@ -0,0 +1,13 @@ +package com.thomasvitale.ai.spring; + +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; + +@SpringBootTest +class ObservabilityModelsOpenAiApplicationTests { + + @Test + void contextLoads() { + } + +} diff --git a/10-observability/observability-models-openai/src/test/java/com/thomasvitale/ai/spring/TestObservabilityModelsOpenAiApplication.java b/10-observability/observability-models-openai/src/test/java/com/thomasvitale/ai/spring/TestObservabilityModelsOpenAiApplication.java new file mode 100644 index 0000000..cd183eb --- /dev/null +++ b/10-observability/observability-models-openai/src/test/java/com/thomasvitale/ai/spring/TestObservabilityModelsOpenAiApplication.java @@ -0,0 +1,11 @@ +package com.thomasvitale.ai.spring; + +import org.springframework.boot.SpringApplication; + +public class TestObservabilityModelsOpenAiApplication { + + public static void main(String[] args) { + SpringApplication.from(ObservabilityModelsOpenAiApplication::main).with(TestcontainersConfiguration.class).run(args); + } + +} diff --git a/10-observability/observability-models-openai/src/test/java/com/thomasvitale/ai/spring/TestcontainersConfiguration.java b/10-observability/observability-models-openai/src/test/java/com/thomasvitale/ai/spring/TestcontainersConfiguration.java new file mode 100644 index 0000000..66e5c8f --- /dev/null +++ b/10-observability/observability-models-openai/src/test/java/com/thomasvitale/ai/spring/TestcontainersConfiguration.java @@ -0,0 +1,29 @@ +package com.thomasvitale.ai.spring; + +import org.springframework.boot.devtools.restart.RestartScope; +import org.springframework.boot.test.context.TestConfiguration; +import org.springframework.boot.testcontainers.service.connection.ServiceConnection; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Scope; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; + +import java.time.Duration; + +@TestConfiguration(proxyBeanMethods = false) +class TestcontainersConfiguration { + + @Bean + @RestartScope + @Scope("singleton") + @ServiceConnection("otel/opentelemetry-collector-contrib") + GenericContainer lgtmContainer() { + return new GenericContainer<>("docker.io/grafana/otel-lgtm:0.7.1") + .withExposedPorts(3000, 4317, 4318) + .withEnv("OTEL_METRIC_EXPORT_INTERVAL", "500") + .waitingFor(Wait.forLogMessage(".*The OpenTelemetry collector and the Grafana LGTM stack are up and running.*\\s", 1)) + .withStartupTimeout(Duration.ofMinutes(2)) + .withReuse(true); + } + +} diff --git a/README.md b/README.md index 4f9e0e6..a6534f0 100644 --- a/README.md +++ b/README.md @@ -96,6 +96,12 @@ _Coming soon_ | [audio-models-speech-openai](https://github.com/ThomasVitale/llm-apps-java-spring-ai/tree/main/09-audio-models/audio-models-speech-openai) | Speech generation with LLMs via OpenAI. | | [audio-models-transcription-openai](https://github.com/ThomasVitale/llm-apps-java-spring-ai/tree/main/09-audio-models/audio-models-transcription-openai) | Speech transcription with LLMs via OpenAI. | +### 10. Observability + +| Project | Description | +|--------------------------------------------------------------------------------------------------------------------|-------------------------------| +| [observability-models-openai](https://github.com/ThomasVitale/llm-apps-java-spring-ai/tree/main/10-observability/observability-models-openai) | LLM Observability for OpenAI. | + ## References and Additional Resources * [Spring AI Reference Documentation](https://docs.spring.io/spring-ai/reference/index.html) diff --git a/settings.gradle b/settings.gradle index a368654..962dc9c 100644 --- a/settings.gradle +++ b/settings.gradle @@ -46,3 +46,5 @@ include '08-image-models:image-models-openai' include '09-audio-models:audio-models-speech-openai' include '09-audio-models:audio-models-transcription-openai' + +include '10-observability:observability-models-openai'