Lifetime foo

Cargo.toml

@@ -36,6 +36,7 @@ tokio = { version = "1.38", features = ["full"] }
 tokio-stream = { version = "0.1" }
 tracing = { version = "0.1", features = ["log"] }
 num_cpus = { version = "1.16" }
+pin-project = { version = "1.1.5" }
 pin-project-lite = { version = "0.2" }
 itertools = { version = "0.13" }
 serde = { version = "1.0", features = ["derive"] }
@@ -46,6 +47,7 @@ lazy_static = { version = "1.5.0" }
 chrono = { version = "0.4" }
 indoc = { version = "2.0" }
 regex = { version = "1.10.6" }
+dyn-clone = { version = "1.0" }
 
 # Integrations
 spider = { version = "2.2" }

swiftide-core/Cargo.toml

@@ -15,7 +15,7 @@ anyhow = { workspace = true }
 tokio = { workspace = true, features = ["full"] }
 tracing = { workspace = true }
 async-trait = { workspace = true }
-pin-project-lite = { workspace = true }
+pin-project = { workspace = true }
 futures-util = { workspace = true }
 tokio-stream = { workspace = true }
 itertools = { workspace = true }
@@ -26,6 +26,7 @@ strum_macros = { workspace = true }
 mockall = { workspace = true, optional = true }
 lazy_static = { workspace = true }
 derive_builder = { workspace = true }
+dyn-clone = { workspace = true }
 
 tera = { version = "1.20", default-features = false }
 uuid = { version = "1.10", features = ["v4"] }

swiftide-core/src/indexing_stream.rs

@@ -5,27 +5,24 @@
 use crate::node::Node;
 use anyhow::Result;
 use futures_util::stream::{self, Stream};
-use pin_project_lite::pin_project;
 use std::pin::Pin;
 use tokio::sync::mpsc::Receiver;
 
 pub use futures_util::StreamExt;
 
-// We need to inform the compiler that `inner` is pinned as well
-pin_project! {
-    /// An asynchronous stream of `Node` items.
-    ///
-    /// Wraps an internal stream of `Result<Node>` items.
-    ///
-    /// Streams, iterators and vectors of `Result<Node>` can be converted into an `IndexingStream`.
-    pub struct IndexingStream {
-        #[pin]
-        pub(crate) inner: Pin<Box<dyn Stream<Item = Result<Node>> + Send>>,
-    }
+/// An asynchronous stream of `Node` items.
+///
+/// Wraps an internal stream of `Result<Node>` items.
+///
+/// Streams, iterators and vectors of `Result<Node>` can be converted into an `IndexingStream`.
+#[pin_project::pin_project]
+pub struct IndexingStream<'stream, I: 'stream = Result<Node>> {
+    #[pin]
+    pub(crate) inner: Pin<Box<dyn Stream<Item = I> + 'stream>>,
 }
 
-impl Stream for IndexingStream {
-    type Item = Result<Node>;
+impl<'stream, T: Send> Stream for IndexingStream<'stream, T> {
+    type Item = T;
 
     fn poll_next(
         self: Pin<&mut Self>,

swiftide-core/src/indexing_traits.rs

@@ -14,15 +14,15 @@ use crate::prompt::Prompt;
 use anyhow::Result;
 use async_trait::async_trait;
 
+use dyn_clone::DynClone;
 /// All traits are easily mockable under tests
 #[cfg(feature = "test-utils")]
 #[doc(hidden)]
 use mockall::{automock, predicate::str};
 
-#[cfg_attr(feature = "test-utils", automock)]
 #[async_trait]
 /// Transforms single nodes into single nodes
-pub trait Transformer: Send + Sync {
+pub trait Transformer: Send + Sync + DynClone {
     async fn transform_node(&self, node: Node) -> Result<Node>;
 
     /// Overrides the default concurrency of the pipeline
@@ -31,11 +31,27 @@ pub trait Transformer: Send + Sync {
     }
 }
 
+#[cfg(feature = "test-utils")]
+mockall::mock! {
+    #[derive(Debug)]
+    pub Transformer {}
+
+    impl Clone for Transformer {
+        fn clone(&self) -> Self;
+    }
+
+    #[async_trait]
+    impl Transformer for Transformer {
+        async fn transform_node(&self, node: Node) -> Result<Node>;
+        fn concurrency(&self) -> Option<usize>;
+    }
+}
+
 #[async_trait]
 /// Use a closure as a transformer
 impl<F> Transformer for F
 where
-    F: Fn(Node) -> Result<Node> + Send + Sync,
+    F: Fn(Node) -> Result<Node> + Send + Sync + Clone,
 {
     async fn transform_node(&self, node: Node) -> Result<Node> {
         self(node)
@@ -111,14 +127,44 @@ pub trait SparseEmbeddingModel: Send + Sync + Debug {
     async fn sparse_embed(&self, input: Vec<String>) -> Result<SparseEmbeddings>;
 }
 
-#[cfg_attr(feature = "test-utils", automock)]
+// #[cfg_attr(feature = "test-utils", automock)]
 #[async_trait]
 /// Given a string prompt, queries an LLM
-pub trait SimplePrompt: Debug + Send + Sync {
+pub trait SimplePrompt: Debug + Send + Sync + DynClone {
     // Takes a simple prompt, prompts the llm and returns the response
     async fn prompt(&self, prompt: Prompt) -> Result<String>;
 }
 
+#[cfg(feature = "test-utils")]
+mockall::mock! {
+    #[derive(Debug)]
+    pub SimplePrompt {}
+
+
+    impl Clone for SimplePrompt {
+        fn clone(&self) -> Self;
+    }
+
+    #[async_trait]
+    impl SimplePrompt for SimplePrompt {
+        async fn prompt(&self, prompt: Prompt) -> Result<String>;
+    }
+}
+
+#[async_trait]
+impl<T: SimplePrompt> SimplePrompt for &T {
+    async fn prompt(&self, prompt: Prompt) -> Result<String> {
+        (*self).prompt(prompt).await
+    }
+}
+
+#[async_trait]
+impl<T: SimplePrompt + Clone> SimplePrompt for Box<T> {
+    async fn prompt(&self, prompt: Prompt) -> Result<String> {
+        (*self).prompt(prompt).await
+    }
+}
+
 #[cfg_attr(feature = "test-utils", automock)]
 #[async_trait]
 /// Persists nodes

swiftide-core/src/lib.rs

@@ -41,3 +41,11 @@ pub mod prelude;
 
 #[cfg(feature = "test-utils")]
 pub mod test_utils;
+
+#[doc(hidden)]
+/// Re-export of commonly used dependencies.
+pub mod ext {
+    pub use anyhow;
+    pub use derive_builder;
+    pub use dyn_clone;
+}

swiftide-core/src/query_stream.rs

@@ -8,21 +8,19 @@ use tokio_stream::wrappers::ReceiverStream;
 
 use futures_util::stream::Stream;
 pub use futures_util::{StreamExt, TryStreamExt};
-use pin_project_lite::pin_project;
 
 use crate::querying::Query;
 
-pin_project! {
-    /// Internally used by a query pipeline
-    ///
-    /// Has a sender and receiver to initialize the stream
-    pub struct QueryStream<'stream, Q: 'stream> {
-        #[pin]
-        pub(crate) inner: Pin<Box<dyn Stream<Item = Result<Query<Q>>> + Send + 'stream>>,
+#[pin_project::pin_project]
+/// Internally used by a query pipeline
+///
+/// Has a sender and receiver to initialize the stream
+pub struct QueryStream<'stream, Q: 'stream> {
+    #[pin]
+    pub(crate) inner: Pin<Box<dyn Stream<Item = Result<Query<Q>>> + Send + 'stream>>,
 
-        #[pin]
-        pub sender: Option<Sender<Result<Query<Q>>>>
-    }
+    #[pin]
+    pub sender: Option<Sender<Result<Query<Q>>>>,
 }
 
 impl<'stream, T: Send + Sync + 'stream> Default for QueryStream<'stream, T> {

swiftide-indexing/Cargo.toml

@@ -28,6 +28,7 @@ serde_json = { workspace = true }
 strum = { workspace = true }
 strum_macros = { workspace = true }
 indoc = { workspace = true }
+dyn-clone = { workspace = true }
 
 ignore = "0.4"
 text-splitter = { version = "0.16", features = ["markdown"] }

swiftide-indexing/src/pipeline.rs

@@ -63,8 +63,12 @@ impl Pipeline {
     /// Sets the default LLM client to be used for LLM prompts for all transformers in the
     /// pipeline.
     #[must_use]
-    pub fn with_default_llm_client(mut self, client: impl SimplePrompt + 'static) -> Self {
-        self.indexing_defaults = IndexingDefaults::from_simple_prompt(Box::new(client));
+    pub fn with_default_llm_client<C>(mut self, client: C) -> Self
+    where
+        for<'a> C: SimplePrompt + Clone + 'a,
+    {
+        let client = dyn_clone::clone_box(&client.clone());
+        self.indexing_defaults = IndexingDefaults::from_simple_prompt(client);
         self
     }
 
@@ -172,15 +176,15 @@ impl Pipeline {
     ///
     /// An instance of `Pipeline` with the updated stream that applies the transformer to each node.
     #[must_use]
-    pub fn then(
-        mut self,
-        mut transformer: impl Transformer + WithIndexingDefaults + 'static,
-    ) -> Self {
+    pub fn then<T>(mut self, mut transformer: T) -> Self
+    where
+        for<'b> T: Transformer + WithIndexingDefaults + Clone + 'b,
+    {
         let concurrency = transformer.concurrency().unwrap_or(self.concurrency);
 
         transformer.with_indexing_defaults(self.indexing_defaults.clone());
-
-        let transformer = Arc::new(transformer);
+        let boxed = dyn_clone::clone_box(&transformer.clone());
+        let transformer = Arc::new(boxed);
         self.stream = self
             .stream
             .map_ok(move |node| {
@@ -213,11 +217,10 @@ impl Pipeline {
     ///
     /// An instance of `Pipeline` with the updated stream that applies the batch transformer to each batch of nodes.
     #[must_use]
-    pub fn then_in_batch(
-        mut self,
-        batch_size: usize,
-        mut transformer: impl BatchableTransformer + WithBatchIndexingDefaults + 'static,
-    ) -> Self {
+    pub fn then_in_batch<T>(mut self, batch_size: usize, mut transformer: T) -> Self
+    where
+        for<'b> T: BatchableTransformer + WithBatchIndexingDefaults + 'b,
+    {
         let concurrency = transformer.concurrency().unwrap_or(self.concurrency);
 
         transformer.with_indexing_defaults(self.indexing_defaults.clone());

swiftide-indexing/src/transformers/metadata_keywords.rs

@@ -18,7 +18,7 @@ use swiftide_core::{indexing::Node, Transformer};
 pub struct MetadataKeywords {}
 
 #[async_trait]
-impl Transformer for MetadataKeywords<'_> {
+impl Transformer for MetadataKeywords {
     /// Transforms an `Node` by extracting a keywords
     /// based on the text chunk within the node.
     ///

swiftide-indexing/src/transformers/metadata_qa_text.rs

@@ -22,7 +22,7 @@ pub struct MetadataQAText {
 }
 
 #[async_trait]
-impl Transformer for MetadataQAText<'_> {
+impl Transformer for MetadataQAText {
     /// Transforms an `Node` by generating questions and answers
     /// based on the text chunk within the node.
     ///

swiftide-indexing/src/transformers/metadata_summary.rs

@@ -19,7 +19,7 @@ use swiftide_core::{indexing::Node, Transformer};
 pub struct MetadataSummary {}
 
 #[async_trait]
-impl Transformer for MetadataSummary<'_> {
+impl Transformer for MetadataSummary {
     /// Transforms an `Node` by extracting a summary
     /// based on the text chunk within the node.
     ///

swiftide-indexing/src/transformers/metadata_title.rs

@@ -19,7 +19,7 @@ use swiftide_core::{indexing::Node, Transformer};
 pub struct MetadataTitle {}
 
 #[async_trait]
-impl Transformer for MetadataTitle<'_> {
+impl Transformer for MetadataTitle {
     /// Transforms an `Node` by generating questions and answers
     /// based on the text chunk within the node.
     ///

swiftide-integrations/src/scraping/html_to_markdown_transformer.rs

@@ -17,7 +17,7 @@ pub struct HtmlToMarkdownTransformer {
     htmd: Arc<HtmlToMarkdown>,
 }
 
-impl Default for HtmlToMarkdownTransformer<'_> {
+impl Default for HtmlToMarkdownTransformer {
     fn default() -> Self {
         Self {
             htmd: HtmlToMarkdown::builder()
@@ -31,14 +31,14 @@ impl Default for HtmlToMarkdownTransformer<'_> {
     }
 }
 
-impl std::fmt::Debug for HtmlToMarkdownTransformer<'_> {
+impl std::fmt::Debug for HtmlToMarkdownTransformer {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("HtmlToMarkdownTransformer").finish()
     }
 }
 
 #[async_trait]
-impl Transformer for HtmlToMarkdownTransformer<'_> {
+impl Transformer for HtmlToMarkdownTransformer {
     /// Converts the HTML content in the `Node` to markdown.
     ///
     /// Will Err the node if the conversion fails.

swiftide-integrations/src/treesitter/compress_code_outline.rs

@@ -21,7 +21,7 @@ fn extract_markdown_codeblock(text: String) -> String {
 }
 
 #[async_trait]
-impl Transformer for CompressCodeOutline<'_> {
+impl Transformer for CompressCodeOutline {
     /// Asynchronously transforms an `Node` by reducing the size of the outline to make it more relevant to the chunk.
     ///
     /// This method uses the `SimplePrompt` client to compress the outline of the `Node` and updates the `Node` with the compressed outline.

swiftide-integrations/src/treesitter/metadata_qa_code.rs

@@ -17,7 +17,7 @@ pub struct MetadataQACode {
 }
 
 #[async_trait]
-impl Transformer for MetadataQACode<'_> {
+impl Transformer for MetadataQACode {
     /// Asynchronously transforms a `Node` by generating questions and answers for its code chunk.
     ///
     /// This method uses the `SimplePrompt` client to generate questions and answers based on the code chunk