(feat) Dynamic Wildcards

src/dynamicprompts/commands/wildcard_command.py

@@ -8,13 +8,12 @@
 
 @dataclasses.dataclass(frozen=True)
 class WildcardCommand(Command):
-    wildcard: str
+    wildcard: Command | str
     sampling_method: SamplingMethod | None = None
     variables: dict[str, Command] = dataclasses.field(default_factory=dict)
 
     def __post_init__(self):
-        if not isinstance(self.wildcard, str):
-            raise TypeError(f"Wildcard must be a string, not {type(self.wildcard)}")
-
-    def with_content(self, content: str) -> WildcardCommand:
-        return dataclasses.replace(self, wildcard=content)
+        if not isinstance(self.wildcard, (Command, str)):
+            raise TypeError(
+                f"Wildcard must be a Command or str, not {type(self.wildcard)}",
+            )

src/dynamicprompts/parser/parse.py

@@ -109,25 +109,21 @@ def _configure_range() -> pp.ParserElement:
 
 def _configure_wildcard(
     parser_config: ParserConfig,
-    variable_ref: pp.ParserElement,
+    prompt: pp.ParserElement,
 ) -> pp.ParserElement:
-    wildcard = _configure_wildcard_path(
-        parser_config=parser_config,
-        variable_ref=variable_ref,
-    )
-
     wildcard_enclosure = pp.Suppress(parser_config.wildcard_wrap)
     wildcard_variable_spec = (
         OPT_WS
-        + pp.Suppress("(")
+        + pp.Literal("(")
         + pp.Regex(r"[^)]+")("variable_spec")
         + pp.Suppress(")")
     )
+    wc_path = prompt()("path")
 
     wildcard = (
         wildcard_enclosure
         + pp.Opt(sampler_symbol)("sampling_method")
-        + wildcard
+        + wc_path
         + pp.Opt(wildcard_variable_spec)
         + wildcard_enclosure
     )
@@ -149,6 +145,7 @@ def _configure_wildcard_path(
 def _configure_literal_sequence(
     parser_config: ParserConfig,
     is_variant_literal: bool = False,
+    is_wildcard_literal: bool = False,
 ) -> pp.ParserElement:
     # Characters that are not allowed in a literal
     # - { denotes the start of a variant (or whatever variant_start is set to  )
@@ -168,6 +165,12 @@ def _configure_literal_sequence(
         # - $ denotes the end of a bound expression
         non_literal_chars += rf"|${parser_config.variant_end}"
 
+    if is_wildcard_literal:
+        # Inside a wildcard the following characters are also not allowed
+        # - ( denotes the beginning of wildcard variable parameters
+        # - ) denotes the end of wildcard variable parameters
+        non_literal_chars += r")("
+
     non_literal_chars = re.escape(non_literal_chars)
     literal = pp.Regex(
         rf"((?!{re.escape(parser_config.wildcard_wrap)})[^{non_literal_chars}])+",
@@ -356,7 +359,9 @@ def _parse_wildcard_command(
     else:
         variables = {}
 
-    assert isinstance(wildcard, str)
+    assert isinstance(wildcard, (Command, str))
+    if isinstance(wildcard, LiteralCommand):
+        wildcard = wildcard.literal
     return WildcardCommand(
         wildcard=wildcard,
         sampling_method=sampling_method,
@@ -394,6 +399,18 @@ def _parse_variable_access_command(
     return VariableAccessCommand(name=parts["name"], default=parts.get("default"))
 
 
+def _parse_wildcard_variable_access_command(
+    parse_result: pp.ParseResults,
+) -> VariableAccessCommand:
+    parts = parse_result[0].as_dict()
+    name = parts["name"]
+    default = parts.get("default") or LiteralCommand(name)
+    return VariableAccessCommand(
+        name=name,
+        default=LiteralCommand(default.literal.strip()),
+    )
+
+
 def _parse_variable_assignment_command(
     parse_result: pp.ParseResults,
 ) -> VariableAssignmentCommand:
@@ -423,28 +440,16 @@ def create_parser(
 
     prompt = pp.Forward()
     variant_prompt = pp.Forward()
+    wildcard_prompt = pp.Forward()
 
     variable_access = _configure_variable_access(
         parser_config=parser_config,
         prompt=variant_prompt,
     )
-
-    variable_ref = pp.Combine(
-        _configure_variable_access(
-            parser_config=parser_config,
-            prompt=pp.SkipTo(parser_config.variable_end),
-        ),
-    ).leave_whitespace()
-
-    # by default, the variable starting "${" and end "}" characters are
-    # stripped with variable access. This restores them so that the
-    # variable can be properly recognized at a later stage
-    variable_ref.set_parse_action(
-        lambda string,
-        location,
-        token: f"{parser_config.variable_start}{''.join(token)}{parser_config.variable_end}",
+    wildcard_variable_access = _configure_variable_access(
+        parser_config=parser_config,
+        prompt=variant_prompt,
     )
-
     variable_assignment = _configure_variable_assignment(
         parser_config=parser_config,
         prompt=variant_prompt,
@@ -455,9 +460,13 @@ def create_parser(
     )
     wildcard = _configure_wildcard(
         parser_config=parser_config,
-        variable_ref=variable_ref,
+        prompt=wildcard_prompt,
     )
     literal_sequence = _configure_literal_sequence(parser_config=parser_config)
+    wildcard_literal_sequence = _configure_literal_sequence(
+        parser_config=parser_config,
+        is_wildcard_literal=True,
+    )
     variant_literal_sequence = _configure_literal_sequence(
         is_variant_literal=True,
         parser_config=parser_config,
@@ -479,9 +488,16 @@ def create_parser(
     variant_chunk = (
         variable_access | wrap_command | variants | wildcard | variant_literal_sequence
     )
+    wildcard_chunk = (
+        wildcard_variable_access
+        | variants
+        | wildcard_literal_sequence
+        | variant_literal_sequence
+    )
 
     prompt <<= pp.ZeroOrMore(chunk)("prompt")
     variant_prompt <<= pp.ZeroOrMore(variant_chunk)("prompt")
+    wildcard_prompt <<= pp.OneOrMore(wildcard_chunk, stop_on=pp.Char("("))("prompt")
 
     # Configure comments
     prompt.ignore("#" + pp.restOfLine)
@@ -493,12 +509,15 @@ def create_parser(
     )
     variants.set_parse_action(_parse_variant_command)
     literal_sequence.set_parse_action(_parse_literal_command)
+    wildcard_literal_sequence.set_parse_action(_parse_literal_command)
     variant_literal_sequence.set_parse_action(_parse_literal_command)
     variable_access.set_parse_action(_parse_variable_access_command)
+    wildcard_variable_access.set_parse_action(_parse_wildcard_variable_access_command)
     variable_assignment.set_parse_action(_parse_variable_assignment_command)
     prompt.set_parse_action(_parse_sequence_or_single_command)
     variant_prompt.set_parse_action(_parse_sequence_or_single_command)
     wrap_command.set_parse_action(_parse_wrap_command)
+    wildcard_prompt.set_parse_action(_parse_sequence_or_single_command)
     return prompt
 
 

src/dynamicprompts/samplers/combinatorial.py

@@ -16,7 +16,6 @@
 from dynamicprompts.samplers.command_collection import CommandCollection
 from dynamicprompts.samplers.utils import (
     get_wildcard_not_found_fallback,
-    replace_wildcard_variables,
     wildcard_to_variant,
 )
 from dynamicprompts.sampling_context import SamplingContext
@@ -144,9 +143,10 @@ def _get_wildcard(
         context: SamplingContext,
     ) -> ResultGen:
         # TODO: doesn't support weights
-        command = replace_wildcard_variables(command=command, context=context)
+        wildcard_path = next(iter(context.sample_prompts(command.wildcard, 1))).text
         context = context.with_variables(command.variables)
-        values = context.wildcard_manager.get_values(command.wildcard)
+        values = context.wildcard_manager.get_values(wildcard_path)
+
         if not values:
             yield from get_wildcard_not_found_fallback(command, context)
             return

src/dynamicprompts/samplers/cycle.py

@@ -13,7 +13,6 @@
 from dynamicprompts.samplers.base import Sampler
 from dynamicprompts.samplers.utils import (
     get_wildcard_not_found_fallback,
-    replace_wildcard_variables,
     wildcard_to_variant,
 )
 from dynamicprompts.sampling_context import SamplingContext
@@ -101,8 +100,8 @@ def _get_wildcard(
         context: SamplingContext,
     ) -> ResultGen:
         # TODO: doesn't support weights
-        command = replace_wildcard_variables(command=command, context=context)
-        wc_values = context.wildcard_manager.get_values(command.wildcard)
+        wildcard_path = next(iter(context.sample_prompts(command.wildcard, 1))).text
+        wc_values = context.wildcard_manager.get_values(wildcard_path)
         new_context = context.with_variables(
             command.variables,
         ).with_sampling_method(SamplingMethod.CYCLICAL)

src/dynamicprompts/samplers/random.py

@@ -13,7 +13,6 @@
 from dynamicprompts.samplers.base import Sampler
 from dynamicprompts.samplers.utils import (
     get_wildcard_not_found_fallback,
-    replace_wildcard_variables,
     wildcard_to_variant,
 )
 from dynamicprompts.sampling_context import SamplingContext
@@ -115,9 +114,9 @@ def _get_wildcard(
         command: WildcardCommand,
         context: SamplingContext,
     ) -> ResultGen:
-        command = replace_wildcard_variables(command=command, context=context)
+        wildcard_path = next(iter(context.sample_prompts(command.wildcard, 1))).text
         context = context.with_variables(command.variables)
-        values = context.wildcard_manager.get_values(command.wildcard)
+        values = context.wildcard_manager.get_values(wildcard_path)
 
         if len(values) == 0:
             yield from get_wildcard_not_found_fallback(command, context)

src/dynamicprompts/samplers/utils.py

@@ -1,21 +1,14 @@
 from __future__ import annotations
 
 import logging
-from functools import partial
-
-import pyparsing as pp
 
 from dynamicprompts.commands import (
-    LiteralCommand,
+    Command,
     VariantCommand,
     VariantOption,
     WildcardCommand,
 )
-from dynamicprompts.parser.parse import (
-    _configure_variable_access,
-    _configure_wildcard_path,
-    parse,
-)
+from dynamicprompts.parser.parse import parse
 from dynamicprompts.sampling_context import SamplingContext
 from dynamicprompts.sampling_result import SamplingResult
 from dynamicprompts.types import ResultGen
@@ -31,8 +24,8 @@ def wildcard_to_variant(
     max_bound=1,
     separator=",",
 ) -> VariantCommand:
-    command = replace_wildcard_variables(command=command, context=context)
-    values = context.wildcard_manager.get_values(command.wildcard)
+    wildcard = next(iter(context.sample_prompts(command.wildcard, 1))).text
+    values = context.wildcard_manager.get_values(wildcard)
     min_bound = min(min_bound, len(values))
     max_bound = min(max_bound, len(values))
 
@@ -58,64 +51,12 @@ def get_wildcard_not_found_fallback(
     """
     Logs a warning, then infinitely yields the wrapped wildcard.
     """
-    logger.warning(f"No values found for wildcard {command.wildcard!r}")
-    wrapped_wildcard = context.wildcard_manager.to_wildcard(command.wildcard)
+    if isinstance(command.wildcard, Command):
+        wildcard = next(iter(context.sample_prompts(command.wildcard, 1))).text
+    else:
+        wildcard = str(command.wildcard)
+    logger.warning(f"No values found for wildcard {wildcard!r}")
+    wrapped_wildcard = context.wildcard_manager.to_wildcard(wildcard)
     res = SamplingResult(text=wrapped_wildcard)
     while True:
         yield res
-
-
-def replace_wildcard_variables(
-    command: WildcardCommand,
-    *,
-    context: SamplingContext,
-) -> WildcardCommand:
-    if context.parser_config.variable_start not in command.wildcard:
-        return command
-
-    prompt = pp.SkipTo(context.parser_config.variable_end)
-    variable_access = _configure_variable_access(
-        parser_config=context.parser_config,
-        prompt=prompt,
-    )
-    variable_access.set_parse_action(
-        partial(_replace_variable, variables=context.variables),
-    )
-    wildcard = _configure_wildcard_path(
-        parser_config=context.parser_config,
-        variable_ref=variable_access,
-    )
-
-    try:
-        wildcard_result = wildcard.parse_string(command.wildcard)
-        return command.with_content("".join(wildcard_result))
-    except Exception:
-        logger.warning("Unable to parse wildcard %r", command.wildcard, exc_info=True)
-        return command
-
-
-def _replace_variable(string, location, token, *, variables: dict):
-    if isinstance(token, pp.ParseResults):
-        var_parts = token[0].as_dict()
-        var_name = var_parts.get("name")
-        if var_name:
-            var_name = var_name.strip()
-
-        default = var_parts.get("default")
-        if default:
-            default = default.strip()
-
-    else:
-        var_name = token
-
-    variable = None
-    if var_name:
-        variable = variables.get(var_name)
-
-        if isinstance(variable, LiteralCommand):
-            variable = variable.literal
-        if variable and not isinstance(variable, str):
-            raise NotImplementedError(
-                "evaluating complex commands within wildcards is not supported right now",
-            )
-    return variable or default or var_name