feat: add cli option for ideascale output_dir, generate funds.json ar…

…tifact

utilities/ideascale-importer/ideascale_importer/cli/ideascale.py

@@ -1,6 +1,7 @@
 """IdeaScale CLI commands."""
 
 import asyncio
+from pathlib import Path
 from typing import Optional
 import typer
 
@@ -39,6 +40,9 @@ def import_all(
         envvar="IDEASCALE_API_URL",
         help="IdeaScale API URL",
     ),
+    output_dir: Optional[str] = typer.Option(
+        default=None, envvar="IDEASCALE_OUTPUT_DIR", help="Output directory for generated files"
+    ),
 ):
     """Import all event data from IdeaScale for a given event."""
     configure_logger(log_level, log_format)
@@ -47,13 +51,23 @@ async def inner(
         event_id: int,
         proposals_scores_csv_path: Optional[str],
         ideascale_api_url: str,
+        output_dir: Optional[str]
     ):
+        # check if output_dir path exists, or create otherwise
+        if output_dir is None:
+            logger.info("No output directory was defined.")
+        else:
+            output_dir = Path(output_dir)
+            output_dir.mkdir(exist_ok=True, parents=True)
+            logger.info(f"Output directory for artifacts: {output_dir}")
+
         importer = Importer(
             api_token,
             database_url,
             event_id,
             proposals_scores_csv_path,
             ideascale_api_url,
+            output_dir
         )
 
         try:
@@ -63,4 +77,4 @@ async def inner(
         except Exception as e:
             logger.error(e)
 
-    asyncio.run(inner(event_id, proposals_scores_csv, ideascale_api_url))
+    asyncio.run(inner(event_id, proposals_scores_csv, ideascale_api_url, output_dir))

utilities/ideascale-importer/ideascale_importer/db/__init__.py

@@ -64,6 +64,7 @@ async def insert(conn: asyncpg.Connection, model: Model) -> Any:
         return ret[0]
     return None
 
+
 async def select(conn: asyncpg.Connection, model: Model, cond: Dict[str, str] = {}) -> List[Any]:
     """Select a single model."""
 
@@ -77,7 +78,7 @@ async def select(conn: asyncpg.Connection, model: Model, cond: Dict[str, str] =
         SELECT {cols_str}
         FROM {model.table()}
         {f' WHERE {cond_str}' if cond_str else ' '}
-    """.strip()  
+    """.strip()
 
     result = await conn.fetch(stmt_template)
 
@@ -123,9 +124,13 @@ async def upsert_many(
     pre_update_set_str = ",".join([f"{col} = {val}" for col, val in pre_update_cols.items()])
     pre_update_cond_str = " ".join([f"{col} {cond}" for col, cond in pre_update_cond.items()])
 
-    pre_update_template = f"""
+    pre_update_template = (
+        f"""
         WITH updated AS ({ f"UPDATE {models[0].table()} SET {pre_update_set_str} {f' WHERE {pre_update_cond_str}' if pre_update_cond_str else ' '}" })
-        """.strip() if pre_update_set_str else " "     
+        """.strip()
+        if pre_update_set_str
+        else " "
+    )
 
     stmt_template = f"""
         {pre_update_template}
@@ -172,6 +177,20 @@ async def event_exists(conn: asyncpg.Connection, id: int) -> bool:
     return row is not None
 
 
+class EventThesholdNotFound(Exception):
+    """Raised when the event's voting power threshold is not found."""
+
+    ...
+
+
+async def event_threshold(conn: asyncpg.Connection, row_id: int) -> int:
+    """Fetch the event's voting power threshold in ADA."""
+    res = await conn.fetchrow("SELECT voting_power_threshold FROM event WHERE row_id = $1", row_id)
+    if res is None:
+        raise EventThesholdNotFound()
+    threshold = int(res["voting_power_threshold"]/1000000)
+    return threshold
+
 async def update_event_description(conn: asyncpg.Connection, row_id: int, description: str):
     """Update the event description.
 

utilities/ideascale-importer/ideascale_importer/ideascale/artifacts.py

@@ -87,7 +87,8 @@ def json_from_proposal(prop: Proposal, challenge: ChallengesJson, fund_id: int,
 
 
 class FundsJson(BaseModel):
+    """Current Fund (Event) information in JSON used for output artifacts."""
     id: int
     goal: str
     threshold: int
-    rewards_info: str
+    rewards_info: str = ""

utilities/ideascale-importer/ideascale_importer/ideascale/importer.py

@@ -13,7 +13,7 @@
 from typing import Any, Dict, List, Mapping, Optional, Union
 
 from ideascale_importer.db.models import Objective
-from ideascale_importer.ideascale.artifacts import json_from_proposal, objective_to_challenge_json
+from ideascale_importer.ideascale.artifacts import FundsJson, json_from_proposal, objective_to_challenge_json
 
 from .client import Campaign, CampaignGroup, Client, Idea
 import ideascale_importer.db
@@ -232,13 +232,15 @@ def __init__(
         event_id: int,
         proposals_scores_csv_path: Optional[str],
         ideascale_api_url: str,
+        output_dir: Optional[Path],
     ):
         """Initialize the importer."""
         self.api_token = api_token
         self.database_url = database_url
         self.event_id = event_id
         self.conn: asyncpg.Connection | None = None
         self.ideascale_api_url = ideascale_api_url
+        self.output_dir = output_dir
 
         self.proposals_impact_scores: Dict[int, int] = {}
         if proposals_scores_csv_path is not None:
@@ -286,10 +288,6 @@ async def run(self):
 
         await self.load_config()
 
-        output_dir = Path(tempfile.gettempdir()).joinpath("catalyst-artifacts")
-        output_dir.mkdir(exist_ok=True)
-        logger.debug("Created temporary directory for artifact storage", output_dir=output_dir)
-
         if not await ideascale_importer.db.event_exists(self.conn, self.event_id):
             logger.error("No event exists with the given id")
             return
@@ -315,9 +313,6 @@ async def run(self):
         for stage_id in self.config.stage_ids:
             ideas.extend(await client.stage_ideas(stage_id=stage_id))
 
-        outuput_ideas = output_dir.joinpath("ideas.json")
-        out_data = [i.model_dump() for i in ideas]
-        outuput_ideas.write_text(json.dumps(out_data, indent=2))
         vote_options_id = await ideascale_importer.db.get_vote_options_id(self.conn, ["yes", "no"])
 
         # mapper used to convert ideascale data to db and json formats.
@@ -332,8 +327,17 @@ async def run(self):
         proposals = []
 
         # Hijack `event.description` with JSON string used by the mobile app.
-        fund_goal_str = json.dumps({"timestamp": strict_rfc3339.now_to_rfc3339_utcoffset(integer=True), "themes": themes})
+        fund_goal = {"timestamp": strict_rfc3339.now_to_rfc3339_utcoffset(integer=True), "themes": themes}
+        fund_goal_str = json.dumps(fund_goal)
+
+        threshold = await ideascale_importer.db.event_threshold(self.conn, self.event_id)
 
+        funds_json = FundsJson(id=self.event_id, goal=str(fund_goal), threshold=threshold)
+
+        if self.output_dir is not None:
+            outuput_ideas = self.output_dir.joinpath("funds.json")
+            out_data = funds_json.model_dump()
+            outuput_ideas.write_text(json.dumps(out_data, indent=4))
         async with self.conn.transaction():
             try:
                 await ideascale_importer.db.update_event_description(self.conn, self.event_id, fund_goal_str)
@@ -354,14 +358,17 @@ async def run(self):
                     objective_to_challenge_json(o, self.ideascale_api_url, idx + 1) for idx, o in enumerate(inserted_objectives)
                 ]
                 challenges_ix = {c.internal_id: c for c in challenges}
-                outuput_objs = output_dir.joinpath("challenges.json")
-                out_data = [c.model_dump() for c in challenges]
-                outuput_objs.write_text(json.dumps(out_data, indent=4))
+
+                if self.output_dir is not None:
+                    outuput_objs = self.output_dir.joinpath("challenges.json")
+                    out_data = [c.model_dump() for c in challenges]
+                    outuput_objs.write_text(json.dumps(out_data, indent=4))
 
                 proposals, proposals_json = self.convert_ideas_to_proposals(ideas, mapper, inserted_objectives_ix, challenges_ix)
 
-                outuput_f = output_dir.joinpath("proposals.json")
-                outuput_f.write_text(json.dumps(proposals_json, indent=4))
+                if self.output_dir is not None:
+                    outuput_f = self.output_dir.joinpath("proposals.json")
+                    outuput_f.write_text(json.dumps(proposals_json, indent=4))
 
                 all_objectives = await ideascale_importer.db.select(self.conn, objectives[0], cond={"event": f"= {self.event_id}"})
                 all_objectives_str = ",".join([f"{objective.row_id}" for objective in all_objectives])
@@ -396,7 +403,6 @@ def convert_ideas_to_proposals(self, ideas, mapper, inserted_objectives_ix, chal
             objective_id, p = a.campaign_id, mapper.map_proposal(a, self.proposals_impact_scores)
             if objective_id in inserted_objectives_ix:
                 objective = inserted_objectives_ix[objective_id]
-                print(f"objective {objective}")
                 p.objective = objective.row_id
                 proposals.append(p)
                 p_json = json_from_proposal(p, challenges_ix[objective.id], self.event_id, cnt)