feat: ideascale importer generates json artifacts

utilities/ideascale-importer/ideascale_importer/cli/ideascale.py

@@ -1,7 +1,7 @@
 """IdeaScale CLI commands."""
 
 import asyncio
-from typing import Optional, List
+from typing import Optional
 import typer
 
 from ideascale_importer.ideascale.client import Client

utilities/ideascale-importer/ideascale_importer/db/models.py

@@ -219,6 +219,7 @@ def table() -> str:
         """Return the name of the table that this model is stored in."""
         return "snapshot"
 
+
 @dataclass
 class Config(Model):
     """Represents a database config."""

utilities/ideascale-importer/ideascale_importer/ideascale/artifacts.py

@@ -0,0 +1,93 @@
+from typing import Optional
+from ideascale_importer.db.models import Objective, Proposal
+from pydantic import BaseModel
+
+
+class ProposalJson(BaseModel):
+    """A proposal in JSON used for output artifacts."""
+
+    category_name: str
+    chain_vote_options: str
+    challenge_id: str
+    challenge_type: str
+    chain_vote_type: str
+    internal_id: str
+    proposal_funds: str
+    proposal_id: str
+    proposal_impact_score: str
+    proposal_summary: str
+    proposal_title: str
+    proposal_url: str
+    proposer_email: Optional[str] = None
+    proposer_name: Optional[str] = None
+    proposer_relevant_experience: Optional[str] = None
+    proposer_url: Optional[str] = None
+    proposal_solution: Optional[str] = None
+    files_url: str
+
+
+class ChallengesJson(BaseModel):
+    id: str
+    internal_id: int
+    title: str
+    challenge_type: str
+    challenge_url: str
+    description: str
+    fund_id: str
+    rewards_total: str
+    proposers_rewards: str
+
+
+def objective_to_challenge_json(obj: Objective, ideascale_url: str, idx: int = 0) -> ChallengesJson:
+    c_url = f"{ideascale_url}/c/campaigns/{obj.id}/"
+    return ChallengesJson.model_validate(
+        {
+            "id": f"{idx}",
+            "internal_id": obj.id,
+            "title": obj.title,
+            "challenge_type": obj.category.removeprefix("catalyst-"),
+            "challenge_url": c_url,
+            "description": obj.description,
+            "fund_id": f"{obj.event}",
+            "rewards_total": f"{obj.rewards_total}",
+            "proposers_rewards": f"{obj.proposers_rewards}",
+        }
+    )
+
+
+def json_from_proposal(prop: Proposal, challenge: ChallengesJson, fund_id: int, idx: int = 0) -> ProposalJson:
+    if prop.proposer_relevant_experience == "":
+        experience = None
+    else:
+        experience = prop.proposer_relevant_experience
+    if prop.extra is not None:
+        solution = prop.extra.get("solution", None)
+    else:
+        solution = None
+    return ProposalJson.model_validate(
+        {
+            "category_name": f"Fund {fund_id}",
+            "chain_vote_options": "blank,yes,no",
+            "challenge_id": challenge.id,
+            "challenge_type": challenge.challenge_type,
+            "chain_vote_type": "private",
+            "internal_id": f"{idx}",
+            "proposal_funds": f"{prop.funds}",
+            "proposal_id": f"{prop.id}",
+            "proposal_impact_score": f"{prop.impact_score}",
+            "proposal_summary": prop.summary,
+            "proposal_title": prop.title,
+            "proposal_url": prop.url,
+            "proposer_name": prop.proposer_name,
+            "proposer_relevant_experience": experience,
+            "proposal_solution": solution,
+            "files_url": prop.files_url,
+        }
+    )
+
+
+class FundsJson(BaseModel):
+    id: int
+    goal: str
+    threshold: int
+    rewards_info: str

utilities/ideascale-importer/ideascale_importer/ideascale/client.py

@@ -213,8 +213,8 @@ async def event_themes(self, campaign_id: int, themes_custom_key: str) -> List[s
         """Get the list of themes for this Fund,by IdeaScale `campaign_id`."""
         try:
             res = await self._get(f"/a/rest/v1/customFields/idea/campaigns/{campaign_id}")
-            themes_fields = [f for f in res if f['key'] and f['key'] == themes_custom_key]
-            themes = themes_fields[0]["options"].split('\r\n')
+            themes_fields = [f for f in res if f["key"] and f["key"] == themes_custom_key]
+            themes = themes_fields[0]["options"].split("\r\n")
             return themes
         except Exception as e:
             raise Exception(f"Unable to fetch themes: {e}")

utilities/ideascale-importer/ideascale_importer/ideascale/importer.py

@@ -5,12 +5,15 @@
 import json
 import csv
 import strict_rfc3339
+import tempfile
 from loguru import logger
 from markdownify import markdownify
+from pathlib import Path
 from pydantic import BaseModel
-from typing import Any, Dict, List, Mapping, Optional, Set, Union
+from typing import Any, Dict, List, Mapping, Optional, Union
 
 from ideascale_importer.db.models import Objective
+from ideascale_importer.ideascale.artifacts import json_from_proposal, objective_to_challenge_json
 
 from .client import Campaign, CampaignGroup, Client, Idea
 import ideascale_importer.db
@@ -55,6 +58,7 @@ def from_json(val: dict):
         """Load configuration from a JSON object."""
         return Config.model_validate(val)
 
+
 class ReadProposalsScoresCsv(Exception):
     """Raised when the proposals impact scores csv cannot be read."""
 
@@ -84,12 +88,13 @@ def map_objective(self, a: Campaign, event_id: int) -> ideascale_importer.db.mod
         except InvalidRewardsString as e:
             raise MapObjectiveError("reward", "tagline", str(e))
 
+        title = a.name.replace(f"F{event_id}:", "").strip()
         return ideascale_importer.db.models.Objective(
             row_id=0,
             id=a.id,
             event=event_id,
             category=get_objective_category(a),
-            title=a.name,
+            title=title,
             description=html_to_md(a.description),
             deleted=False,
             rewards_currency=reward.currency,
@@ -124,13 +129,15 @@ def map_proposal(
                 extra[k] = html_to_md(mv)
 
         # Hijack `proposal.files_url` with JSON string used by the mobile app.
-        files_url_str = json.dumps({
-            "open_source": a.custom_fields_by_key.get("f11_open_source_choice"),
-            "external_link1": a.custom_fields_by_key.get("f11_link_1"),
-            "external_link2": a.custom_fields_by_key.get("f11_link_2"),
-            "external_link3": a.custom_fields_by_key.get("f11_link_3"),
-            "themes": a.custom_fields_by_key.get("f11_themes"),
-        })
+        files_url_str = str(
+            {
+                "open_source": a.custom_fields_by_key.get("f11_open_source_choice"),
+                "external_link1": a.custom_fields_by_key.get("f11_link_1"),
+                "external_link2": a.custom_fields_by_key.get("f11_link_2"),
+                "external_link3": a.custom_fields_by_key.get("f11_link_3"),
+                "themes": a.custom_fields_by_key.get("f11_themes"),
+            }
+        )
         return ideascale_importer.db.models.Proposal(
             id=a.id,
             objective=0,  # should be set later
@@ -141,7 +148,7 @@ def map_proposal(
             public_key=public_key,
             funds=funds,
             url=a.url,
-            files_url=f"'{files_url_str}'",
+            files_url=files_url_str,
             impact_score=impact_scores.get(a.id, 0),
             extra=extra,
             proposer_name=proposer_name,
@@ -198,8 +205,8 @@ def parse_reward(s: str) -> Reward:
     if result is None:
         raise InvalidRewardsString()
     else:
-        rewards = re.sub("\D", "", result.group(2))
-        currency = "ADA" #result.group(1)
+        rewards = re.sub("\\D", "", result.group(2))
+        currency = "ADA"  # result.group(1)
         return Reward(amount=int(rewards, base=10), currency=currency)
 
 
@@ -209,7 +216,7 @@ def get_objective_category(c: Campaign) -> str:
 
     if "catalyst natives" in r:
         return "catalyst-native"
-    elif "objective setting" in r:
+    elif "challenge setting" in r:
         return "catalyst-community-choice"
     else:
         return "catalyst-simple"
@@ -253,14 +260,10 @@ def __init__(
 
     async def load_config(self):
         """Load the configuration setting from the event db."""
-
         logger.debug("Loading ideascale config from the event-db")
 
         config = ideascale_importer.db.models.Config(row_id=0, id="ideascale", id2=f"{self.event_id}", id3="", value=None)
-        res = await ideascale_importer.db.select(self.conn, config,  cond={
-            "id": f"= '{config.id}'",
-            "AND id2": f"= '{config.id2}'"
-            })
+        res = await ideascale_importer.db.select(self.conn, config, cond={"id": f"= '{config.id}'", "AND id2": f"= '{config.id2}'"})
         if len(res) == 0:
             raise Exception("Cannot find ideascale config in the event-db database")
         self.config = Config.from_json(res[0].value)
@@ -283,6 +286,10 @@ async def run(self):
 
         await self.load_config()
 
+        output_dir = Path(tempfile.gettempdir()).joinpath("catalyst-artifacts")
+        output_dir.mkdir(exist_ok=True)
+        logger.debug("Created temporary directory for artifact storage", output_dir=output_dir)
+
         if not await ideascale_importer.db.event_exists(self.conn, self.event_id):
             logger.error("No event exists with the given id")
             return
@@ -308,30 +315,24 @@ async def run(self):
         for stage_id in self.config.stage_ids:
             ideas.extend(await client.stage_ideas(stage_id=stage_id))
 
+        outuput_ideas = output_dir.joinpath("ideas.json")
+        out_data = [i.model_dump() for i in ideas]
+        outuput_ideas.write_text(json.dumps(out_data, indent=2))
         vote_options_id = await ideascale_importer.db.get_vote_options_id(self.conn, ["yes", "no"])
+
+        # mapper used to convert ideascale data to db and json formats.
         mapper = Mapper(vote_options_id, self.config)
 
-        async def _process_campaigns(campaigns):
-            objectives: List[Objective] = []
-            themes: List[str] = []
-            for campaign in campaigns:
-                objectives.append(mapper.map_objective(campaign, self.event_id))
-                campaign_themes = await client.event_themes(campaign.id, "f11_themes")
-                themes.extend(campaign_themes)
-            themes = list(set(themes))
-            themes.sort()
-            return objectives, themes
-        objectives, themes  = await _process_campaigns(group.campaigns)
+        objectives, themes = await self.process_campaigns(client, mapper, group.campaigns)
 
         await client.close()
 
         objective_count = len(objectives)
-        proposal_count = 0
+
+        proposals = []
+
         # Hijack `event.description` with JSON string used by the mobile app.
-        fund_goal_str = json.dumps({
-            "timestamp": strict_rfc3339.now_to_rfc3339_utcoffset(integer=True),
-            "themes": themes
-        })
+        fund_goal_str = json.dumps({"timestamp": strict_rfc3339.now_to_rfc3339_utcoffset(integer=True), "themes": themes})
 
         async with self.conn.transaction():
             try:
@@ -340,22 +341,64 @@ async def _process_campaigns(campaigns):
                 logger.error("Error updating event description", error=e)
 
             try:
-                inserted_objectives = await ideascale_importer.db.upsert_many(self.conn, objectives, conflict_cols=["id", "event"], pre_update_cols={"deleted": True}, pre_update_cond={"event": f"= {self.event_id}"})
+                inserted_objectives = await ideascale_importer.db.upsert_many(
+                    self.conn,
+                    objectives,
+                    conflict_cols=["id", "event"],
+                    pre_update_cols={"deleted": True},
+                    pre_update_cond={"event": f"= {self.event_id}"},
+                )
                 inserted_objectives_ix = {o.id: o for o in inserted_objectives}
 
-                proposals_with_campaign_id = [(a.campaign_id, mapper.map_proposal(a, self.proposals_impact_scores)) for a in ideas]
-                proposals = []
-                for objective_id, p in proposals_with_campaign_id:
-                    if objective_id in inserted_objectives_ix:
-                        p.objective = inserted_objectives_ix[objective_id].row_id
-                        proposals.append(p)
+                challenges = [
+                    objective_to_challenge_json(o, self.ideascale_api_url, idx + 1) for idx, o in enumerate(inserted_objectives)
+                ]
+                challenges_ix = {c.internal_id: c for c in challenges}
+                outuput_objs = output_dir.joinpath("challenges.json")
+                out_data = [c.model_dump() for c in challenges]
+                outuput_objs.write_text(json.dumps(out_data, indent=4))
 
-                proposal_count = len(proposals)
+                proposals, proposals_json = self.convert_ideas_to_proposals(ideas, mapper, inserted_objectives_ix, challenges_ix)
+
+                outuput_f = output_dir.joinpath("proposals.json")
+                outuput_f.write_text(json.dumps(proposals_json, indent=4))
 
                 all_objectives = await ideascale_importer.db.select(self.conn, objectives[0], cond={"event": f"= {self.event_id}"})
-                all_objectives_str = ','.join([f"{objective.row_id}" for objective in all_objectives])
-                await ideascale_importer.db.upsert_many(self.conn, proposals, conflict_cols=["id", "objective"], pre_update_cols={"deleted": True}, pre_update_cond={"objective": f"IN ({all_objectives_str})"})
+                all_objectives_str = ",".join([f"{objective.row_id}" for objective in all_objectives])
+                await ideascale_importer.db.upsert_many(
+                    self.conn,
+                    proposals,
+                    conflict_cols=["id", "objective"],
+                    pre_update_cols={"deleted": True},
+                    pre_update_cond={"objective": f"IN ({all_objectives_str})"},
+                )
+
             except Exception as e:
                 logger.error("Error updating DB objectives and proposals", error=e)
 
-        logger.info("Imported objectives and proposals", objective_count=objective_count, proposal_count=proposal_count)
+        logger.info("Imported objectives and proposals", objective_count=objective_count, proposal_count=len(proposals))
+
+    async def process_campaigns(self, client, mapper, campaigns):
+        objectives: List[Objective] = []
+        themes: List[str] = []
+        for campaign in campaigns:
+            objectives.append(mapper.map_objective(campaign, self.event_id))
+            campaign_themes = await client.event_themes(campaign.id, "f11_themes")
+            themes.extend(campaign_themes)
+        themes = list(set(themes))
+        themes.sort()
+        return objectives, themes
+
+    def convert_ideas_to_proposals(self, ideas, mapper, inserted_objectives_ix, challenges_ix):
+        proposals = []
+        proposals_json = []
+        for cnt, a in enumerate(ideas):
+            objective_id, p = a.campaign_id, mapper.map_proposal(a, self.proposals_impact_scores)
+            if objective_id in inserted_objectives_ix:
+                objective = inserted_objectives_ix[objective_id]
+                print(f"objective {objective}")
+                p.objective = objective.row_id
+                proposals.append(p)
+                p_json = json_from_proposal(p, challenges_ix[objective.id], self.event_id, cnt)
+                proposals_json.append(p_json.model_dump(exclude_none=True))
+        return proposals, proposals_json