From 79195400becaed98e127ecc36c61580885ba7740 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Rosales?= Date: Tue, 23 Jan 2024 12:21:37 -0600 Subject: [PATCH] feat: add cli option for ideascale output_dir, generate funds.json artifact --- .../ideascale_importer/cli/ideascale.py | 16 ++++++++- .../ideascale_importer/db/__init__.py | 25 +++++++++++-- .../ideascale_importer/ideascale/artifacts.py | 3 +- .../ideascale_importer/ideascale/importer.py | 36 +++++++++++-------- 4 files changed, 60 insertions(+), 20 deletions(-) diff --git a/utilities/ideascale-importer/ideascale_importer/cli/ideascale.py b/utilities/ideascale-importer/ideascale_importer/cli/ideascale.py index f0c3438992..12f83baac7 100644 --- a/utilities/ideascale-importer/ideascale_importer/cli/ideascale.py +++ b/utilities/ideascale-importer/ideascale_importer/cli/ideascale.py @@ -1,6 +1,7 @@ """IdeaScale CLI commands.""" import asyncio +from pathlib import Path from typing import Optional import typer @@ -39,6 +40,9 @@ def import_all( envvar="IDEASCALE_API_URL", help="IdeaScale API URL", ), + output_dir: Optional[str] = typer.Option( + default=None, envvar="IDEASCALE_OUTPUT_DIR", help="Output directory for generated files" + ), ): """Import all event data from IdeaScale for a given event.""" configure_logger(log_level, log_format) @@ -47,13 +51,23 @@ async def inner( event_id: int, proposals_scores_csv_path: Optional[str], ideascale_api_url: str, + output_dir: Optional[str] ): + # check if output_dir path exists, or create otherwise + if output_dir is None: + logger.info("No output directory was defined.") + else: + output_dir = Path(output_dir) + output_dir.mkdir(exist_ok=True, parents=True) + logger.info(f"Output directory for artifacts: {output_dir}") + importer = Importer( api_token, database_url, event_id, proposals_scores_csv_path, ideascale_api_url, + output_dir ) try: @@ -63,4 +77,4 @@ async def inner( except Exception as e: logger.error(e) - asyncio.run(inner(event_id, proposals_scores_csv, ideascale_api_url)) + asyncio.run(inner(event_id, proposals_scores_csv, ideascale_api_url, output_dir)) diff --git a/utilities/ideascale-importer/ideascale_importer/db/__init__.py b/utilities/ideascale-importer/ideascale_importer/db/__init__.py index ade6ce92fc..3692a6146e 100644 --- a/utilities/ideascale-importer/ideascale_importer/db/__init__.py +++ b/utilities/ideascale-importer/ideascale_importer/db/__init__.py @@ -64,6 +64,7 @@ async def insert(conn: asyncpg.Connection, model: Model) -> Any: return ret[0] return None + async def select(conn: asyncpg.Connection, model: Model, cond: Dict[str, str] = {}) -> List[Any]: """Select a single model.""" @@ -77,7 +78,7 @@ async def select(conn: asyncpg.Connection, model: Model, cond: Dict[str, str] = SELECT {cols_str} FROM {model.table()} {f' WHERE {cond_str}' if cond_str else ' '} - """.strip() + """.strip() result = await conn.fetch(stmt_template) @@ -123,9 +124,13 @@ async def upsert_many( pre_update_set_str = ",".join([f"{col} = {val}" for col, val in pre_update_cols.items()]) pre_update_cond_str = " ".join([f"{col} {cond}" for col, cond in pre_update_cond.items()]) - pre_update_template = f""" + pre_update_template = ( + f""" WITH updated AS ({ f"UPDATE {models[0].table()} SET {pre_update_set_str} {f' WHERE {pre_update_cond_str}' if pre_update_cond_str else ' '}" }) - """.strip() if pre_update_set_str else " " + """.strip() + if pre_update_set_str + else " " + ) stmt_template = f""" {pre_update_template} @@ -172,6 +177,20 @@ async def event_exists(conn: asyncpg.Connection, id: int) -> bool: return row is not None +class EventThesholdNotFound(Exception): + """Raised when the event's voting power threshold is not found.""" + + ... + + +async def event_threshold(conn: asyncpg.Connection, row_id: int) -> int: + """Fetch the event's voting power threshold in ADA.""" + res = await conn.fetchrow("SELECT voting_power_threshold FROM event WHERE row_id = $1", row_id) + if res is None: + raise EventThesholdNotFound() + threshold = int(res["voting_power_threshold"]/1000000) + return threshold + async def update_event_description(conn: asyncpg.Connection, row_id: int, description: str): """Update the event description. diff --git a/utilities/ideascale-importer/ideascale_importer/ideascale/artifacts.py b/utilities/ideascale-importer/ideascale_importer/ideascale/artifacts.py index feac585d5d..1094f3fd41 100644 --- a/utilities/ideascale-importer/ideascale_importer/ideascale/artifacts.py +++ b/utilities/ideascale-importer/ideascale_importer/ideascale/artifacts.py @@ -87,7 +87,8 @@ def json_from_proposal(prop: Proposal, challenge: ChallengesJson, fund_id: int, class FundsJson(BaseModel): + """Current Fund (Event) information in JSON used for output artifacts.""" id: int goal: str threshold: int - rewards_info: str + rewards_info: str = "" diff --git a/utilities/ideascale-importer/ideascale_importer/ideascale/importer.py b/utilities/ideascale-importer/ideascale_importer/ideascale/importer.py index 29f191490e..ae18f0138d 100644 --- a/utilities/ideascale-importer/ideascale_importer/ideascale/importer.py +++ b/utilities/ideascale-importer/ideascale_importer/ideascale/importer.py @@ -13,7 +13,7 @@ from typing import Any, Dict, List, Mapping, Optional, Union from ideascale_importer.db.models import Objective -from ideascale_importer.ideascale.artifacts import json_from_proposal, objective_to_challenge_json +from ideascale_importer.ideascale.artifacts import FundsJson, json_from_proposal, objective_to_challenge_json from .client import Campaign, CampaignGroup, Client, Idea import ideascale_importer.db @@ -232,6 +232,7 @@ def __init__( event_id: int, proposals_scores_csv_path: Optional[str], ideascale_api_url: str, + output_dir: Optional[Path], ): """Initialize the importer.""" self.api_token = api_token @@ -239,6 +240,7 @@ def __init__( self.event_id = event_id self.conn: asyncpg.Connection | None = None self.ideascale_api_url = ideascale_api_url + self.output_dir = output_dir self.proposals_impact_scores: Dict[int, int] = {} if proposals_scores_csv_path is not None: @@ -286,10 +288,6 @@ async def run(self): await self.load_config() - output_dir = Path(tempfile.gettempdir()).joinpath("catalyst-artifacts") - output_dir.mkdir(exist_ok=True) - logger.debug("Created temporary directory for artifact storage", output_dir=output_dir) - if not await ideascale_importer.db.event_exists(self.conn, self.event_id): logger.error("No event exists with the given id") return @@ -315,9 +313,6 @@ async def run(self): for stage_id in self.config.stage_ids: ideas.extend(await client.stage_ideas(stage_id=stage_id)) - outuput_ideas = output_dir.joinpath("ideas.json") - out_data = [i.model_dump() for i in ideas] - outuput_ideas.write_text(json.dumps(out_data, indent=2)) vote_options_id = await ideascale_importer.db.get_vote_options_id(self.conn, ["yes", "no"]) # mapper used to convert ideascale data to db and json formats. @@ -332,8 +327,17 @@ async def run(self): proposals = [] # Hijack `event.description` with JSON string used by the mobile app. - fund_goal_str = json.dumps({"timestamp": strict_rfc3339.now_to_rfc3339_utcoffset(integer=True), "themes": themes}) + fund_goal = {"timestamp": strict_rfc3339.now_to_rfc3339_utcoffset(integer=True), "themes": themes} + fund_goal_str = json.dumps(fund_goal) + + threshold = await ideascale_importer.db.event_threshold(self.conn, self.event_id) + funds_json = FundsJson(id=self.event_id, goal=str(fund_goal), threshold=threshold) + + if self.output_dir is not None: + outuput_ideas = self.output_dir.joinpath("funds.json") + out_data = funds_json.model_dump() + outuput_ideas.write_text(json.dumps(out_data, indent=4)) async with self.conn.transaction(): try: await ideascale_importer.db.update_event_description(self.conn, self.event_id, fund_goal_str) @@ -354,14 +358,17 @@ async def run(self): objective_to_challenge_json(o, self.ideascale_api_url, idx + 1) for idx, o in enumerate(inserted_objectives) ] challenges_ix = {c.internal_id: c for c in challenges} - outuput_objs = output_dir.joinpath("challenges.json") - out_data = [c.model_dump() for c in challenges] - outuput_objs.write_text(json.dumps(out_data, indent=4)) + + if self.output_dir is not None: + outuput_objs = self.output_dir.joinpath("challenges.json") + out_data = [c.model_dump() for c in challenges] + outuput_objs.write_text(json.dumps(out_data, indent=4)) proposals, proposals_json = self.convert_ideas_to_proposals(ideas, mapper, inserted_objectives_ix, challenges_ix) - outuput_f = output_dir.joinpath("proposals.json") - outuput_f.write_text(json.dumps(proposals_json, indent=4)) + if self.output_dir is not None: + outuput_f = self.output_dir.joinpath("proposals.json") + outuput_f.write_text(json.dumps(proposals_json, indent=4)) all_objectives = await ideascale_importer.db.select(self.conn, objectives[0], cond={"event": f"= {self.event_id}"}) all_objectives_str = ",".join([f"{objective.row_id}" for objective in all_objectives]) @@ -396,7 +403,6 @@ def convert_ideas_to_proposals(self, ideas, mapper, inserted_objectives_ix, chal objective_id, p = a.campaign_id, mapper.map_proposal(a, self.proposals_impact_scores) if objective_id in inserted_objectives_ix: objective = inserted_objectives_ix[objective_id] - print(f"objective {objective}") p.objective = objective.row_id proposals.append(p) p_json = json_from_proposal(p, challenges_ix[objective.id], self.event_id, cnt)