Merge branch 'main' into feature/allocation-tool

src/audit/balance/README.md

@@ -15,7 +15,7 @@ MOUNT_PATH=/tmp/fund9-leader-1:/leader1stuff
 HISTORICAL_STATE=/leader1stuff/persist/leader-1
 BLOCK_0=/leader1stuff/artifacts/block0.bin
 
-earthly +build && earthly +docker
+earthly +build && earthly +docker-local
 docker run  --net=host -v $MOUNT_PATH --env STORAGE_PATH=$HISTORICAL_STATE --env GENESIS_PATH=$BLOCK_0 jormungandr
 ```
 
@@ -37,4 +37,4 @@ curl http://127.0.0.1:10000/api/v0/vote/active/plans > activevoteplans.json
 ##### Make sure the jormungandr container has been stopped once you have successfully retrieved the results. 
 ```bash
 sudo docker docker stop $JORMUNGANDR_CONTAINER_ID
-```
+```

utilities/ideascale-importer/ideascale_importer/gvc.py

@@ -1,21 +1,17 @@
 """GVC API module."""
-
-from pydantic.dataclasses import dataclass
-import pydantic.tools
+from pydantic import BaseModel
 from typing import List
 
 from ideascale_importer import utils
 
 
-@dataclass
-class DrepAttributes:
+class DrepAttributes(BaseModel):
     """Represents DREP attributes from the GVC API."""
 
     voting_key: str
 
 
-@dataclass
-class Drep:
+class Drep(BaseModel):
     """Represents a DREP from the GVC API."""
 
     id: int
@@ -38,4 +34,4 @@ async def dreps(self) -> List[Drep]:
         if not isinstance(res, dict):
             raise utils.BadResponse()
 
-        return [pydantic.tools.parse_obj_as(Drep, e) for e in res["data"]]
+        return [Drep.model_validate(e) for e in res["data"]]

utilities/ideascale-importer/ideascale_importer/ideascale/client.py

@@ -2,16 +2,14 @@
 
 import asyncio
 import json
-from pydantic.dataclasses import dataclass
-import pydantic.tools
+from pydantic import BaseModel, Field
 from typing import Any, Iterable, List, Mapping
 
 from ideascale_importer import utils
 from ideascale_importer.utils import GetFailed
 
 
-@dataclass
-class Campaign:
+class Campaign(BaseModel):
     """Represents a campaign from IdeaScale.
 
     (Contains only the fields that are used by the importer).
@@ -25,8 +23,7 @@ class Campaign:
     campaign_url: str
 
 
-@dataclass
-class CampaignGroup:
+class CampaignGroup(BaseModel):
     """Represents a campaign group from IdeaScale.
 
     (Contains only the fields that are used by the importer).
@@ -37,8 +34,7 @@ class CampaignGroup:
     campaigns: List[Campaign]
 
 
-@dataclass
-class IdeaAuthorInfo:
+class IdeaAuthorInfo(BaseModel):
     """Represents an author info from IdeaScale.
 
     (Contains only the fields that are used by the importer).
@@ -47,8 +43,7 @@ class IdeaAuthorInfo:
     name: str
 
 
-@dataclass
-class Idea:
+class Idea(BaseModel):
     """Represents an idea from IdeaScale.
 
     (Contains only the fields that are used by the importer).
@@ -61,15 +56,14 @@ class Idea:
     author_info: IdeaAuthorInfo
     contributors: List[IdeaAuthorInfo]
     url: str
-    custom_fields_by_key: Mapping[str, str] = pydantic.Field(default={})
+    custom_fields_by_key: Mapping[str, str] = Field(default={})
 
     def contributors_name(self) -> List[str]:
         """Get the names of all contributors."""
         return list(map(lambda c: c.name, self.contributors))
 
 
-@dataclass
-class Stage:
+class Stage(BaseModel):
     """Represents a stage from IdeaScale.
 
     (Contains only the fields that are used by the importer).
@@ -81,8 +75,7 @@ class Stage:
     funnel_name: str
 
 
-@dataclass
-class Funnel:
+class Funnel(BaseModel):
     """Represents a funnel from IdeaScale.
 
     (Contains only the fields that are used by the importer).
@@ -125,7 +118,7 @@ async def campaigns(self, group_id: int) -> List[Campaign]:
             if "campaigns" in group:
                 group_campaigns = []
                 for c in group["campaigns"]:
-                    group_campaigns.append(pydantic.tools.parse_obj_as(Campaign, c))
+                    group_campaigns.append(Campaign.model_validate(c))
                     await asyncio.sleep(0)
 
                 campaigns.extend(group_campaigns)
@@ -138,7 +131,7 @@ async def campaign_groups(self) -> List[CampaignGroup]:
 
         campaign_groups: List[CampaignGroup] = []
         for cg in res:
-            campaign_groups.append(pydantic.tools.parse_obj_as(CampaignGroup, cg))
+            campaign_groups.append(CampaignGroup.model_validate(cg))
             await asyncio.sleep(0)
 
         return campaign_groups
@@ -149,7 +142,7 @@ async def campaign_ideas(self, campaign_id: int) -> List[Idea]:
 
         ideas = []
         for i in res:
-            ideas.append(pydantic.tools.parse_obj_as(Idea, i))
+            ideas.append(Idea.model_validate(i))
             await asyncio.sleep(0)
 
         return ideas
@@ -178,7 +171,7 @@ async def worker(d: WorkerData):
 
                 res_ideas: List[Idea] = []
                 for i in res:
-                    res_ideas.append(pydantic.tools.parse_obj_as(Idea, i))
+                    res_ideas.append(Idea.model_validate(i))
 
                 d.ideas.extend(res_ideas)
 
@@ -214,7 +207,7 @@ async def campaign_group_ideas(self, group_id: int) -> List[Idea]:
     async def funnel(self, funnel_id: int) -> Funnel:
         """Get the funnel with the given id."""
         res = await self._get(f"/a/rest/v1/funnels/{funnel_id}")
-        return pydantic.tools.parse_obj_as(Funnel, res)
+        return Funnel.model_validate(res)
 
     async def _get(self, path: str) -> Mapping[str, Any] | Iterable[Mapping[str, Any]]:
         """Execute a GET request on IdeaScale API."""

utilities/ideascale-importer/ideascale_importer/ideascale/importer.py

@@ -3,11 +3,9 @@
 import re
 import asyncpg
 import csv
-from dataclasses import dataclass
-import json
 from loguru import logger
 from markdownify import markdownify
-import pydantic
+from pydantic import BaseModel
 from typing import Any, Dict, List, Mapping, Optional, Union
 
 from .client import Campaign, CampaignGroup, Client, Idea
@@ -17,8 +15,7 @@
 FieldMapping = Union[str, List[str]]
 
 
-@dataclass
-class ProposalsFieldsMappingConfig:
+class ProposalsFieldsMappingConfig(BaseModel):
     """Represents the available configuration fields used in proposal fields mapping."""
 
     proposer_url: FieldMapping
@@ -27,35 +24,32 @@ class ProposalsFieldsMappingConfig:
     public_key: FieldMapping
 
 
-@dataclass
-class ProposalsConfig:
+class ProposalsConfig(BaseModel):
     """Represents the available configuration fields used in proposal processing."""
 
     field_mappings: ProposalsFieldsMappingConfig
     extra_field_mappings: Mapping[str, FieldMapping]  # noqa: F821
 
 
-@dataclass
-class ProposalsScoresCsvConfig:
+class ProposalsScoresCsvConfig(BaseModel):
     """Represents the available configuration fields for proposal scores from the CSV file."""
 
     id_field: str
     score_field: str
 
 
-@dataclass
-class Config:
+class Config(BaseModel):
     """Represents the available configuration fields."""
 
     campaign_group_id: int
     stage_ids: List[int]
     proposals: ProposalsConfig
     proposals_scores_csv: ProposalsScoresCsvConfig
-    
+
     @staticmethod
     def from_json(val: dict):
         """Load configuration from a JSON object."""
-        return pydantic.tools.parse_obj_as(Config, val)
+        return Config.model_validate(val)
 
 class ReadProposalsScoresCsv(Exception):
     """Raised when the proposals impact scores csv cannot be read."""
@@ -165,8 +159,7 @@ def html_to_md(s: str) -> str:
     return markdownify(s, strip=tags_to_strip).strip()
 
 
-@dataclass
-class Reward:
+class Reward(BaseModel):
     """Represents a reward."""
 
     amount: int
@@ -257,7 +250,7 @@ async def load_config(self):
         if len(res) == 0:
             raise Exception("Cannot find ideascale config in the event-db database")
         self.config = Config.from_json(res[0].value)
-        
+
     async def connect(self, *args, **kwargs):
         """Connect to the database."""
         if self.conn is None:
@@ -310,7 +303,7 @@ async def run(self):
         proposal_count = 0
 
         async with self.conn.transaction():
-            inserted_objectives = await ideascale_importer.db.upsert_many(self.conn, objectives, conflict_cols=["id", "event"], pre_update_cols={"deleted": True}, pre_update_cond={"event": f"= {self.event_id}"})            
+            inserted_objectives = await ideascale_importer.db.upsert_many(self.conn, objectives, conflict_cols=["id", "event"], pre_update_cols={"deleted": True}, pre_update_cond={"event": f"= {self.event_id}"})
             inserted_objectives_ix = {o.id: o for o in inserted_objectives}
 
             proposals_with_campaign_id = [(a.campaign_id, mapper.map_proposal(a, self.proposals_impact_scores)) for a in ideas]