Automatically download data sets (#760)

Cargo.toml

@@ -116,6 +116,7 @@ itertools = "*"
 tokio-util = "0"
 uuid = { version = "1", features = ["v4", "serde"] }
 tower-http = { version = "0", features = ["trace"] }
+derive_builder = "0"
 
 [patch.crates-io]
 sqlx = { git = "https://github.com/helium/sqlx.git", rev = "92a2268f02e0cac6fccb34d3e926347071dbb88d" }

file_store/Cargo.toml

@@ -45,7 +45,7 @@ base64 = {workspace = true}
 beacon = {workspace = true}
 sqlx = {workspace = true, optional = true}
 async-trait = {workspace = true}
-derive_builder = "0"
+derive_builder = {workspace = true}
 retainer = {workspace = true}
 uuid = {workspace = true}
 h3o = {workspace = true}

file_store/src/file_info.rs

@@ -154,6 +154,9 @@ pub const COVERAGE_OBJECT_INGEST_REPORT: &str = "coverage_object_ingest_report";
 pub const SENIORITY_UPDATE: &str = "seniority_update";
 pub const BOOSTED_HEX_UPDATE: &str = "boosted_hex_update";
 pub const ORACLE_BOOSTING_REPORT: &str = "oracle_boosting_report";
+pub const URBANIZATION_DATA_SET: &str = "urbanization";
+pub const FOOTFALL_DATA_SET: &str = "footfall";
+pub const LANDTYPE_DATA_SET: &str = "landtype";
 
 #[derive(Debug, PartialEq, Eq, Clone, Serialize, Copy, strum::EnumCount)]
 #[serde(rename_all = "snake_case")]
@@ -200,6 +203,9 @@ pub enum FileType {
     RadioThresholdReq,
     RadioThresholdIngestReport,
     VerifiedRadioThresholdIngestReport,
+    UrbanizationDataSet,
+    FootfallDataSet,
+    LandtypeDataSet,
     InvalidatedRadioThresholdReq,
     InvalidatedRadioThresholdIngestReport,
     VerifiedInvalidatedRadioThresholdIngestReport,
@@ -261,6 +267,9 @@ impl fmt::Display for FileType {
             Self::SeniorityUpdate => SENIORITY_UPDATE,
             Self::BoostedHexUpdate => BOOSTED_HEX_UPDATE,
             Self::OracleBoostingReport => ORACLE_BOOSTING_REPORT,
+            Self::UrbanizationDataSet => URBANIZATION_DATA_SET,
+            Self::FootfallDataSet => FOOTFALL_DATA_SET,
+            Self::LandtypeDataSet => LANDTYPE_DATA_SET,
         };
         f.write_str(s)
     }
@@ -322,6 +331,9 @@ impl FileType {
             Self::SeniorityUpdate => SENIORITY_UPDATE,
             Self::BoostedHexUpdate => BOOSTED_HEX_UPDATE,
             Self::OracleBoostingReport => ORACLE_BOOSTING_REPORT,
+            Self::UrbanizationDataSet => URBANIZATION_DATA_SET,
+            Self::FootfallDataSet => FOOTFALL_DATA_SET,
+            Self::LandtypeDataSet => LANDTYPE_DATA_SET,
         }
     }
 }
@@ -383,6 +395,9 @@ impl FromStr for FileType {
             SENIORITY_UPDATE => Self::SeniorityUpdate,
             BOOSTED_HEX_UPDATE => Self::BoostedHexUpdate,
             ORACLE_BOOSTING_REPORT => Self::OracleBoostingReport,
+            URBANIZATION_DATA_SET => Self::UrbanizationDataSet,
+            FOOTFALL_DATA_SET => Self::FootfallDataSet,
+            LANDTYPE_DATA_SET => Self::LandtypeDataSet,
             _ => return Err(Error::from(io::Error::from(io::ErrorKind::InvalidInput))),
         };
         Ok(result)

mobile_verifier/Cargo.toml

@@ -8,6 +8,7 @@ authors.workspace = true
 
 [dependencies]
 anyhow = { workspace = true }
+async-compression = { version = "0", features = ["tokio", "gzip"] }
 config = { workspace = true }
 thiserror = { workspace = true }
 serde = { workspace = true }
@@ -37,6 +38,7 @@ rust_decimal = { workspace = true }
 rust_decimal_macros = { workspace = true }
 tonic = { workspace = true }
 tokio-stream = { workspace = true }
+tokio-util = { workspace = true }
 metrics = { workspace = true }
 metrics-exporter-prometheus = { workspace = true }
 mobile-config = { path = "../mobile_config" }
@@ -51,6 +53,8 @@ retainer = { workspace = true }
 uuid = { workspace = true }
 task-manager = { path = "../task_manager" }
 solana-sdk = { workspace = true }
+derive_builder = { workspace = true }
+regex = "1"
 humantime-serde = { workspace = true }
 custom-tracing = { path = "../custom_tracing" }
 

mobile_verifier/migrations/33_data_sets.sql

@@ -0,0 +1,26 @@
+DO $$ BEGIN
+CREATE TYPE data_set_status AS enum (
+       'pending',
+       'downloaded',
+       'processed'
+);
+EXCEPTION
+    WHEN duplicate_object THEN null;
+END $$;
+
+DO $$ BEGIN
+CREATE TYPE data_set_type AS enum (
+       'urbanization',
+       'footfall',
+       'landtype'
+);
+EXCEPTION
+    WHEN duplicate_object THEN null;
+END $$;
+
+CREATE TABLE IF NOT EXISTS hex_assignment_data_set_status (
+       filename TEXT PRIMARY KEY,
+       data_set data_set_type NOT NULL,
+       time_to_use TIMESTAMPTZ NOT NULL,
+       status data_set_status NOT NULL
+);

mobile_verifier/src/boosting_oracles/assignment.rs

@@ -1,8 +1,11 @@
+use anyhow::Result;
 use helium_proto::services::poc_mobile::oracle_boosting_hex_assignment::Assignment as ProtoAssignment;
 use rust_decimal::Decimal;
 use rust_decimal_macros::dec;
 use std::fmt;
 
+use super::HexAssignment;
+
 #[derive(Debug, Clone, PartialEq, Eq, sqlx::FromRow)]
 pub struct HexAssignments {
     pub footfall: Assignment,
@@ -58,11 +61,20 @@ impl fmt::Display for Assignment {
 }
 
 impl HexAssignments {
+    pub fn builder(cell: hextree::Cell) -> HexAssignmentsBuilder {
+        HexAssignmentsBuilder {
+            cell,
+            footfall: None,
+            landtype: None,
+            urbanized: None,
+        }
+    }
+
     pub fn boosting_multiplier(&self) -> Decimal {
         let HexAssignments {
             footfall,
-            urbanized,
             landtype,
+            urbanized,
         } = self;
 
         use Assignment::*;
@@ -97,6 +109,47 @@ impl HexAssignments {
     }
 }
 
+pub struct HexAssignmentsBuilder {
+    cell: hextree::Cell,
+    footfall: Option<Result<Assignment>>,
+    landtype: Option<Result<Assignment>>,
+    urbanized: Option<Result<Assignment>>,
+}
+
+impl HexAssignmentsBuilder {
+    pub fn footfall(mut self, footfall: &impl HexAssignment) -> Self {
+        self.footfall = Some(footfall.assignment(self.cell));
+        self
+    }
+
+    pub fn landtype(mut self, landtype: &impl HexAssignment) -> Self {
+        self.landtype = Some(landtype.assignment(self.cell));
+        self
+    }
+
+    pub fn urbanized(mut self, urbanized: &impl HexAssignment) -> Self {
+        self.urbanized = Some(urbanized.assignment(self.cell));
+        self
+    }
+
+    pub fn build(self) -> anyhow::Result<HexAssignments> {
+        let Some(footfall) = self.footfall else {
+            anyhow::bail!("footfall assignment not set");
+        };
+        let Some(landtype) = self.landtype else {
+            anyhow::bail!("landtype assignment not set");
+        };
+        let Some(urbanized) = self.urbanized else {
+            anyhow::bail!("urbanized assignment not set");
+        };
+        Ok(HexAssignments {
+            footfall: footfall?,
+            urbanized: urbanized?,
+            landtype: landtype?,
+        })
+    }
+}
+
 #[cfg(test)]
 impl HexAssignments {
     pub fn test_best() -> Self {