Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Modeled Coverage - Phase 1 #578

Closed
wants to merge 50 commits into from
Closed
Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
50 commits
Select commit Hold shift + click to select a range
0479b5f
Add ingest service for coverage object
maplant Jun 21, 2023
498685e
implement basic algorithm for computing coverage points
maplant Jun 26, 2023
bab457f
Fix bug where multiple signal levels would get mixed
maplant Jun 26, 2023
60814f7
Implement logic for modeled coverage
maplant Jun 27, 2023
e52dc60
Fmt
maplant Jun 27, 2023
e35ee73
Convert rewarding to take coverage into account
maplant Jun 28, 2023
7e04c77
Convert over most of the reward share tests, add fixes
Jul 3, 2023
54bcaed
Port over one more test
Jul 5, 2023
ca0434d
Address remaining issues
Jul 7, 2023
da1ccb2
Update file_store/src/error.rs
Jul 7, 2023
8a1e2a6
Update mobile_verifier/src/coverage.rs
Jul 10, 2023
57da60b
Fix tests, add setting, improve some of the code
maplant Jul 10, 2023
ba97a7e
Add coverage_object column to heartbeats
maplant Jul 10, 2023
74d6eb8
s/no/none/g
maplant Jul 10, 2023
64fa0b3
Correct hex_coverage table
maplant Jul 11, 2023
8c52ceb
Merge remote-tracking branch 'origin/main' into map/modeled-coverage
maplant Jul 11, 2023
a1c254d
Fmt and Fix
maplant Jul 11, 2023
6619bfd
Move coverage object around to satisfy OCD
maplant Jul 11, 2023
99505a3
Clippy
maplant Jul 12, 2023
6b5c69e
Fix CellHeartbeat to allow backwards compat w/ empty coverage_object
maplant Jul 12, 2023
035e4a9
Fix coverage claim time with first_timestamp
maplant Jul 12, 2023
242ad5a
Address comments
maplant Jul 13, 2023
349a92e
Move coverage claim time adjustment outside of stream
maplant Jul 13, 2023
567bd35
Move coverage claim time adjustment into heartbeat save
maplant Jul 13, 2023
a5c9d4d
Adjust migration
maplant Jul 17, 2023
9d1dffb
Move things around and cache
maplant Jul 17, 2023
0f5cd3d
Add fun todo :-)
maplant Jul 17, 2023
cb0ae3c
Correct seniority updates
maplant Jul 18, 2023
d886af8
Merge remote-tracking branch 'origin/main' into map/modeled-coverage
maplant Jul 18, 2023
5cd1b75
Fix coverage claim time adjustment
maplant Jul 18, 2023
fee3eb3
Fix seniority
maplant Jul 19, 2023
e35c82b
CMove to insert or update model for seniority
maplant Jul 19, 2023
985c990
Refactor coverage and heartbeats a little bit
maplant Jul 21, 2023
d066c29
Merge remote-tracking branch 'origin/main' into map/modeled-coverage
maplant Jul 24, 2023
a71a601
Refactor into_rewards a bit
maplant Jul 24, 2023
1deea42
Clippy
maplant Jul 24, 2023
9c88a27
Delete old hex coverages
maplant Jul 24, 2023
45638d5
Fix deletion and validated heartbeats stream
maplant Jul 25, 2023
cf527c1
Remove code that isn't part of phase 1
maplant Jul 25, 2023
c37f583
Go back to modeled coverage migration
maplant Jul 25, 2023
f4ca43e
Address comments
maplant Jul 31, 2023
aaa172f
Fix bad insert
maplant Jul 31, 2023
f6ee1fa
No longer need mutability!
maplant Jul 31, 2023
20e656e
Fmt
maplant Jul 31, 2023
d95f40d
Fix bug where we update seniority for non valid heartbeats
maplant Jul 31, 2023
a662860
Update heartbeats to not validate coverage objects
maplant Aug 1, 2023
49c2f1e
Address comments
maplant Aug 8, 2023
3c8fb99
Merge remote-tracking branch 'origin/main' into map/modeled-coverage-…
maplant Aug 8, 2023
4223a4c
Fix error in query
maplant Aug 8, 2023
1de328f
Merge remote-tracking branch 'origin/main' into map/modeled-coverage-…
maplant Aug 17, 2023
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 2 additions & 1 deletion mobile_verifier/migrations/15_modeled_coverage.sql
Original file line number Diff line number Diff line change
Expand Up @@ -12,13 +12,14 @@ CREATE TABLE hex_coverage (
cbsd_id TEXT NOT NULL,
signal_level signal_level NOT NULL,
coverage_claim_time TIMESTAMPTZ NOT NULL,
inserted_at TIMESTAMPTZ NOT NULL,
inserted_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
PRIMARY KEY (uuid, hex)
);

CREATE TABLE seniority (
cbsd_id TEXT NOT NULL,
seniority_ts TIMESTAMPTZ NOT NULL,
inserted_at TIMESTAMPTZ NOT NULL,
last_heartbeat TIMESTAMPTZ NOT NULL,
uuid UUID NOT NULL,
PRIMARY KEY (cbsd_id, seniority_ts)
Expand Down
33 changes: 19 additions & 14 deletions mobile_verifier/src/coverage.rs
Original file line number Diff line number Diff line change
Expand Up @@ -176,9 +176,9 @@ impl CoverageObject {
sqlx::query(
r#"
INSERT INTO hex_coverage
(uuid, hex, indoor, cbsd_id, signal_level, coverage_claim_time, inserted_at)
(uuid, hex, indoor, cbsd_id, signal_level, coverage_claim_time)
VALUES
($1, $2, $3, $4, $5, $6, $7)
($1, $2, $3, $4, $5, $6)
"#,
)
.bind(self.coverage_obj.uuid)
Expand All @@ -187,7 +187,6 @@ impl CoverageObject {
.bind(&self.coverage_obj.cbsd_id)
.bind(SignalLevel::from(hex.signal_level))
.bind(self.coverage_obj.coverage_claim_time)
.bind(Utc::now())
.execute(&mut *transaction)
.await?;
}
Expand Down Expand Up @@ -445,37 +444,41 @@ impl CoverageClaimTimeCache {
cbsd_id: &str,
coverage_object: &Option<Uuid>,
exec: &mut Transaction<'_, Postgres>,
) -> Result<DateTime<Utc>, sqlx::Error> {
) -> Result<Option<DateTime<Utc>>, sqlx::Error> {
let key = (cbsd_id.to_string(), *coverage_object);
if let Some(coverage_claim_time) = self.cache.get(&key).await {
Ok(*coverage_claim_time)
Ok(Some(*coverage_claim_time))
} else {
let coverage_claim_time: DateTime<Utc> = sqlx::query_scalar(
let coverage_claim_time: Option<DateTime<Utc>> = sqlx::query_scalar(
r#"
SELECT coverage_claim_time FROM hex_coverage WHERE cbsd_id = $1 AND uuid = $2 LIMIT 1
"#,
)
.bind(cbsd_id)
.bind(coverage_object)
.fetch_one(&mut *exec)
.fetch_optional(&mut *exec)
.await?;
self.cache
.insert(
key,
coverage_claim_time,
std::time::Duration::from_secs(60 * 60 * 24),
)
.await;
if let Some(coverage_claim_time) = coverage_claim_time {
self.cache
.insert(
key,
coverage_claim_time,
std::time::Duration::from_secs(60 * 60 * 24),
)
.await;
}
Ok(coverage_claim_time)
}
}
}

#[allow(dead_code)]
pub struct CoveredHexCache {
pool: Pool<Postgres>,
covered_hexes: Arc<Cache<Uuid, CachedCoverage>>,
}

#[allow(dead_code)]
impl CoveredHexCache {
pub fn new(pool: &Pool<Postgres>) -> Self {
let cache = Arc::new(Cache::new());
Expand Down Expand Up @@ -524,11 +527,13 @@ impl CoveredHexCache {
}

#[derive(Clone)]
#[allow(dead_code)]
pub struct CachedCoverage {
pub cbsd_id: String,
coverage: Vec<CellIndex>,
}

#[allow(dead_code)]
impl CachedCoverage {
pub fn max_distance_km(&self, latlng: LatLng) -> f64 {
self.coverage.iter().fold(0.0, |curr_max, curr_cov| {
Expand Down
29 changes: 18 additions & 11 deletions mobile_verifier/src/heartbeats.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ use futures::{
stream::{Stream, StreamExt, TryStreamExt},
TryFutureExt,
};
use h3o::LatLng;
// use h3o::LatLng;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
// use h3o::LatLng;

not needed or will this come into play in phase 3?

use helium_crypto::PublicKeyBinary;
use helium_proto::services::poc_mobile as proto;
use mobile_config::{gateway_info::GatewayInfoResolver, GatewayClient};
Expand Down Expand Up @@ -132,16 +132,22 @@ impl HeartbeatDaemon {

while let Some(heartbeat) = validated_heartbeats.next().await.transpose()? {
if heartbeat.is_valid() && heartbeat.coverage_object.is_some() {
let coverage_claim_time = coverage_claim_time_cache
if let Some(coverage_claim_time) = coverage_claim_time_cache
.fetch_coverage_claim_time(
&heartbeat.cbsd_id,
&heartbeat.coverage_object,
&mut transaction,
)
.await?;
heartbeat
.update_seniority(coverage_claim_time, &self.seniority_sink, &mut transaction)
.await?;
.await?
{
heartbeat
.update_seniority(
coverage_claim_time,
&self.seniority_sink,
&mut transaction,
)
.await?;
}
}

heartbeat.write(&self.heartbeat_sink).await?;
Expand Down Expand Up @@ -393,7 +399,7 @@ impl Heartbeat {
INSERT INTO heartbeats (cbsd_id, hotspot_key, cell_type, latest_timestamp, truncated_timestamp, coverage_object)
VALUES ($1, $2, $3, $4, $5, $6)
maplant marked this conversation as resolved.
Show resolved Hide resolved
ON CONFLICT (cbsd_id, truncated_timestamp) DO UPDATE SET
latest_timestamp = EXCLUDED.latest_timestamp
latest_timestamp = EXCLUDED.latest_timestamp,
coverage_object = EXCLUDED.coverage_object
RETURNING (xmax = 0) as inserted
"#
Expand All @@ -414,9 +420,9 @@ impl Heartbeat {
async fn validate_heartbeat(
heartbeat: &CellHeartbeatIngestReport,
gateway_client: &GatewayClient,
coverage_cache: &CoveredHexCache,
_coverage_cache: &CoveredHexCache,
epoch: &Range<DateTime<Utc>>,
max_distance: f64,
_max_distance: f64,
) -> anyhow::Result<(Option<CellType>, proto::HeartbeatValidity)> {
let cell_type = match CellType::from_cbsd_id(&heartbeat.report.cbsd_id) {
Some(ty) => Some(ty),
Expand All @@ -439,9 +445,9 @@ async fn validate_heartbeat(
return Ok((cell_type, proto::HeartbeatValidity::GatewayOwnerNotFound));
}

/*
let Some(coverage_object) = heartbeat.report.coverage_object() else {
maplant marked this conversation as resolved.
Show resolved Hide resolved
return Ok((cell_type, proto::HeartbeatValidity::Valid));
// return Ok((cell_type, proto::HeartbeatValidity::BadCoverageObject));
return Ok((cell_type, proto::HeartbeatValidity::BadCoverageObject));
};

let Some(coverage) = coverage_cache.fetch_coverage(&coverage_object).await? else {
Expand All @@ -459,6 +465,7 @@ async fn validate_heartbeat(
if coverage.max_distance_km(latlng) > max_distance {
return Ok((cell_type, proto::HeartbeatValidity::TooFarFromCoverage));
}
*/
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

you said "removed" did you mean more specifically that we should be invalidating these in phase 3 so this is only temporarily commented out or was this intended to be deleted once it successfully compiled or something?


Ok((cell_type, proto::HeartbeatValidity::Valid))
}
6 changes: 0 additions & 6 deletions mobile_verifier/src/settings.rs
Original file line number Diff line number Diff line change
Expand Up @@ -27,12 +27,6 @@ pub struct Settings {
pub start_after: u64,
#[serde(default = "default_max_heartbeat_distance_from_coverage_km")]
pub max_heartbeat_distance_from_coverage_km: f64,
#[serde(default = "default_disable_discovery_loc_rewards_to_s3")]
pub disable_discovery_loc_rewards_to_s3: bool,
}

pub fn default_disable_discovery_loc_rewards_to_s3() -> bool {
true
}

pub fn default_log() -> String {
Expand Down