Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Added JsonForecast datasource #312

Closed
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions docs/selecting-a-data-source.md
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ configuration fields that must be set in order to access the raw data.
| Type | WattTime | ElectricityMaps | ElectricityMapsFree | JSON |
|-------------------------------|-----------|------------------------------------------------------------------------------------|---------------------|----------|
| Is Emissions DataSource | ✅ | ✅ | ✅ | ✅ |
| Is Forecast DataSource | ✅ | ✅ | ❌ | ❌ |
| Is Forecast DataSource | ✅ | ✅ | ❌ | ✅ |
| Makes HTTP(s) call | ✅ | ✅ | ✅ | ❌ |
| Can Use Custom Data | ❌ | ❌ | ❌ | ✅ |
| Supports Trial + Full Account | ✅ | ✅ (\*[see restriction below](#restrictions-electricitymaps-free-trial-user)) | N/A | N/A |
Expand All @@ -34,7 +34,7 @@ Not all data sources support all the routes provided in the interfaces
| Methods | WattTime | ElectricityMaps | ElectricityMapsFree | JSON | CLI Usage | Web Api Usage | SDK Usage |
| ----------------------- | :------: | :-------------: | :-----------------: | :------: | :--------------------------------------------------------------------: | :-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------: | :-----------------------------------------------------------------------------------------------------------: |
| GetCarbonIntensityAsync | ✅ | ✅ | ✅ | ✅ | `emissions` | `emissions/bylocation` or `emissions/bylocations` or `emissions/bylocations/best` or `emissions/average`‑`carbon`‑`intensity` or `emissions/average`‑`carbon`‑`intensity/batch` | `GetEmissionsDataAsync(...)` or `GetBestEmissionsDataAsync(...)` or `GetAverageCarbonIntensityDataAsync(...)` |
| GetCurrentForecastAsync | ✅ | ✅ | ❌ | ❌ | `emissions`‑`forecasts` | `forecasts/current` | `GetCurrentForecastAsync(...)` |
| GetCurrentForecastAsync | ✅ | ✅ | ❌ | ✅ | `emissions`‑`forecasts` | `forecasts/current` | `GetCurrentForecastAsync(...)` |
| GetForecastByDateAsync | ✅ | ❌ | ❌ | ❌ | `emissions`‑`forecasts` ‑‑`requested`‑`at` | `forecasts/batch` with `requestedAt` field | `GetForecastByDateAsync(...)` |

## Location Coverage
Expand Down
4 changes: 2 additions & 2 deletions samples/lib-integration/ConsoleApp/Program.cs
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,8 @@
.BuildServiceProvider();
var handlerEmissions = serviceProvider.GetRequiredService<IEmissionsHandler>();

const string startDate = "2022-03-01T15:30:00Z";
const string endDate = "2022-03-01T18:30:00Z";
const string startDate = "2023-03-10T15:30:00Z";
const string endDate = "2023-03-11T18:30:00Z";
const string location = "eastus";

var parsedStart = DateTimeOffset.Parse(startDate);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ namespace CarbonAware.CLI.IntegrationTests.Commands.EmissionsForecasts;
/// Tests that the CLI handles and packages various responses from handlers
/// and data sources properly, including empty responses and exceptions.
/// </summary>
[TestFixture(DataSourceType.JSON)]
[TestFixture(DataSourceType.WattTime)]
[TestFixture(DataSourceType.ElectricityMaps)]
//[TestFixture(DataSourceType.ElectricityMapsFree)] // Left out because these tests are not relevant for this data source
Expand Down Expand Up @@ -96,7 +97,7 @@ public async Task EmissionsForecasts_StartAndEndOptions_ReturnsExpectedData()
[Test]
public async Task EmissionsForecasts_RequestedAtOptions_ReturnsExpectedData()
{
IgnoreTestForDataSource("data source does not implement '--requested-at'", DataSourceType.ElectricityMaps);
IgnoreTestForDataSource("data source does not implement '--requested-at'", DataSourceType.ElectricityMaps, DataSourceType.JSON);

// Arrange
_dataSourceMocker.SetupBatchForecastMock();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,9 @@ public void Setup()
case DataSourceType.JSON:
{
Environment.SetEnvironmentVariable("DataSources__EmissionsDataSource", "Json");
Environment.SetEnvironmentVariable("DataSources__Configurations__Json__Type", "JSON");
Environment.SetEnvironmentVariable("DataSources__ForecastDataSource", "Json");
Environment.SetEnvironmentVariable("DataSources__Configurations__Json__Type", "Json");
Environment.SetEnvironmentVariable("DataSources__Configurations__Json__DataFileLocation", "test-data-azure-emissions.json");
_dataSourceMocker = new JsonDataSourceMocker();
break;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,6 @@ private static EmissionsForecast ToEmissionsForecast(Location location, Forecast
/// <inheritdoc />
public async Task<EmissionsForecast> GetCarbonIntensityForecastAsync(Location location, DateTimeOffset requestedAt)
{
await Task.Run(() => true);
in4margaret marked this conversation as resolved.
Show resolved Hide resolved
throw new NotImplementedException();
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,14 +30,53 @@ public void SetupDataMock(DateTimeOffset start, DateTimeOffset end, string locat
pointTime = newDataPoint.Time + duration;
}

var json = new
var json = new EmissionsJsonFile
{
Emissions = data
Emissions = data,
};

File.WriteAllText(path, JsonSerializer.Serialize(json));
}
public void SetupForecastMock()
{
List<EmissionsForecast> emissionsForecasts = new List<EmissionsForecast>();

var locations = new string[] { "eastus", "westus" };

var ran = new Random(DateTimeOffset.Now.Millisecond);
var startTime = DateTimeOffset.Now;
var maxMinutesOffset = 24 * 60;

foreach (var location in locations)
{
List<EmissionsData> emissionsData = new List<EmissionsData>();

for (var minutes = 0; minutes < maxMinutesOffset; minutes += 5)
{
var e = new EmissionsData
{
Time = startTime + TimeSpan.FromMinutes(minutes),
Location = location ?? string.Empty,
Rating = ran.Next(100)
};
emissionsData.Add(e);
}

emissionsForecasts.Add(new EmissionsForecast
{
Location = new Location { Name = location },
ForecastData = emissionsData,
});
}

string path = new JsonDataSourceConfiguration().DataFileLocation;
var json = new EmissionsJsonFile
{
EmissionsForecasts = emissionsForecasts,
};

File.WriteAllText(path, JsonSerializer.Serialize(json));
}
public void SetupForecastMock() { }
public void Initialize() { }
public void Reset() { }
public void Dispose() { }
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ namespace CarbonAware.DataSources.Json.Configuration;
/// <summary>
/// A configuration class for holding Json Data config values.
/// </summary>
internal class JsonDataSourceConfiguration
public class JsonDataSourceConfiguration
{
private const string BaseDirectory = "data-sources/json";
private const string DefaultDataFile = "test-data-azure-emissions.json";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,26 @@ namespace CarbonAware.DataSources.Json.Configuration;

internal static class ServiceCollectionExtensions
{
public static void AddJsonEmissionsDataSource(this IServiceCollection services, DataSourcesConfiguration dataSourcesConfig)
public static IServiceCollection AddJsonEmissionsDataSource(this IServiceCollection services, DataSourcesConfiguration dataSourcesConfig)
{
// configuring dependency injection to have config.
services.Configure<JsonDataSourceConfiguration>(config =>
{
dataSourcesConfig.EmissionsConfigurationSection().Bind(config);
});
services.TryAddSingleton<IEmissionsDataSource, JsonDataSource>();

return services;
}

public static IServiceCollection AddJsonForecastDataSource(this IServiceCollection services, DataSourcesConfiguration dataSourcesConfig)
{
services.Configure<JsonDataSourceConfiguration>(config =>
{
dataSourcesConfig.ForecastConfigurationSection().Bind(config);
});
services.TryAddSingleton<IForecastDataSource, JsonDataSource>();

return services;
}
}
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
using CarbonAware.Interfaces;
using CarbonAware.DataSources.Json.Configuration;
using CarbonAware.DataSources.Json.Configuration;
using CarbonAware.Exceptions;
using CarbonAware.Interfaces;
using CarbonAware.Model;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
Expand All @@ -10,7 +11,7 @@ namespace CarbonAware.DataSources.Json;
/// <summary>
/// Represents a JSON data source.
/// </summary>
internal class JsonDataSource : IEmissionsDataSource
public class JsonDataSource : IEmissionsDataSource, IForecastDataSource
{
public string Name => "JsonDataSource";

Expand All @@ -24,14 +25,14 @@ internal class JsonDataSource : IEmissionsDataSource

private List<EmissionsData>? _emissionsData;

private List<EmissionsForecast>? _emissionsForecasts;

private readonly ILogger<JsonDataSource> _logger;

private IOptionsMonitor<JsonDataSourceConfiguration> _configurationMonitor { get; }

private JsonDataSourceConfiguration _configuration => _configurationMonitor.CurrentValue;



/// <summary>
/// Creates a new instance of the <see cref="JsonDataSource"/> class.
/// </summary>
Expand Down Expand Up @@ -73,6 +74,32 @@ public async Task<IEnumerable<EmissionsData>> GetCarbonIntensityAsync(IEnumerabl
return emissionsData;
}

public async Task<EmissionsForecast> GetCurrentCarbonIntensityForecastAsync(Location location)
{
_logger.LogInformation("JSON data source getting current carbon intensity forecast for location {locations}", location);

IEnumerable<EmissionsForecast>? emissionsForecasts = await GetJsonEmissionsForecastsAsync();
if (emissionsForecasts is null || !emissionsForecasts.Any())
{
throw new CarbonAwareException("EmissionsForecasts data is empty in JSON data source file.");
}

var emissionsForecast = GetEmissionsForecastByLocation(emissionsForecasts, location);

if (emissionsForecast is null)
{
throw new CarbonAwareException($"JSON data source file doesn't contain forecast data for location: {location}.");
}

return emissionsForecast;
}

public Task<EmissionsForecast> GetCarbonIntensityForecastAsync(Location location, DateTimeOffset requestedAt)
{
// Similarly to ElectricityMaps Data Source this Data Source doesn't implement RequestedAt since JSON source is static.
throw new NotImplementedException();
}

private IEnumerable<EmissionsData> FilterByDateRange(IEnumerable<EmissionsData> data, DateTimeOffset startTime, DateTimeOffset endTime)
{
var (newStartTime, newEndTime) = IntervalHelper.ExtendTimeByWindow(startTime, endTime, MinSamplingWindow);
Expand All @@ -96,24 +123,41 @@ private IEnumerable<EmissionsData> FilterByLocation(IEnumerable<EmissionsData> d
return data;
}

private EmissionsForecast? GetEmissionsForecastByLocation(IEnumerable<EmissionsForecast> data, Location location)
{
return data.Where(ef => location.Name == ef.Location.Name).SingleOrDefault();
}

protected virtual async Task<List<EmissionsData>?> GetJsonDataAsync()
{
if (_emissionsData is not null)
{
return _emissionsData;
}
using Stream stream = GetStreamFromFileLocation();
var jsonObject = await JsonSerializer.DeserializeAsync<EmissionsJsonFile>(stream);
}
if (_emissionsData is null || !_emissionsData.Any())
{
_emissionsData = jsonObject?.Emissions;
_emissionsData = (await GetEmissionsJsonFile())?.Emissions;
}
return _emissionsData;
}

private Stream GetStreamFromFileLocation()
protected virtual async Task<List<EmissionsForecast>?> GetJsonEmissionsForecastsAsync()
{
_logger.LogInformation($"Reading Json data from {_configuration.DataFileLocation}");
return File.OpenRead(_configuration.DataFileLocation!);
if (_emissionsForecasts is not null)
{
return _emissionsForecasts;
}
if (_emissionsForecasts is null || !_emissionsForecasts.Any())
{
_emissionsForecasts = (await GetEmissionsJsonFile())?.EmissionsForecasts;
}
return _emissionsForecasts;
}

private async Task<EmissionsJsonFile?> GetEmissionsJsonFile()
{
_logger.LogInformation($"Reading Json data from {_configuration.DataFileLocation}");
using Stream stream = File.OpenRead(_configuration.DataFileLocation!);
return await JsonSerializer.DeserializeAsync<EmissionsJsonFile>(stream);
}
}
Loading