Skip to content

Commit e047c9a

Browse files
vaughanknightvaughanknight
committed
Updates for historical data API
Updates for historical data API
1 parent 7c115fa commit e047c9a

17 files changed

+198
-126
lines changed

‎casdk-docs/docs/architecture/decisions/0016-watt-time-v3.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ The following is configured at `CarbonAware.DataSources.WattTime/src/Constants/
3838
| Forecast | Get forecast| /forecast | /forecast | **TODO: CHECK IMPACT** <br> No longer be used for historical data <br> _Request_ <li> `ba` is now `region` <li> `extended_forecast` removed <li> `horizon_hours` added <li> `signal_type` added <li> Historical forecasts are now at `/forecast/historical` <br> _Response_ <li> `signal_type` added
3939
| Historical | Get historical forecast data | /historical (?) | /forecast/historical (?) | **We need to validate why historical was being used for the API, and what historical used to be, and whether this should be the new /forecast/historical or not.**
4040
| Balancing Authority From Location | Get balancing authority from location | /ba-from-loc | /region-from-loc | Check if the CA SDK uses BA at all <br><br> _Request_ <li> `name` is now `region_full_name` <li> `abbrev` is now `region` <li> `signal_type` added <br> _Response_ <li> `id` removed <li> `signal_type` added |
41-
| Login | User login | https://api2.watttime.org/v2/login | https://api.watttime.org/login | Path has changed from being version specific to being no longer related to the API version. <br><br> **TODO: CHECK HOW BASE URL IS DEFINED AS THIS WILL NOW HAVE DIFFERENT VALUES**
41+
| Login | User login | https://api2.watttime.org/v2/login | https://api.watttime.org/login | Path has changed from being version specific to being no longer related to the API version. <br><br> NOTE: Updated in wattTime client to now have 2 HTTP clients to decouple versions from the login.
4242

4343
### Query Strings
4444

‎src/CarbonAware.DataSources/CarbonAware.DataSources.WattTime/mock/WattTimeDataSourceMocker.cs

+10-10
Original file line numberDiff line numberDiff line change
@@ -13,11 +13,11 @@ internal class WattTimeDataSourceMocker : IDataSourceMocker
1313
{
1414
protected WireMockServer _server;
1515

16-
private static readonly BalancingAuthority defaultBalancingAuthority = new()
16+
private static readonly RegionResponse defaultBalancingAuthority = new()
1717
{
1818
Id = 12345,
19-
Abbreviation = "TEST_BA",
20-
Name = "Test Balancing Authority"
19+
Region = "TEST_BA",
20+
RegionFullName = "Test Balancing Authority"
2121
};
2222

2323
private static readonly LoginResult defaultLoginResult = new() { Token = "myDefaultToken123" };
@@ -39,11 +39,11 @@ public void SetupDataMock(DateTimeOffset start, DateTimeOffset end, string locat
3939
{
4040
var newDataPoint = new GridEmissionDataPoint()
4141
{
42-
BalancingAuthorityAbbreviation = defaultBalancingAuthority.Abbreviation,
42+
Region = defaultBalancingAuthority.Region,
4343
PointTime = pointTime,
4444
Value = 999.99F,
4545
Version = "1.0",
46-
Datatype = "dt",
46+
SignalType = "dt",
4747
Frequency = 300,
4848
Market = "mkt",
4949
};
@@ -70,8 +70,8 @@ public void SetupForecastMock()
7070
{
7171
var newForecastPoint = new GridEmissionDataPoint()
7272
{
73-
BalancingAuthorityAbbreviation = defaultBalancingAuthority.Abbreviation,
74-
Datatype = "dt",
73+
Region = defaultBalancingAuthority.Region,
74+
SignalType = "dt",
7575
Frequency = 300,
7676
Market = "mkt",
7777
PointTime = start,
@@ -104,8 +104,8 @@ public void SetupBatchForecastMock()
104104
{
105105
var newForecastPoint = new GridEmissionDataPoint()
106106
{
107-
BalancingAuthorityAbbreviation = defaultBalancingAuthority.Abbreviation,
108-
Datatype = "dt",
107+
Region = defaultBalancingAuthority.Region,
108+
SignalType = "dt",
109109
Frequency = 300,
110110
Market = "mkt",
111111
PointTime = start,
@@ -156,7 +156,7 @@ private void SetupResponseGivenGetRequest(string path, string body)
156156
.WithBody(body)
157157
);
158158
}
159-
private void SetupBaMock(BalancingAuthority? content = null) =>
159+
private void SetupBaMock(RegionResponse? content = null) =>
160160
SetupResponseGivenGetRequest(Paths.BalancingAuthorityFromLocation, JsonSerializer.Serialize(content ?? defaultBalancingAuthority));
161161

162162
private void SetupLoginMock(LoginResult? content = null) =>

‎src/CarbonAware.DataSources/CarbonAware.DataSources.WattTime/src/Client/IWattTimeClient.cs

+7-7
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ namespace CarbonAware.DataSources.WattTime.Client;
88
internal interface IWattTimeClient
99
{
1010
public const string NamedClient = "WattTimeClient";
11-
public const string NamedAuthenticationClient = "WattTimeAuthenticationClient"
11+
public const string NamedAuthenticationClient = "WattTimeAuthenticationClient";
1212

1313
/// <summary>
1414
/// Async method to get observed emission data for a given balancing authority and time period.
@@ -18,7 +18,7 @@ internal interface IWattTimeClient
1818
/// <param name="endTime">End time of the time period</param>
1919
/// <returns>An <see cref="Task{IEnumerable}{GridEmissionDataPoint}"/> which contains all emissions data points in a period.</returns>
2020
/// <exception cref="WattTimeClientException">Can be thrown when errors occur connecting to WattTime client. See the WattTimeClientException class for documentation of expected status codes.</exception>
21-
Task<IEnumerable<GridEmissionDataPoint>> GetDataAsync(string balancingAuthorityAbbreviation, DateTimeOffset startTime, DateTimeOffset endTime);
21+
Task<GridEmissionsDataResponse> GetDataAsync(string balancingAuthorityAbbreviation, DateTimeOffset startTime, DateTimeOffset endTime);
2222

2323
/// <summary>
2424
/// Async method to get observed emission data for a given balancing authority and time period.
@@ -28,7 +28,7 @@ internal interface IWattTimeClient
2828
/// <param name="endTime">End time of the time period</param>
2929
/// <returns>An <see cref="Task{IEnumerable}{GridEmissionDataPoint}"/> which contains all emissions data points in a period.</returns>
3030
/// <exception cref="WattTimeClientException">Can be thrown when errors occur connecting to WattTime client. See the WattTimeClientException class for documentation of expected status codes.</exception>
31-
Task<IEnumerable<GridEmissionDataPoint>> GetDataAsync(BalancingAuthority balancingAuthority, DateTimeOffset startTime, DateTimeOffset endTime);
31+
Task<GridEmissionsDataResponse> GetDataAsync(RegionResponse balancingAuthority, DateTimeOffset startTime, DateTimeOffset endTime);
3232

3333
/// <summary>
3434
/// Async method to get the most recent 24 hour forecasted emission data for a given balancing authority.
@@ -44,7 +44,7 @@ internal interface IWattTimeClient
4444
/// <param name="balancingAuthority">Balancing authority</param>
4545
/// <returns>An <see cref="Task{Forecast}"/> which contains forecasted emissions data points.</returns>
4646
/// <exception cref="WattTimeClientException">Can be thrown when errors occur connecting to WattTime client. See the WattTimeClientException class for documentation of expected status codes.</exception>
47-
Task<Forecast> GetCurrentForecastAsync(BalancingAuthority balancingAuthority);
47+
Task<Forecast> GetCurrentForecastAsync(RegionResponse balancingAuthority);
4848

4949
/// <summary>
5050
/// Async method to get generated forecast at requested time and balancing authority.
@@ -62,7 +62,7 @@ internal interface IWattTimeClient
6262
/// <param name="requestedAt">The historical time used to fetch the most recent forecast generated as of that time.</param>
6363
/// <returns>An <see cref="Task{Forecast}"/> which contains forecasted emissions data points or null if no Forecast generated at the requested time.</returns>
6464
/// <exception cref="WattTimeClientException">Can be thrown when errors occur connecting to WattTime client. See the WattTimeClientException class for documentation of expected status codes.</exception>
65-
Task<Forecast?> GetForecastOnDateAsync(BalancingAuthority balancingAuthority, DateTimeOffset requestedAt);
65+
Task<Forecast?> GetForecastOnDateAsync(RegionResponse balancingAuthority, DateTimeOffset requestedAt);
6666

6767
/// <summary>
6868
/// Async method to get the balancing authority for a given location.
@@ -71,7 +71,7 @@ internal interface IWattTimeClient
7171
/// <param name="longitude">Longitude of the location</param>
7272
/// <returns>An <see cref="Task{BalancingAuthority}"/> which contains the balancing authority details.</returns>
7373
/// <exception cref="WattTimeClientException">Can be thrown when errors occur connecting to WattTime client. See the WattTimeClientException class for documentation of expected status codes.</exception>
74-
Task<BalancingAuthority> GetBalancingAuthorityAsync(string latitude, string longitude);
74+
Task<RegionResponse> GetBalancingAuthorityAsync(string latitude, string longitude);
7575

7676
/// <summary>
7777
/// Async method to get the balancing authority abbreviation for a given location.
@@ -96,5 +96,5 @@ internal interface IWattTimeClient
9696
/// <param name="balancingAuthority">Balancing authority</param>
9797
/// <returns>An <see cref="Task{Stream}"/> which contains the data Stream of the .zip file.</returns>
9898
/// <exception cref="WattTimeClientException">Can be thrown when errors occur connecting to WattTime client. See the WattTimeClientException class for documentation of expected status codes.</exception>
99-
Task<Stream> GetHistoricalDataAsync(BalancingAuthority balancingAuthority);
99+
Task<Stream> GetHistoricalDataAsync(RegionResponse balancingAuthority);
100100
}

‎src/CarbonAware.DataSources/CarbonAware.DataSources.WattTime/src/Client/WattTimeClient.cs

+31-24
Original file line numberDiff line numberDiff line change
@@ -55,61 +55,66 @@ public WattTimeClient(IHttpClientFactory factory, IOptionsMonitor<WattTimeClient
5555
}
5656

5757
/// <inheritdoc/>
58-
public async Task<IEnumerable<GridEmissionDataPoint>> GetDataAsync(string balancingAuthorityAbbreviation, DateTimeOffset startTime, DateTimeOffset endTime)
58+
public async Task<GridEmissionsDataResponse> GetDataAsync(string regionAbbreviation, DateTimeOffset startTime, DateTimeOffset endTime)
5959
{
6060
_log.LogInformation("Requesting grid emission data using start time {startTime} and endTime {endTime}", startTime, endTime);
6161

6262
var parameters = new Dictionary<string, string>()
6363
{
64-
{ QueryStrings.Region, balancingAuthorityAbbreviation },
64+
{ QueryStrings.Region, regionAbbreviation },
6565
{ QueryStrings.StartTime, startTime.ToUniversalTime().ToString("O", CultureInfo.InvariantCulture) },
66-
{ QueryStrings.EndTime, endTime.ToUniversalTime().ToString("O", CultureInfo.InvariantCulture) }
66+
{ QueryStrings.EndTime, endTime.ToUniversalTime().ToString("O", CultureInfo.InvariantCulture) },
67+
{ QueryStrings.SignalType, SignalTypes.co2_moer},
6768
};
6869

6970
var tags = new Dictionary<string, string>()
7071
{
71-
{ QueryStrings.Region, balancingAuthorityAbbreviation }
72+
{ QueryStrings.Region, regionAbbreviation }
7273
};
7374

7475
using (var result = await this.MakeRequestGetStreamAsync(Paths.Data, parameters, tags))
7576
{
76-
return await JsonSerializer.DeserializeAsync<List<GridEmissionDataPoint>>(result, _options) ?? throw new WattTimeClientException($"Error getting forecasts for {balancingAuthorityAbbreviation}");
77+
return await JsonSerializer.DeserializeAsync<GridEmissionsDataResponse>(result, _options) ?? throw new WattTimeClientException($"Error getting forecasts for {regionAbbreviation}");
7778
}
7879
}
7980

8081
/// <inheritdoc/>
81-
public Task<IEnumerable<GridEmissionDataPoint>> GetDataAsync(BalancingAuthority balancingAuthority, DateTimeOffset startTime, DateTimeOffset endTime)
82+
public Task<GridEmissionsDataResponse> GetDataAsync(RegionResponse region, DateTimeOffset startTime, DateTimeOffset endTime)
8283
{
83-
return this.GetDataAsync(balancingAuthority.Abbreviation, startTime, endTime);
84+
return this.GetDataAsync(region.Region, startTime, endTime);
8485
}
8586

8687
/// <inheritdoc/>
87-
public async Task<Forecast> GetCurrentForecastAsync(string balancingAuthorityAbbreviation)
88+
public async Task<Forecast> GetCurrentForecastAsync(string region)
8889
{
8990

90-
_log.LogInformation("Requesting current forecast from balancing authority {balancingAuthority}", balancingAuthorityAbbreviation);
91+
_log.LogInformation("Requesting current forecast from balancing authority {balancingAuthority}", region);
9192

9293
var parameters = new Dictionary<string, string>()
9394
{
94-
{ QueryStrings.Region, balancingAuthorityAbbreviation }
95+
{ QueryStrings.Region, region },
96+
{ QueryStrings.SignalType, SignalTypes.co2_moer }
9597
};
9698

9799
var tags = new Dictionary<string, string>()
98100
{
99-
{ QueryStrings.Region, balancingAuthorityAbbreviation }
101+
{ QueryStrings.Region, region }
100102
};
101103

102104
var result = await this.MakeRequestGetStreamAsync(Paths.Forecast, parameters, tags);
103105

104-
var forecast = await JsonSerializer.DeserializeAsync<Forecast?>(result, _options) ?? throw new WattTimeClientException($"Error getting forecast for {balancingAuthorityAbbreviation}");
106+
var sr = new StreamReader(result);
107+
var s = sr.ReadToEnd();
108+
109+
var forecast = await JsonSerializer.DeserializeAsync<Forecast?>(result, _options) ?? throw new WattTimeClientException($"Error getting forecast for {region}");
105110

106111
return forecast;
107112
}
108113

109114
/// <inheritdoc/>
110-
public Task<Forecast> GetCurrentForecastAsync(BalancingAuthority balancingAuthority)
115+
public Task<Forecast> GetCurrentForecastAsync(RegionResponse balancingAuthority)
111116
{
112-
return this.GetCurrentForecastAsync(balancingAuthority.Abbreviation);
117+
return this.GetCurrentForecastAsync(balancingAuthority.Region);
113118
}
114119

115120
/// <inheritdoc/>
@@ -136,13 +141,13 @@ public Task<Forecast> GetCurrentForecastAsync(BalancingAuthority balancingAuthor
136141
}
137142

138143
/// <inheritdoc/>
139-
public Task<Forecast?> GetForecastOnDateAsync(BalancingAuthority balancingAuthority, DateTimeOffset requestedAt)
144+
public Task<Forecast?> GetForecastOnDateAsync(RegionResponse balancingAuthority, DateTimeOffset requestedAt)
140145
{
141-
return this.GetForecastOnDateAsync(balancingAuthority.Abbreviation, requestedAt);
146+
return this.GetForecastOnDateAsync(balancingAuthority.Region, requestedAt);
142147
}
143148

144149
/// <inheritdoc/>
145-
public async Task<BalancingAuthority> GetBalancingAuthorityAsync(string latitude, string longitude)
150+
public async Task<RegionResponse> GetBalancingAuthorityAsync(string latitude, string longitude)
146151
{
147152
_log.LogInformation("Requesting balancing authority for lattitude {lattitude} and longitude {longitude}", latitude, longitude);
148153
return await GetBalancingAuthorityFromCacheAsync(latitude, longitude);
@@ -151,7 +156,7 @@ public async Task<BalancingAuthority> GetBalancingAuthorityAsync(string latitude
151156
/// <inheritdoc/>
152157
public async Task<string?> GetBalancingAuthorityAbbreviationAsync(string latitude, string longitude)
153158
{
154-
return (await this.GetBalancingAuthorityAsync(latitude, longitude))?.Abbreviation;
159+
return (await this.GetBalancingAuthorityAsync(latitude, longitude))?.Region;
155160
}
156161

157162
/// <inheritdoc/>
@@ -176,9 +181,9 @@ public async Task<Stream> GetHistoricalDataAsync(string balancingAuthorityAbbrev
176181
}
177182

178183
/// <inheritdoc/>
179-
public Task<Stream> GetHistoricalDataAsync(BalancingAuthority balancingAuthority)
184+
public Task<Stream> GetHistoricalDataAsync(RegionResponse balancingAuthority)
180185
{
181-
return this.GetHistoricalDataAsync(balancingAuthority.Abbreviation);
186+
return this.GetHistoricalDataAsync(balancingAuthority.Region);
182187
}
183188

184189
private async Task<HttpResponseMessage> GetAsyncWithAuthRetry(string uriPath)
@@ -290,24 +295,26 @@ private string BuildUrlWithQueryString(string url, IDictionary<string, string> q
290295
return result;
291296
}
292297

293-
private async Task<BalancingAuthority> GetBalancingAuthorityFromCacheAsync(string latitude, string longitude)
298+
private async Task<RegionResponse> GetBalancingAuthorityFromCacheAsync(string latitude, string longitude)
294299
{
295300
var key = new Tuple<string, string>(latitude, longitude);
296301
var balancingAuthority = await this._memoryCache.GetOrCreateAsync(key, async entry =>
297302
{
298303
var parameters = new Dictionary<string, string>()
299304
{
300305
{ QueryStrings.Latitude, latitude },
301-
{ QueryStrings.Longitude, longitude }
306+
{ QueryStrings.Longitude, longitude },
307+
{ QueryStrings.SignalType, SignalTypes.co2_moer}
302308
};
303309

304310
var tags = new Dictionary<string, string>()
305311
{
306312
{ QueryStrings.Latitude, latitude },
307-
{ QueryStrings.Longitude, longitude }
313+
{ QueryStrings.Longitude, longitude },
314+
{ QueryStrings.SignalType, SignalTypes.co2_moer }
308315
};
309316
var result = await this.MakeRequestGetStreamAsync(Paths.BalancingAuthorityFromLocation, parameters, tags);
310-
var baValue = await JsonSerializer.DeserializeAsync<BalancingAuthority>(result, _options) ?? throw new WattTimeClientException($"Error getting Balancing Authority for latitude {latitude} and longitude {longitude}");
317+
var baValue = await JsonSerializer.DeserializeAsync<RegionResponse>(result, _options) ?? throw new WattTimeClientException($"Error getting Balancing Authority for latitude {latitude} and longitude {longitude}");
311318
entry.AbsoluteExpirationRelativeToNow = TimeSpan.FromSeconds(_configuration.BalancingAuthorityCacheTTL);
312319
entry.Value = baValue;
313320
return baValue;

‎src/CarbonAware.DataSources/CarbonAware.DataSources.WattTime/src/Configuration/WattTimeClientConfiguration.cs

+1-1
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ internal class WattTimeClientConfiguration
2323
/// <summary>
2424
/// Gets or sets the base url to use when connecting to WattTime
2525
/// </summary>
26-
public string BaseUrl { get; set; } = "https://api2.watttime.org/v2/";
26+
public string BaseUrl { get; set; } = "https://api.watttime.org/v3/";
2727

2828
/// <summary>
2929
/// Authentication base url. This changed between v2 and v3

‎src/CarbonAware.DataSources/CarbonAware.DataSources.WattTime/src/Constants/Paths.cs

+2-2
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,9 @@
22

33
internal class Paths
44
{
5-
public const string Data = "data";
5+
public const string Data = "historical";
66
public const string Forecast = "forecast";
7-
public const string BalancingAuthorityFromLocation = "ba-from-loc";
7+
public const string BalancingAuthorityFromLocation = "region-from-loc";
88
public const string Login = "login";
99
public const string Historical = "historical";
1010
}

‎src/CarbonAware.DataSources/CarbonAware.DataSources.WattTime/src/Constants/QueryStrings.cs

+1
Original file line numberDiff line numberDiff line change
@@ -8,4 +8,5 @@ internal class QueryStrings
88
public const string Latitude = "latitude";
99
public const string Longitude = "longitude";
1010
public const string Username = "username";
11+
public const string SignalType = "signal_type";
1112
}

‎src/CarbonAware.DataSources/CarbonAware.DataSources.WattTime/src/Constants/SignalTypes.cs

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
namespace CarbonAware.DataSources.WattTime.Constants;
2+
3+
internal class SignalTypes
4+
{
5+
public const string co2_moer = "co2_moer";
6+
}

‎src/CarbonAware.DataSources/CarbonAware.DataSources.WattTime/src/Model/GridEmissionDataPoint.cs

-12
Original file line numberDiff line numberDiff line change
@@ -8,18 +8,6 @@ namespace CarbonAware.DataSources.WattTime.Model;
88
[Serializable]
99
internal record GridEmissionDataPoint
1010
{
11-
/// <summary>
12-
/// Balancing authority abbreviation
13-
/// </summary>
14-
[JsonPropertyName("ba")]
15-
public string BalancingAuthorityAbbreviation { get; set; } = string.Empty;
16-
17-
/// <summary>
18-
/// Type of data. eg MOER
19-
/// </summary>
20-
[JsonPropertyName("datatype")]
21-
public string? Datatype { get; set; }
22-
2311
/// <summary>
2412
/// Duration in seconds for which the data is valid from point_time.
2513
/// </summary>

‎src/CarbonAware.DataSources/CarbonAware.DataSources.WattTime/src/Model/GridEmissionsDataResponse.cs

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
using System.Text.Json.Serialization;
2+
3+
namespace CarbonAware.DataSources.WattTime.Model;
4+
5+
[Serializable]
6+
internal record GridEmissionsDataResponse
7+
{
8+
[JsonPropertyName("data")]
9+
public List<GridEmissionDataPoint> Data { get; set; } = new List<GridEmissionDataPoint>();
10+
11+
12+
[JsonPropertyName("meta")]
13+
public GridEmissionsMetaData Meta { get; set; }
14+
}

0 commit comments

Comments
 (0)