Skip to content

Commit 924bf3b

Browse files
westey-mglorious-beard
authored andcommitted
.Net: Qdrant CRUD datetime support (microsoft#11285)
### Motivation and Context microsoft#11086 ### Description Adding datetime & datetimeoffset support for CRUD operations with Qdrant. Also supporting filtering with legacy filter options, to test end to end scenario. LINQ based filtering to follow in separate PR. ### Contribution Checklist <!-- Before submitting this PR, please make sure: --> - [ ] The code builds clean without any errors or warnings - [ ] The PR follows the [SK Contribution Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md) and the [pre-submission formatting script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts) raises no violations - [ ] All unit tests pass, and I have added new tests where possible - [ ] I didn't break anyone 😄
1 parent 9a84377 commit 924bf3b

File tree

6 files changed

+98
-16
lines changed

6 files changed

+98
-16
lines changed

dotnet/src/Connectors/Connectors.Memory.Qdrant/QdrantVectorStoreCollectionCreateMapping.cs

+2
Original file line numberDiff line numberDiff line change
@@ -41,9 +41,11 @@ internal static class QdrantVectorStoreCollectionCreateMapping
4141

4242
{ typeof(string), PayloadSchemaType.Keyword },
4343
{ typeof(DateTime), PayloadSchemaType.Datetime },
44+
{ typeof(DateTimeOffset), PayloadSchemaType.Datetime },
4445
{ typeof(bool), PayloadSchemaType.Bool },
4546

4647
{ typeof(DateTime?), PayloadSchemaType.Datetime },
48+
{ typeof(DateTimeOffset?), PayloadSchemaType.Datetime },
4749
{ typeof(bool?), PayloadSchemaType.Bool },
4850
};
4951

dotnet/src/Connectors/Connectors.Memory.Qdrant/QdrantVectorStoreCollectionSearchMapping.cs

+23-6
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,29 @@ public static Filter BuildFromLegacyFilter(VectorSearchFilter basicVectorSearchF
4646
throw new InvalidOperationException($"Unsupported filter clause type '{filterClause.GetType().Name}'.");
4747
}
4848

49+
// Get the storage name for the field.
50+
if (!storagePropertyNames.TryGetValue(fieldName, out var storagePropertyName))
51+
{
52+
throw new InvalidOperationException($"Property name '{fieldName}' provided as part of the filter clause is not a valid property name.");
53+
}
54+
55+
// Map datetime equality.
56+
if (filterValue is DateTime or DateTimeOffset)
57+
{
58+
var dateTimeOffset = filterValue is DateTime dateTime
59+
? new DateTimeOffset(dateTime, TimeSpan.Zero)
60+
: (DateTimeOffset)filterValue;
61+
62+
var range = new global::Qdrant.Client.Grpc.DatetimeRange
63+
{
64+
Gte = Google.Protobuf.WellKnownTypes.Timestamp.FromDateTimeOffset(dateTimeOffset),
65+
Lte = Google.Protobuf.WellKnownTypes.Timestamp.FromDateTimeOffset(dateTimeOffset),
66+
};
67+
68+
filter.Must.Add(new Condition() { Field = new FieldCondition() { Key = storagePropertyName, DatetimeRange = range } });
69+
continue;
70+
}
71+
4972
// Map each type of filter value to the appropriate Qdrant match type.
5073
var match = filterValue switch
5174
{
@@ -56,12 +79,6 @@ public static Filter BuildFromLegacyFilter(VectorSearchFilter basicVectorSearchF
5679
_ => throw new InvalidOperationException($"Unsupported filter value type '{filterValue.GetType().Name}'.")
5780
};
5881

59-
// Get the storage name for the field.
60-
if (!storagePropertyNames.TryGetValue(fieldName, out var storagePropertyName))
61-
{
62-
throw new InvalidOperationException($"Property name '{fieldName}' provided as part of the filter clause is not a valid property name.");
63-
}
64-
6582
filter.Must.Add(new Condition() { Field = new FieldCondition() { Key = storagePropertyName, Match = match } });
6683
}
6784

dotnet/src/Connectors/Connectors.Memory.Qdrant/QdrantVectorStoreRecordFieldMapping.cs

+28-3
Original file line numberDiff line numberDiff line change
@@ -23,11 +23,15 @@ internal static class QdrantVectorStoreRecordFieldMapping
2323
typeof(double),
2424
typeof(float),
2525
typeof(bool),
26+
typeof(DateTime),
27+
typeof(DateTimeOffset),
2628
typeof(int?),
2729
typeof(long?),
2830
typeof(double?),
2931
typeof(float?),
30-
typeof(bool?)
32+
typeof(bool?),
33+
typeof(DateTime?),
34+
typeof(DateTimeOffset?),
3135
];
3236

3337
/// <summary>A set of types that vectors on the provided model may have.</summary>
@@ -57,7 +61,8 @@ internal static class QdrantVectorStoreRecordFieldMapping
5761
targetType == typeof(int) || targetType == typeof(int?) ?
5862
(object)(int)payloadValue.IntegerValue :
5963
(object)payloadValue.IntegerValue,
60-
Value.KindOneofCase.StringValue => payloadValue.StringValue,
64+
Value.KindOneofCase.StringValue =>
65+
ConvertStringValue(payloadValue.StringValue),
6166
Value.KindOneofCase.DoubleValue =>
6267
targetType == typeof(float) || targetType == typeof(float?) ?
6368
(object)(float)payloadValue.DoubleValue :
@@ -69,6 +74,16 @@ internal static class QdrantVectorStoreRecordFieldMapping
6974
targetType),
7075
_ => throw new VectorStoreRecordMappingException($"Unsupported grpc value kind {payloadValue.KindCase}."),
7176
};
77+
78+
object ConvertStringValue(string stringValue)
79+
{
80+
return targetType switch
81+
{
82+
Type t when t == typeof(DateTime) || t == typeof(DateTime?) => DateTime.Parse(payloadValue.StringValue),
83+
Type t when t == typeof(DateTimeOffset) || t == typeof(DateTimeOffset?) => DateTimeOffset.Parse(payloadValue.StringValue),
84+
_ => stringValue,
85+
};
86+
}
7287
}
7388

7489
/// <summary>
@@ -108,12 +123,22 @@ public static Value ConvertToGrpcFieldValue(object? sourceValue)
108123
{
109124
value.BoolValue = boolValue;
110125
}
126+
else if (sourceValue is DateTime datetimeValue)
127+
{
128+
value.StringValue = datetimeValue.ToString("O");
129+
}
130+
else if (sourceValue is DateTimeOffset dateTimeOffsetValue)
131+
{
132+
value.StringValue = dateTimeOffsetValue.ToString("O");
133+
}
111134
else if (sourceValue is IEnumerable<int> ||
112135
sourceValue is IEnumerable<long> ||
113136
sourceValue is IEnumerable<string> ||
114137
sourceValue is IEnumerable<float> ||
115138
sourceValue is IEnumerable<double> ||
116-
sourceValue is IEnumerable<bool>)
139+
sourceValue is IEnumerable<bool> ||
140+
sourceValue is IEnumerable<DateTime> ||
141+
sourceValue is IEnumerable<DateTimeOffset>)
117142
{
118143
var listValue = sourceValue as IEnumerable;
119144
value.ListValue = new ListValue();

dotnet/src/Connectors/Connectors.Qdrant.UnitTests/QdrantVectorStoreRecordMapperTests.cs

+22-2
Original file line numberDiff line numberDiff line change
@@ -139,13 +139,15 @@ public void MapsMultiPropsFromDataToStorageModelWithUlong()
139139
// Assert.
140140
Assert.NotNull(actual);
141141
Assert.Equal(5ul, actual.Id.Num);
142-
Assert.Equal(7, actual.Payload.Count);
142+
Assert.Equal(9, actual.Payload.Count);
143143
Assert.Equal("data 1", actual.Payload["dataString"].StringValue);
144144
Assert.Equal(5, actual.Payload["dataInt"].IntegerValue);
145145
Assert.Equal(5, actual.Payload["dataLong"].IntegerValue);
146146
Assert.Equal(5.5f, actual.Payload["dataFloat"].DoubleValue);
147147
Assert.Equal(5.5d, actual.Payload["dataDouble"].DoubleValue);
148148
Assert.True(actual.Payload["dataBool"].BoolValue);
149+
Assert.Equal("2025-02-10T05:10:15.0000000Z", actual.Payload["dataDateTime"].StringValue);
150+
Assert.Equal("2025-02-10T05:10:15.0000000+01:00", actual.Payload["dataDateTimeOffset"].StringValue);
149151
Assert.Equal(new int[] { 1, 2, 3, 4 }, actual.Payload["dataArrayInt"].ListValue.Values.Select(x => (int)x.IntegerValue).ToArray());
150152
Assert.Equal(new float[] { 1, 2, 3, 4 }, actual.Vectors.Vectors_.Vectors["vector1"].Data.ToArray());
151153
Assert.Equal(new float[] { 5, 6, 7, 8 }, actual.Vectors.Vectors_.Vectors["vector2"].Data.ToArray());
@@ -165,13 +167,15 @@ public void MapsMultiPropsFromDataToStorageModelWithGuid()
165167
// Assert.
166168
Assert.NotNull(actual);
167169
Assert.Equal(Guid.Parse("11111111-1111-1111-1111-111111111111"), Guid.Parse(actual.Id.Uuid));
168-
Assert.Equal(7, actual.Payload.Count);
170+
Assert.Equal(9, actual.Payload.Count);
169171
Assert.Equal("data 1", actual.Payload["dataString"].StringValue);
170172
Assert.Equal(5, actual.Payload["dataInt"].IntegerValue);
171173
Assert.Equal(5, actual.Payload["dataLong"].IntegerValue);
172174
Assert.Equal(5.5f, actual.Payload["dataFloat"].DoubleValue);
173175
Assert.Equal(5.5d, actual.Payload["dataDouble"].DoubleValue);
174176
Assert.True(actual.Payload["dataBool"].BoolValue);
177+
Assert.Equal("2025-02-10T05:10:15.0000000Z", actual.Payload["dataDateTime"].StringValue);
178+
Assert.Equal("2025-02-10T05:10:15.0000000+01:00", actual.Payload["dataDateTimeOffset"].StringValue);
175179
Assert.Equal(new int[] { 1, 2, 3, 4 }, actual.Payload["dataArrayInt"].ListValue.Values.Select(x => (int)x.IntegerValue).ToArray());
176180
Assert.Equal(new float[] { 1, 2, 3, 4 }, actual.Vectors.Vectors_.Vectors["vector1"].Data.ToArray());
177181
Assert.Equal(new float[] { 5, 6, 7, 8 }, actual.Vectors.Vectors_.Vectors["vector2"].Data.ToArray());
@@ -199,6 +203,8 @@ public void MapsMultiPropsFromStorageToDataModelWithUlong(bool includeVectors)
199203
Assert.Equal(5.5f, actual.DataFloat);
200204
Assert.Equal(5.5d, actual.DataDouble);
201205
Assert.True(actual.DataBool);
206+
Assert.Equal(new DateTime(2025, 2, 10, 5, 10, 15, DateTimeKind.Utc), actual.DataDateTime);
207+
Assert.Equal(new DateTimeOffset(2025, 2, 10, 5, 10, 15, TimeSpan.FromHours(1)), actual.DataDateTimeOffset);
202208
Assert.Equal(new int[] { 1, 2, 3, 4 }, actual.DataArrayInt);
203209

204210
if (includeVectors)
@@ -235,6 +241,8 @@ public void MapsMultiPropsFromStorageToDataModelWithGuid(bool includeVectors)
235241
Assert.Equal(5.5f, actual.DataFloat);
236242
Assert.Equal(5.5d, actual.DataDouble);
237243
Assert.True(actual.DataBool);
244+
Assert.Equal(new DateTime(2025, 2, 10, 5, 10, 15, DateTimeKind.Utc), actual.DataDateTime);
245+
Assert.Equal(new DateTimeOffset(2025, 2, 10, 5, 10, 15, TimeSpan.FromHours(1)), actual.DataDateTimeOffset);
238246
Assert.Equal(new int[] { 1, 2, 3, 4 }, actual.DataArrayInt);
239247

240248
if (includeVectors)
@@ -271,6 +279,8 @@ private static MultiPropsModel<TKey> CreateMultiPropsModel<TKey>(TKey key)
271279
DataFloat = 5.5f,
272280
DataDouble = 5.5d,
273281
DataBool = true,
282+
DataDateTime = new DateTime(2025, 2, 10, 5, 10, 15, DateTimeKind.Utc),
283+
DataDateTimeOffset = new DateTimeOffset(2025, 2, 10, 5, 10, 15, TimeSpan.FromHours(1)),
274284
DataArrayInt = new List<int> { 1, 2, 3, 4 },
275285
Vector1 = new float[] { 1, 2, 3, 4 },
276286
Vector2 = new float[] { 5, 6, 7, 8 },
@@ -334,6 +344,8 @@ private static void AddDataToMultiPropsPointStruct(PointStruct pointStruct)
334344
pointStruct.Payload.Add("dataFloat", 5.5f);
335345
pointStruct.Payload.Add("dataDouble", 5.5d);
336346
pointStruct.Payload.Add("dataBool", true);
347+
pointStruct.Payload.Add("dataDateTime", "2025-02-10T05:10:15.0000000Z");
348+
pointStruct.Payload.Add("dataDateTimeOffset", "2025-02-10T05:10:15.0000000+01:00");
337349

338350
var dataIntArray = new ListValue();
339351
dataIntArray.Values.Add(1);
@@ -383,6 +395,8 @@ private sealed class SinglePropsModel<TKey>
383395
new VectorStoreRecordDataProperty("DataFloat", typeof(float)) { StoragePropertyName = "dataFloat" },
384396
new VectorStoreRecordDataProperty("DataDouble", typeof(double)) { StoragePropertyName = "dataDouble" },
385397
new VectorStoreRecordDataProperty("DataBool", typeof(bool)) { StoragePropertyName = "dataBool" },
398+
new VectorStoreRecordDataProperty("DataDateTime", typeof(DateTime)) { StoragePropertyName = "dataDateTime" },
399+
new VectorStoreRecordDataProperty("DataDateTimeOffset", typeof(DateTimeOffset)) { StoragePropertyName = "dataDateTimeOffset" },
386400
new VectorStoreRecordDataProperty("DataArrayInt", typeof(List<int>)) { StoragePropertyName = "dataArrayInt" },
387401
new VectorStoreRecordVectorProperty("Vector1", typeof(ReadOnlyMemory<float>)) { StoragePropertyName = "vector1" },
388402
new VectorStoreRecordVectorProperty("Vector2", typeof(ReadOnlyMemory<float>)) { StoragePropertyName = "vector2" },
@@ -413,6 +427,12 @@ private sealed class MultiPropsModel<TKey>
413427
[VectorStoreRecordData(StoragePropertyName = "dataBool")]
414428
public bool DataBool { get; set; } = false;
415429

430+
[VectorStoreRecordData(StoragePropertyName = "dataDateTime")]
431+
public DateTime DataDateTime { get; set; }
432+
433+
[VectorStoreRecordData(StoragePropertyName = "dataDateTimeOffset")]
434+
public DateTimeOffset DataDateTimeOffset { get; set; }
435+
416436
[VectorStoreRecordData(StoragePropertyName = "dataArrayInt")]
417437
public List<int>? DataArrayInt { get; set; }
418438

dotnet/src/IntegrationTests/Connectors/Memory/Qdrant/QdrantVectorStoreFixture.cs

+14-4
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,8 @@ public QdrantVectorStoreFixture()
5757
new VectorStoreRecordDataProperty("HotelCode", typeof(int)) { IsFilterable = true },
5858
new VectorStoreRecordDataProperty("ParkingIncluded", typeof(bool)) { IsFilterable = true, StoragePropertyName = "parking_is_included" },
5959
new VectorStoreRecordDataProperty("HotelRating", typeof(float)) { IsFilterable = true },
60+
new VectorStoreRecordDataProperty("LastRenovationDate", typeof(DateTime)) { IsFilterable = true },
61+
new VectorStoreRecordDataProperty("OpeningDate", typeof(DateTimeOffset)) { IsFilterable = true },
6062
new VectorStoreRecordDataProperty("Tags", typeof(List<string>)) { IsFilterable = true },
6163
new VectorStoreRecordDataProperty("Description", typeof(string)),
6264
new VectorStoreRecordVectorProperty("DescriptionEmbedding", typeof(ReadOnlyMemory<float>?)) { Dimensions = VectorDimensions, DistanceFunction = DistanceFunction.ManhattanDistance }
@@ -177,7 +179,7 @@ await this.QdrantClient.CreateCollectionAsync(
177179
{
178180
Id = 11,
179181
Vectors = new Vectors { Vectors_ = namedVectors1 },
180-
Payload = { ["HotelName"] = "My Hotel 11", ["HotelCode"] = 11, ["parking_is_included"] = true, ["Tags"] = tagsValue, ["HotelRating"] = 4.5f, ["Description"] = "This is a great hotel." }
182+
Payload = { ["HotelName"] = "My Hotel 11", ["HotelCode"] = 11, ["parking_is_included"] = true, ["Tags"] = tagsValue, ["HotelRating"] = 4.5f, ["Description"] = "This is a great hotel.", ["LastRenovationDate"] = "2025-02-10T05:10:15.0000000Z", ["OpeningDate"] = "2025-02-10T05:10:15.0000000+01:00" }
181183
},
182184
new PointStruct
183185
{
@@ -189,7 +191,7 @@ await this.QdrantClient.CreateCollectionAsync(
189191
{
190192
Id = 13,
191193
Vectors = new Vectors { Vectors_ = namedVectors3 },
192-
Payload = { ["HotelName"] = "My Hotel 13", ["HotelCode"] = 13, ["parking_is_included"] = false, ["Tags"] = tagsValue2, ["Description"] = "This is a great hotel." }
194+
Payload = { ["HotelName"] = "My Hotel 13", ["HotelCode"] = 13, ["parking_is_included"] = false, ["Tags"] = tagsValue2, ["Description"] = "This is a great hotel.", ["LastRenovationDate"] = "2020-02-01T00:00:00.0000000Z" }
193195
},
194196
new PointStruct
195197
{
@@ -208,7 +210,7 @@ await this.QdrantClient.CreateCollectionAsync(
208210
{
209211
Id = 11,
210212
Vectors = embeddingArray,
211-
Payload = { ["HotelName"] = "My Hotel 11", ["HotelCode"] = 11, ["parking_is_included"] = true, ["Tags"] = tagsValue, ["HotelRating"] = 4.5f, ["Description"] = "This is a great hotel." }
213+
Payload = { ["HotelName"] = "My Hotel 11", ["HotelCode"] = 11, ["parking_is_included"] = true, ["Tags"] = tagsValue, ["HotelRating"] = 4.5f, ["Description"] = "This is a great hotel.", ["LastRenovationDate"] = "2025-02-10T05:10:15.0000000Z", ["OpeningDate"] = "2025-02-10T05:10:15.0000000+01:00" }
212214
},
213215
new PointStruct
214216
{
@@ -220,7 +222,7 @@ await this.QdrantClient.CreateCollectionAsync(
220222
{
221223
Id = 13,
222224
Vectors = embeddingArray,
223-
Payload = { ["HotelName"] = "My Hotel 13", ["HotelCode"] = 13, ["parking_is_included"] = false, ["Tags"] = tagsValue2, ["Description"] = "This is a great hotel." }
225+
Payload = { ["HotelName"] = "My Hotel 13", ["HotelCode"] = 13, ["parking_is_included"] = false, ["Tags"] = tagsValue2, ["Description"] = "This is a great hotel.", ["LastRenovationDate"] = "2020-02-01T00:00:00.0000000Z" }
224226
},
225227
];
226228

@@ -336,6 +338,14 @@ public record HotelInfo()
336338
[VectorStoreRecordData(IsFilterable = true)]
337339
public List<string> Tags { get; set; } = new List<string>();
338340

341+
/// <summary>A datetime metadata field.</summary>
342+
[VectorStoreRecordData(IsFilterable = true)]
343+
public DateTime? LastRenovationDate { get; set; }
344+
345+
/// <summary>A datetimeoffset metadata field.</summary>
346+
[VectorStoreRecordData(IsFilterable = true)]
347+
public DateTimeOffset? OpeningDate { get; set; }
348+
339349
/// <summary>A data field.</summary>
340350
[VectorStoreRecordData]
341351
public string Description { get; set; }

dotnet/src/IntegrationTests/Connectors/Memory/Qdrant/QdrantVectorStoreRecordCollectionTests.cs

+9-1
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,8 @@ public async Task ItCanCreateACollectionUpsertGetAndSearchAsync(bool hasNamedVec
8181
Assert.Equal(record.HotelCode, getResult?.HotelCode);
8282
Assert.Equal(record.HotelRating, getResult?.HotelRating);
8383
Assert.Equal(record.ParkingIncluded, getResult?.ParkingIncluded);
84+
Assert.Equal(record.LastRenovationDate, getResult?.LastRenovationDate);
85+
Assert.Equal(record.OpeningDate, getResult?.OpeningDate);
8486
Assert.Equal(record.Tags.ToArray(), getResult?.Tags.ToArray());
8587
Assert.Equal(record.Description, getResult?.Description);
8688

@@ -92,6 +94,8 @@ public async Task ItCanCreateACollectionUpsertGetAndSearchAsync(bool hasNamedVec
9294
Assert.Equal(record.HotelCode, searchResultRecord?.HotelCode);
9395
Assert.Equal(record.HotelRating, searchResultRecord?.HotelRating);
9496
Assert.Equal(record.ParkingIncluded, searchResultRecord?.ParkingIncluded);
97+
Assert.Equal(record.LastRenovationDate, searchResultRecord?.LastRenovationDate);
98+
Assert.Equal(record.OpeningDate, searchResultRecord?.OpeningDate);
9599
Assert.Equal(record.Tags.ToArray(), searchResultRecord?.Tags.ToArray());
96100
Assert.Equal(record.Description, searchResultRecord?.Description);
97101

@@ -222,6 +226,8 @@ public async Task ItCanGetDocumentFromVectorStoreAsync(bool useRecordDefinition,
222226
Assert.Equal(11, getResult?.HotelCode);
223227
Assert.True(getResult?.ParkingIncluded);
224228
Assert.Equal(4.5f, getResult?.HotelRating);
229+
Assert.Equal(new DateTime(2025, 2, 10, 5, 10, 15, DateTimeKind.Utc), getResult?.LastRenovationDate);
230+
Assert.Equal(new DateTimeOffset(2025, 2, 10, 5, 10, 15, TimeSpan.FromHours(1)), getResult?.OpeningDate);
225231
Assert.Equal(2, getResult?.Tags.Count);
226232
Assert.Equal("t11.1", getResult?.Tags[0]);
227233
Assert.Equal("t11.2", getResult?.Tags[1]);
@@ -391,7 +397,7 @@ public async Task ItCanSearchWithFilterAsync(bool useRecordDefinition, string co
391397

392398
// Act.
393399
var vector = await fixture.EmbeddingGenerator.GenerateEmbeddingAsync("A great hotel");
394-
var filter = filterType == "equality" ? new VectorSearchFilter().EqualTo("HotelName", "My Hotel 13") : new VectorSearchFilter().AnyTagEqualTo("Tags", "t13.2");
400+
var filter = filterType == "equality" ? new VectorSearchFilter().EqualTo("HotelName", "My Hotel 13").EqualTo("LastRenovationDate", new DateTimeOffset(2020, 02, 01, 0, 0, 0, TimeSpan.Zero)) : new VectorSearchFilter().AnyTagEqualTo("Tags", "t13.2");
395401
var actual = await sut.VectorizedSearchAsync(
396402
vector,
397403
new()
@@ -477,6 +483,8 @@ private async Task<HotelInfo> CreateTestHotelAsync(uint hotelId, ITextEmbeddingG
477483
HotelCode = (int)hotelId,
478484
HotelRating = 4.5f,
479485
ParkingIncluded = true,
486+
LastRenovationDate = new DateTime(2025, 2, 10, 5, 10, 15, DateTimeKind.Utc),
487+
OpeningDate = new DateTimeOffset(2025, 2, 10, 5, 10, 15, TimeSpan.FromHours(1)),
480488
Tags = { "t1", "t2" },
481489
Description = "This is a great hotel.",
482490
DescriptionEmbedding = await embeddingGenerator.GenerateEmbeddingAsync("This is a great hotel."),

0 commit comments

Comments
 (0)