Skip to content

Commit 006025c

Browse files
tomlmtomlm
committed
add support for images
1 parent b48be79 commit 006025c

File tree

5 files changed

+137
-21
lines changed

5 files changed

+137
-21
lines changed

‎README.md

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -208,6 +208,31 @@ var results = items.TransformItems(model, "translate to spanish);
208208
// result[2] = "Una mas, por favor"
209209
```
210210
211+
## Linq.AI and Vision
212+
You can use Linq.AI and the model to work with images.
213+
214+
Simply pass a **Uri()** to an image or a **ChatMessageContentPart** for the image and
215+
you can call any of the extension methods.
216+
217+
Examples:
218+
```csharp
219+
var uri = new Uri("https://2cupsoftravel.com/wp-content/uploads/2022/10/Oktoberfest-munich-things-to-know.jpg");
220+
var uri2 = new Uri("https://2cupsoftravel.com/wp-content/uploads/2022/10/20220928_115250-1200x900.jpg");
221+
222+
// summarize an image uri
223+
var summmary = await Model.SummarizeAsync(uri);
224+
225+
// ask a question about a group of image uris
226+
var matches = await Model.MatchesAsync(new [] { uri, uri2 }, "Are these pictures of people drinking beer?");
227+
228+
// extract text out of an image uri.
229+
var text = await Model.SelectAsync<string>(uri, "Extract all phrases from the image");
230+
231+
// upload and classify an image binary
232+
var imagePart = ChatMessageContentPart.CreateImagePart(BinaryData.FromBytes(imageBytes), "image/jpeg");
233+
var classification = await Model.ClassifyAsync<Mood>(imagePart);
234+
```
235+
211236
## Adding Tools
212237
Linq.AI makes it super easy to add tools to your OpenAI model.
213238
There are 2 ways to do it:

‎source/Linq.AI.OpenAI.Tests/TransformerTests.cs renamed to ‎source/Linq.AI.OpenAI.Tests/GenerationTests.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ namespace Linq.AI.OpenAI.Tests
55

66

77
[TestClass]
8-
public class TransformerTests : UnitTestBase
8+
public class GenerationTests : UnitTestBase
99
{
1010

1111
[TestMethod]

‎source/Linq.AI.OpenAI.Tests/TransformTests.cs

Lines changed: 56 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
using Iciclecreek.Async;
2+
using OpenAI.Chat;
23
using System;
34
using System.Diagnostics;
45

@@ -8,14 +9,14 @@ namespace Linq.AI.OpenAI.Tests
89
[TestClass]
910
public class TransformationTests : UnitTestBase
1011
{
11-
12+
1213
[TestMethod]
1314
public async Task Transform_String2String()
1415
{
1516
var source = "My name is Tom.";
1617
var transformation = Model.TransformItem<string>(source, "into spanish");
1718
Assert.IsTrue(Model.Compare("Me llamo Tom.", transformation!));
18-
19+
1920
transformation = await Model.TransformItemAsync<string>(source, "into spanish");
2021
Assert.IsTrue(Model.Compare("Me llamo Tom.", transformation!));
2122
}
@@ -27,7 +28,7 @@ public async Task Transform_String2Object()
2728
var obj = Model.TransformItem<TestObject>(source);
2829
Assert.AreEqual("Inigo Montoya", obj.Name);
2930
Assert.AreEqual(4, obj.Count);
30-
31+
3132
obj = await Model.TransformItemAsync<TestObject>(source);
3233
Assert.AreEqual("Inigo Montoya", obj.Name);
3334
Assert.AreEqual(4, obj.Count);
@@ -69,7 +70,8 @@ public async Task Transform_Classify_text()
6970
[TestMethod]
7071
public async Task Transform_Answer()
7172
{
72-
Assert.IsTrue(Model.Compare("Honolulu", await Model.TransformItemAsync<string>(Text, "What town was he born in?")));
73+
var result = await Model.TransformItemAsync<string>(Text, "What town was he born in?");
74+
Assert.IsTrue(result.ToLower().Contains("honolulu"));
7375
}
7476

7577
[TestMethod]
@@ -103,7 +105,7 @@ public async Task Transform_Text2Objects()
103105
var results = await Model.TransformItemAsync<Article[]>(Text);
104106

105107
Assert.IsTrue(results.Count() > 0);
106-
foreach(var result in results)
108+
foreach (var result in results)
107109
{
108110
Assert.IsNotNull(result.Title);
109111
Assert.IsNotNull(result.Paragraph);
@@ -190,6 +192,55 @@ public void Transform_Collection_Object2Object()
190192

191193
}
192194

195+
[TestMethod]
196+
public async Task Transform_Vision_UriTest()
197+
{
198+
var uri = new Uri("https://2cupsoftravel.com/wp-content/uploads/2022/10/Oktoberfest-munich-things-to-know.jpg");
199+
var result = await Model.SummarizeAsync(uri);
200+
Assert.IsTrue(result.Contains("beer"));
201+
}
202+
203+
[TestMethod]
204+
public async Task Transform_Vision_UrisTest()
205+
{
206+
var uri = new Uri("https://2cupsoftravel.com/wp-content/uploads/2022/10/Oktoberfest-munich-things-to-know.jpg");
207+
var uri2 = new Uri("https://2cupsoftravel.com/wp-content/uploads/2022/10/20220928_115250-1200x900.jpg");
208+
var result = await Model.MatchesAsync(new { uri, uri2 }, "Are these pictures of people drinking beer?");
209+
Assert.IsTrue(result);
210+
}
211+
212+
[TestMethod]
213+
public async Task Transform_Vision_Select()
214+
{
215+
var uri = new Uri("https://static.vecteezy.com/system/resources/previews/001/222/391/original/blue-elegant-decorative-striped-pattern-editable-text-effect-vector.jpg");
216+
var results = await Model.SelectAsync<string>(uri, "Extract all phrases from the image");
217+
Assert.AreEqual("ILLUSTRATOR GRAPHIC STYLE", results[0]);
218+
Assert.AreEqual("Works with text box or shape", results[1]);
219+
Assert.AreEqual("Chicago", results[2]);
220+
Assert.AreEqual("EASY TO USE - 100% EDITABLE", results[3]);
221+
Assert.AreEqual("Open the graphic style menu and apply", results[4]);
222+
}
223+
224+
[TestMethod]
225+
public async Task Transform_PartTest()
226+
{
227+
var uri = new Uri("https://2cupsoftravel.com/wp-content/uploads/2022/10/Oktoberfest-munich-things-to-know.jpg");
228+
var data = await new HttpClient().GetByteArrayAsync(uri);
229+
var result = await Model.SummarizeAsync(ChatMessageContentPart.CreateImagePart(BinaryData.FromBytes(data), "image/jpeg"));
230+
Assert.IsTrue(result.Contains("beer"));
231+
}
232+
233+
[TestMethod]
234+
public async Task Transform_PartsTest()
235+
{
236+
var result = await Model.MatchesAsync(new []
237+
{
238+
ChatMessageContentPart.CreateImagePart(new Uri("https://2cupsoftravel.com/wp-content/uploads/2022/10/Oktoberfest-munich-things-to-know.jpg")),
239+
ChatMessageContentPart.CreateImagePart(new Uri("https://2cupsoftravel.com/wp-content/uploads/2022/10/20220928_115250-1200x900.jpg"))
240+
}, "Are these pictures of people drinking beer?");
241+
Assert.IsTrue(result);
242+
}
243+
193244
internal class TestObject
194245
{
195246
public string? Name { get; set; }

‎source/Linq.AI.OpenAI/OpenAITransformer.cs

Lines changed: 53 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -210,11 +210,14 @@ public async Task<ResultT> TransformItemAsync<ResultT>(object item, string? goal
210210
lock (this)
211211
{
212212
foreach (var message in messages)
213-
Debug.WriteLine(message.Content.Single().Text);
213+
{
214+
foreach(var part in message.Content)
215+
Debug.WriteLine(JsonConvert.SerializeObject(part, JsonSettings));
216+
}
214217
}
215218
#endif
216219
int retries = 2;
217-
while(retries-- > 0)
220+
while (retries-- > 0)
218221
{
219222
ChatCompletion completion = await _chatClient.CompleteChatAsync(messages, options, cancellationToken: cancellationToken);
220223

@@ -282,8 +285,8 @@ public async Task<ResultT> TransformItemAsync<ResultT>(object item, string? goal
282285
if (task != null)
283286
{
284287
await (Task)task;
285-
result = task!.GetType().GetProperty("Result", BindingFlags.FlattenHierarchy |
286-
BindingFlags.Public |
288+
result = task!.GetType().GetProperty("Result", BindingFlags.FlattenHierarchy |
289+
BindingFlags.Public |
287290
BindingFlags.Instance)!.GetValue(task);
288291
}
289292
}
@@ -316,7 +319,7 @@ public async Task<ResultT> TransformItemAsync<ResultT>(object item, string? goal
316319
throw new NotImplementedException(completion.FinishReason.ToString());
317320
}
318321
}
319-
322+
320323
throw new Exception("Too many function calls detected!");
321324
}
322325

@@ -344,27 +347,64 @@ public IList<ResultT> TransformItems<ResultT>(IEnumerable<object> source, string
344347
internal static SystemChatMessage GetTransformerSystemPrompt(string goal, string? instructions = null)
345348
{
346349
return new SystemChatMessage($$"""
347-
You are an expert at transforming text.
350+
You are an expert at transforming.
348351
349352
<GOAL>
350353
{{goal}}
351354
352355
<INSTRUCTIONS>
353-
Given <ITEM> text transform the text using the directions in the provided <GOAL>.
356+
Transform <ITEM> using the directions in the provided <GOAL>.
354357
{{instructions}}
355358
""");
356359
}
357360

358361
internal static UserChatMessage GetTransformerItemMessage(object item)
359362
{
360-
if (!(item is string))
363+
if (item is string)
364+
{
365+
return new UserChatMessage($$"""
366+
<ITEM>
367+
{{item}}
368+
""");
369+
}
370+
else if (item is ChatMessageContentPart contentPart)
371+
{
372+
if (contentPart.Kind == ChatMessageContentPartKind.Image)
373+
return new UserChatMessage(ChatMessageContentPart.CreateTextPart("<ITEM>"), contentPart);
374+
else
375+
return new UserChatMessage($$"""
376+
<ITEM>
377+
{{contentPart.Text}}
378+
""");
379+
}
380+
else if (item is ChatMessageContentPart[] contentParts)
381+
{
382+
return new UserChatMessage(contentParts);
383+
}
384+
else if (item is Uri uri)
385+
{
386+
return new UserChatMessage(ChatMessageContentPart.CreateTextPart("<ITEM>"), ChatMessageContentPart.CreateImagePart(uri));
387+
}
388+
else if (item is Uri[] uris)
389+
{
390+
List<ChatMessageContentPart> parts = new List<ChatMessageContentPart>()
391+
{
392+
ChatMessageContentPart.CreateTextPart("<ITEM>")
393+
};
394+
395+
foreach (var u in uris)
396+
{
397+
parts.Add(ChatMessageContentPart.CreateImagePart(u));
398+
}
399+
return new UserChatMessage(parts);
400+
}
401+
else
361402
{
362-
item = JToken.FromObject(item).ToString();
403+
return new UserChatMessage($$"""
404+
<ITEM>
405+
{{JToken.FromObject(item).ToString()}}
406+
""");
363407
}
364-
return new UserChatMessage($$"""
365-
<ITEM>
366-
{{item}}
367-
""");
368408
}
369409

370410
}

‎source/Linq.AI/SummarizeExtension.cs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ public static class SummarizeExtension
2020
/// <param name="instructions">(OPTIONAL) additional instructions on how to summarize</param>
2121
/// <param name="cancellationToken">(OPTIONAL) Cancellation Token</param>
2222
/// <returns>Summarization text</returns>
23-
public static string Summarize(this ITransformer model, object source, string? goal, string? instructions = null, CancellationToken cancellationToken = default)
23+
public static string Summarize(this ITransformer model, object source, string? goal = null, string? instructions = null, CancellationToken cancellationToken = default)
2424
=> model.TransformItem<string>(source, goal ?? "summarize", instructions, cancellationToken);
2525

2626
/// <summary>
@@ -32,7 +32,7 @@ public static string Summarize(this ITransformer model, object source, string?
3232
/// <param name="instructions">(OPTIONAL) additional instructions on how to summarize</param>
3333
/// <param name="cancellationToken">(OPTIONAL) Cancellation Token</param>
3434
/// <returns>Summarization text</returns>
35-
public static Task<string> SummarizeAsync(this ITransformer model, object source, string? goal, string? instructions = null, CancellationToken cancellationToken = default)
35+
public static Task<string> SummarizeAsync(this ITransformer model, object source, string? goal = null, string? instructions = null, CancellationToken cancellationToken = default)
3636
=> model.TransformItemAsync<string>(source, goal ?? "summarize", instructions, cancellationToken);
3737

3838
/// <summary>

0 commit comments

Comments
 (0)