Merge pull request #58 from RobertKoehlmoos/670-feature-display-image-references

670 feature display image references

Author: Robert Koehlmoos

components/frontend_react/webapp/src/components/query/References.tsx

@@ -26,7 +26,7 @@ const References: React.FC<ReferencesProps> = ({ references }) => {
     const map = new Map<string, T>()
     items.forEach(item => {
       const key: string = String(item[dedupKey])
-        map.set(key, item)
+      map.set(key, item)
     })
     return Array.from(map.values())
   }
@@ -37,6 +37,8 @@ const References: React.FC<ReferencesProps> = ({ references }) => {
         .replace("/b/", "https://storage.googleapis.com/")
         .replace("/o/", "/")
     }
+    if (url.startsWith("gs://"))
+      return url.replace("gs://", "https://storage.googleapis.com/")
     return url
   }
 
@@ -91,7 +93,12 @@ const References: React.FC<ReferencesProps> = ({ references }) => {
                className="text-info hover:text-info-content break-all transition-colors mr-1.5">
               {renderLinkTitle(ref.document_url)}:
             </a>
-            <Markdown children={truncateText(ref.document_text)} rehypePlugins={[rehypeRaw]} />
+            {ref?.modality === "image" ?
+              <img src={renderCloudStorageUrl(ref.chunk_url)} /> :
+              // if the modality is not image assume it is text to handle
+              // legacy query engines that didn't set a modality
+              <Markdown children={truncateText(ref.document_text)} rehypePlugins={[rehypeRaw]} />
+            }
           </span>
           <div className={index < uniqueReferences.length - 1 ? "mx-2 pt-3 border-b" : ""} />
         </div>

components/frontend_react/webapp/src/routes/QueryEngineEdit.tsx

@@ -185,7 +185,6 @@ const QueryEngineEdit: React.FC<IQueryEngineProps> = ({ token }) => {
   useEffect(() => {
     const updateEngineEmbeddings = async () => {
       const llmTypes = await (await fetchEmbeddingTypes(token, createEngineIsMultimodal))()
-      console.log(llmTypes)
       if (llmTypes === null || llmTypes === undefined) console.error("Failed to retrieve embedding types")
       else setCreateEngineEmbeddingOptions(llmTypes.map((embedding) => { return { option: embedding, value: embedding } }))
     }

components/frontend_react/webapp/src/utils/types.ts

@@ -105,8 +105,10 @@ export interface QueryResponse {
 
 export interface QueryReferences {
   chunk_id: string
+  chunk_url: string
   document_url: string
   document_text: string
+  modality: string
 }
 
 export type QueryContents = {
@@ -243,5 +245,5 @@ export type QueryEngineBuildJob = {
   output_gcs_path: any
   errors: any
   job_logs: any
-  metadata: any  
+  metadata: any
 }