Merge pull request #52 from RobertKoehlmoos/586-feature-multimodal-create-engine

586 feature multimodal create engine

Author: Robert Koehlmoos

.gitignore

@@ -41,3 +41,7 @@ node_modules
 
 # VSCode related
 *.code-workspace
+
+# react deployment configurations
+components/frontend_react/webapp/.env.production
+components/frontend_react/webapp/.env.development

components/frontend_react/webapp/src/components/forms/fields/BooleanField.tsx

@@ -37,6 +37,7 @@ const BooleanField: React.FC<BooleanFieldProps> = ({
               type="checkbox"
               className="peer toggle toggle-sm sr-only"
               name={variable.name}
+              onClick={() => { variable.onClick() }}
             />
             <div className="peer-focus:ring-7 h-6 w-11 rounded-full bg-base-300 after:absolute after:left-[2px] after:top-[2px] after:h-5 after:w-5 after:rounded-full after:border-primary after:border-primary after:bg-base-100 after:transition-all after:content-[''] peer-checked:bg-primary peer-checked:after:translate-x-full peer-checked:after:border-primary peer-focus:outline-none peer-focus:ring-primary dark:peer-focus:ring-primary"></div>
           </label>

components/frontend_react/webapp/src/routes/QueryEngineEdit.tsx

@@ -25,6 +25,7 @@ import { ALERT_TYPE } from "@/utils/types"
 import { Link, useNavigate } from "react-router-dom"
 import { useQueryParams } from "@/utils/routing"
 import { userStore, alertStore } from "@/store"
+import { fetchEmbeddingTypes } from "@/utils/api"
 
 interface IQueryEngineProps {
   token: string
@@ -45,6 +46,9 @@ const QueryEngineEdit: React.FC<IQueryEngineProps> = ({ token }) => {
   const [queryEngines, setQueryEngines] = useState<QueryEngine[]>([])
   const [queryEngine, setQueryEngine] = useState<QueryEngine | null>(null)
 
+  const [createEngineIsMultimodal, setCreateEngineIsMultimodal] = useState(false)
+  const [createEngineEmbeddingOptions, setCreateEngineEmbeddingOptions] = useState<{ option: string; value: string; }[]>([])
+
   const { isLoading, data: engineList } = useQuery(
     ["QueryEngines"],
     fetchAllEngines(token),
@@ -166,6 +170,29 @@ const QueryEngineEdit: React.FC<IQueryEngineProps> = ({ token }) => {
     return <Loading />
   }
 
+  const updatedQueryEngineFormData = QUERY_ENGINE_FORM_DATA.map(
+    (entry) => {
+      switch (entry.name) {
+        case "is_multimodal":
+          return { ...entry, onClick: () => { setCreateEngineIsMultimodal(!createEngineIsMultimodal) } }
+        case "embedding_type":
+          return { ...entry, options: createEngineEmbeddingOptions }
+        default:
+          return entry
+      }
+    })
+
+  useEffect(() => {
+    const updateEngineEmbeddings = async () => {
+      const llmTypes = await (await fetchEmbeddingTypes(token, createEngineIsMultimodal))()
+      console.log(llmTypes)
+      if (llmTypes === null || llmTypes === undefined) console.error("Failed to retrieve embedding types")
+      else setCreateEngineEmbeddingOptions(llmTypes.map((embedding) => { return { option: embedding, value: embedding } }))
+    }
+    updateEngineEmbeddings()
+  },
+    [createEngineIsMultimodal])
+
   return (
     <div className="overflow-x-auto custom-scrollbar">
       <div className="min-h-screen">
@@ -204,7 +231,7 @@ const QueryEngineEdit: React.FC<IQueryEngineProps> = ({ token }) => {
               onFailure={onFailure}
               handleFiles={null}
               queryEngine={queryEngine}
-              currentVarsData={QUERY_ENGINE_FORM_DATA}
+              currentVarsData={updatedQueryEngineFormData}
             />
           </div>
 

components/frontend_react/webapp/src/utils/api.ts

@@ -118,6 +118,13 @@ export const resumeChat =
     return axios.post(url, data, { headers }).then(path(["data", "data"]))
   }
 
+export const fetchEmbeddingTypes =
+  (token: string, are_multimodal: boolean) => (): Promise<string[] | undefined | null> => {
+    let url = `${endpoint}/llm/embedding_types?is_multimodal=${are_multimodal}`
+    const headers = { Authorization: `Bearer ${token}` }
+    return axios.get(url, { headers }).then(path(["data", "data"]))
+  }
+
 export const fetchQuery =
   (token: string, queryId: string | null) => (): Promise<Query | undefined | null> => {
     if (!queryId) return Promise.resolve(null)
@@ -168,6 +175,7 @@ export const createQueryEngine =
         "agents": queryEngine.agents,
         "associated_engines": queryEngine.child_engines,
         "manifest_url": queryEngine.manifest_url,
+        "is_multimodal": queryEngine.is_multimodal ? "True" : "False",
       }
     }
     return axios.post(url, data, { headers }).then(path(["data", "data"]))

components/frontend_react/webapp/src/utils/data.ts

@@ -88,7 +88,7 @@ export const QUERY_ENGINE_FORM_DATA: IFormVariable[] = [
     order: 1,
   },
   {
-    name: "type",
+    name: "query_engine_type",
     display: "Query Engine Type",
     tooltip: "Type of Query Engine. GENIE is the native type.",
     type: "select",
@@ -101,6 +101,17 @@ export const QUERY_ENGINE_FORM_DATA: IFormVariable[] = [
     group: "queryengine",
     order: 1,
   },
+  {
+    name: "is_multimodal",
+    display: "Multimodal Engine",
+    tooltip: "Type of Query Engine. GENIE is the native type.",
+    type: "bool",
+    description: "",
+    default: false,
+    required: true,
+    group: "queryengine",
+    order: 1,
+  },
   {
     name: "doc_url",
     display: "Data URL",
@@ -140,9 +151,7 @@ export const QUERY_ENGINE_FORM_DATA: IFormVariable[] = [
     display: "Embedding Type",
     type: "select",
     description: "",
-    options: [{option: "Vertex", value: "VertexAI-Embedding"},
-              {option: "OpenAI", value: "OpenAI-Embedding"},
-              {option: "HuggingFace", value: "HuggingFaceEmbeddings"}],
+    options: [],
     default: "Vertex",
     required: true,
     group: "queryengine",

components/frontend_react/webapp/src/utils/types.ts

@@ -13,7 +13,7 @@
 // limitations under the License.
 
 import { ReactNode } from "react"
-import { z } from "zod"
+import { any, z } from "zod"
 
 export type INavigationItem = {
   name: string
@@ -70,6 +70,7 @@ export const FormVariable = z.object({
   fileLabel: z.string().optional(),
   multiple: z.boolean().default(false).optional(),
   accept: z.string().optional(),
+  onClick: z.function().optional(),
 })
 
 export type IFormVariable = z.infer<typeof FormVariable>
@@ -189,8 +190,9 @@ export type QueryEngine = {
   manifest_url: string | null
   params: string[] | null
   depth_limit: number | null
-  agents: string[] | null  
+  agents: string[] | null
   child_engines: string[] | null
+  is_multimodal: boolean | null
 }
 
 export type QueryEngineBuildParams = {