Merge pull request #61 from GPS-Solutions/resolve_merge_conflicts_from_main_to_dev2

Updating dev branch with latest commits into css main

Author: smcazares

CHANGELOG.md

@@ -1,5 +1,27 @@
 # Releases
 
+## v0.4.0
+
+New features:
+
+### GENIE ("GenAI for Enterprise" platform on GCP)
+- Multimodal RAG for images using multi-modal embeddings for search
+- Chat file upload in both backend and React app.  Upload files or specify URLs to pass to model.
+- RBAC for model access.  See README for LLM Service for docs on how to manage access to models for users.
+- Added user role management in React frontend, used for RBAC for models and query engines
+- Added documentation on authentication in the platform in docs/AUTH.md
+- Updated to recent releases of FastAPI (0.112.2) and associated libraries
+- Use L4 GPUs with Truss models
+- Chunk size and chunking class are now query engine build params
+- Add chunk size to React Query Admin engine build form
+- Switched to using llama_index.core.node_parser.SentenceSplitter for chunking by default
+- Updated default query generation model to Gemini Flash 1.5 - was set to Palm2
+- Added Microsoft login in React frontend
+
+### Fixes
+- Fixed download of PDFs from scraped sites
+
+
 ## v0.3.2
 
 New features:

components/frontend_react/README.md

@@ -1,11 +1,11 @@
 # GENIE React Frontend
 This component is a REACT based frontend UI for GENIE.
 
-# Install
+## Install
 
 You must deploy GENIE first before deploying this frontend app.  See [the install guide for GENIE.](../../INSTALL.md)
 
-## Prerequisites
+### Prerequisites
 
 The following prerequisites must be installed to deploy the React frontend app:
 
@@ -15,14 +15,39 @@ The following prerequisites must be installed to deploy the React frontend app:
 | `npm`               | `>= 10.2`        | [Mac](https://nodejs.org/en/download/) • [Windows](https://nodejs.org/en/download/) • [Linux](https://nodejs.org/en/download/package-manager/) |
 | `firebase CLI`      | `>= v13.1.0`     | `utils/install_firebase.sh v13.1.0` |
 
-## Jump host installation
+### Jump host installation
 If you are installing from the jump host, install npm and the firebase CLI using the links above.
 
-# Build and deploy the app
+## Build and deploy the app
 
-## Add Google identity provider
+### Add Google identity provider
+
+Add Google as an identity provider.  You can do this in the [GCP console](https://console.cloud.google.com/customer-identity/providers) or in the [Firebase console](https://console.firebase.google.com/).  In firebase, navigate to Build > Authentication > Sign-in Method. Refer to authentication component [README.md](https://github.com/GPS-Solutions/core-solution-services/blob/main/components/authentication/README.md) for more information.
+
+### Authorizing User Domains during Sign-in
+The frontend_react component provides an initial check for authorizing user domains during a user's sign-in process with Google. Thus, you'll need to change the `authProviders` and `authorizedDomains` attribute within `AppConfig` with your user's or client's organizational domain.
+
+Under the `frontend_react/src/src/utils/AppConfig.ts` file:
+
+```
+export const AppConfig: IAppConfig = {
+  siteName: "GenAI for Public Sector",
+  locale: "en",
+  logoPath: "/assets/images/rit-logo.png",
+  simpleLogoPath: "/assets/images/rit-brain.png",
+  imagesPath: "/assets/images",
+  theme: "light",
+  authProviders: ["google", "microsoft", "facebook", "password"],
+  authorizedDomains: [/@google\.com$/i, /@gmail\.com$/i, /@\w+\.altostrat\.com$/i],
+}
+```
+
+> Add or Change the `authProviders` and `authorizedDomains` to your respective input.
+
+>**NOTE:** The `authorizedDomain` attributes are in reg expressions. (i.e "/@gmail\.com$/i")
+
+> In addition to this frontend configuration, you'll need to ensure the [Google Cloud Identity](https://console.cloud.google.com/customer-identity/providers) has added the providers on Google Cloud's backend. Each provider (e.g Microsoft, Facebook) will have require an authentication client on the provider-side that Google Cloud refers to via `App ID` and `App Secret` to direct authentication. Ensure Authorized Redirect URIs are set on the authentication provider side. See provider's documentation for more info. 
 
-Add Google as an identity provider.  You can do this in the [GCP console](https://console.cloud.google.com/customer-identity/providers) or in the [Firebase console](https://console.firebase.google.com/).  In firebase, navigate to Build > Authentication > Sign-in Method.
 
 ## Install dependencies
 Execute all commands below from the `components/frontend_react/webapp` directory.  You only need to install dependencies once, unless you update the app.
@@ -59,14 +84,22 @@ You should build the app on first deploy, and every time you make updates to the
 npm run build
 ```
 
-
 ## Deploy with firebase
 Deploy the app to firebase hosting with the following command:
 
 ```bash
 firebase deploy --only hosting
 ```
 
+### Authorizing Redirect URIs (OAuth 2.0 Authentication)
+In the Google Cloud Console -> APIs & Services -> [Credentials](https://console.cloud.google.com/apis/credentials):
+- Click on your default Web Client(auto-created by Google Service).
+- Under Authorized redirect URIs, add the following with your domain name:
+  - `https://<your-domain-name>.web.app/__/auth/handler`
+
+>This allows your backend to authorize your frontend web app in requesting an OAuth 2.0 authentication.Without this authorized redirect URIs, you will receive an unauthorized error.
+
+
 # Development
 
 ## Run a local dev server
@@ -75,4 +108,3 @@ This command will start a local instance of the app for development.
 ```bash
 npm run dev
 ```
-

components/frontend_react/webapp/public/assets/images/navy_logo.png

@@ -19,6 +19,7 @@ import { Link } from "react-router-dom"
 import { IFormValidationData, IFormVariable } from "@/utils/types"
 import { IQueryEngine } from "@/utils/models"
 import { formValidationSchema, initialFormikValues } from "@/utils/forms"
+import * as yup from "yup"
 
 interface QueryEngineFormProps {
   queryEngine: IQueryEngine | null
@@ -78,13 +79,20 @@ const QueryEngineForm: React.FunctionComponent<QueryEngineFormProps> = ({
 
   const initialValues = Object.assign({}, defaultValues, qEngineInitialFormat)
 
+  const validationSettings = {
+    name: yup
+      .string()
+      .max(32, "Query Engine names must be less than or equal to 32 chars")
+      .matches("^[a-zA-Z0-9][\w\s-]*[a-zA-Z0-9]$", "Invalid query engine name.  May contain alphanumerics, dashes or spaces.")
+  }
   const formValidationData: IFormValidationData =
-    formValidationSchema(currentVarsData)
+    formValidationSchema(currentVarsData, validationSettings)
 
   const formik = useFormik({
     initialValues: initialValues,
     enableReinitialize: true,
     validateOnMount: true,
+    validateOnChange: true,
     validationSchema: formValidationData,
     onSubmit: async (values) => {
       await handleSubmit(values)

components/frontend_react/webapp/src/components/forms/QueryEngineForm.tsx

@@ -62,7 +62,7 @@ const QueryWindow: React.FC<QueryWindowProps> = ({ onSubmit, messages, activeJob
       )
     } else if (message.AIReferences) {
       return (
-        <div className="ml-12 mb-2">
+        <div key={index++} className="ml-12 mb-2">
           <Expander title={"References"}>
             <References references={message.AIReferences} />
           </Expander>

components/frontend_react/webapp/src/components/query/QueryWindow.tsx

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 import QueryEngineForm from "@/components/forms/QueryEngineForm"
+import { QUERY_ENGINE_FORM_DATA } from "@/utils/data"
 import Header from "@/components/typography/Header"
 import { fetchEngine, fetchAllEngineJobs } from "@/utils/api"
 import Loading from "@/navigation/Loading"

components/frontend_react/webapp/src/routes/QueryEngineDetail.tsx

@@ -178,6 +178,7 @@ export const createQueryEngine =
         "agents": queryEngine.agents,
         "associated_engines": queryEngine.child_engines,
         "manifest_url": queryEngine.manifest_url,
+        "chunk_size": queryEngine.chunk_size,
         "is_multimodal": queryEngine.is_multimodal ? "True" : "False",
       }
     }

components/frontend_react/webapp/src/utils/api.ts

@@ -162,12 +162,23 @@ export const QUERY_ENGINE_FORM_DATA: IFormVariable[] = [
     display: "Depth Limit",
     type: "select",
     description: "The depth to crawl for web data sources.",
-    options: [0, 1, 2, 3, 4],    
+    options: [0, 1, 2, 3, 4],
     default: 0,
     required: false,
     group: "queryengine",
     order: 4,
   },
+  {
+    name: "chunk_size",
+    display: "Chunk Size",
+    type: "select",
+    description: "Chunking size for RAG.  Smaller is better for accuracy but makes builds take significantly longer.",
+    options: [50, 100, 200, 300, 400, 500],
+    default: "500",
+    required: false,
+    group: "queryengine",
+    order: 4,
+  },
   {
     name: "description",
     display: "Description",

components/frontend_react/webapp/src/utils/data.ts

@@ -87,9 +87,9 @@ export const downloadFile = async (
     })
 }
 
-export const formValidationSchema = (variableList: IFormVariable[]) => {
-  let formValidationData: IFormValidationData = {}
-
+export const formValidationSchema = (variableList: IFormVariable[], formValidationSettings: IFormValidationData = {}) => {
+  let formValidationData: IFormValidationData = formValidationSettings
+  
   variableList.forEach((variable) => {
     variable.required
       ? (formValidationData[variable.name] =

components/frontend_react/webapp/src/utils/forms.ts

@@ -200,7 +200,8 @@ export type QueryEngine = {
     [key: string]: any
   } | null
   depth_limit: number | null
-  agents: string[] | null
+  chunk_size: number | null
+  agents: string[] | null  
   child_engines: string[] | null
   is_multimodal: boolean | null
 }

components/frontend_react/webapp/src/utils/types.ts

@@ -239,7 +239,8 @@ curl --location "$BASE_URL/llm-service/api/v1/query/engine" \
 --header "Authorization: Bearer $ID_TOKEN" \
 --data "{
     \"doc_url\": \"gs://$PROJECT_ID-llm-docs\",
-    \"query_engine\": \"$QUERY_ENGINE_NAME\"
+    \"query_engine\": \"$QUERY_ENGINE_NAME\",
+    \"description\": "test"
 }"
 ```
 

components/llm_service/README.md

@@ -125,7 +125,7 @@ def get_model_config() -> ModelConfig:
 DEFAULT_LLM_TYPE = VERTEX_LLM_TYPE_CHAT
 DEFAULT_CHAT_LLM_TYPE = VERTEX_LLM_TYPE_CHAT
 DEFAULT_MULTIMODAL_LLM_TYPE = VERTEX_LLM_TYPE_GEMINI_FLASH
-DEFAULT_QUERY_CHAT_MODEL = VERTEX_LLM_TYPE_BISON_CHAT
+DEFAULT_QUERY_CHAT_MODEL = VERTEX_LLM_TYPE_GEMINI_FLASH
 DEFAULT_QUERY_EMBEDDING_MODEL = VERTEX_LLM_TYPE_GECKO_EMBEDDING
 DEFAULT_QUERY_MULTIMODAL_EMBEDDING_MODEL = VERTEX_LLM_TYPE_GECKO_EMBEDDING_VISION
 

components/llm_service/src/config/config.py

@@ -43,7 +43,7 @@
 # token limit across all chunks of 20000, and with 250 chunks we often
 # exceeded the 20K limit.
 if REGION == "us-central1":
-  ITEMS_PER_REQUEST = 50
+  ITEMS_PER_REQUEST = 20
 else:
   ITEMS_PER_REQUEST = 5
 

components/llm_service/src/services/embeddings.py

@@ -35,7 +35,8 @@
 from utils.errors import NoDocumentsIndexedException
 from utils import text_helper, gcs_helper
 from llama_index.core import SimpleDirectoryReader
-from llama_index.core.node_parser import SentenceWindowNodeParser
+from llama_index.core.node_parser import (SentenceSplitter,
+                                         SentenceWindowNodeParser)
 from llama_index.core import Document
 
 # pylint: disable=broad-exception-caught
@@ -50,6 +51,13 @@
 # itself be parsed
 GENIE_FOLDER_MARKER = "_genie_"
 
+# default chunk size for doc chunks
+DEFAULT_CHUNK_SIZE = 250
+
+# datasource param keys
+CHUNKING_CLASS_PARAM = "chunking_class"
+CHUNK_SIZE_PARAM = "chuck_size"
+
 class DataSourceFile():
   """ object storing meta data about a data source file """
   def __init__(self,
@@ -85,16 +93,29 @@ class DataSource:
   Super class for query data sources. Also implements GCS DataSource.
   """
 
-  def __init__(self, storage_client):
+  def __init__(self, storage_client, params=None):
     self.storage_client = storage_client
     self.docs_not_processed = []
-    # use llama index sentence window parser
-    self.doc_parser = SentenceWindowNodeParser.from_defaults(
-      window_size=CHUNK_SENTENCE_PADDING,
-      include_metadata=True,
-      window_metadata_key="window_text",
-      original_text_metadata_key="text",
-    )
+    self.params = params or {}
+
+    # set chunk size
+    if CHUNK_SIZE_PARAM in self.params:
+      self.chunk_size = int(self.params[CHUNK_SIZE_PARAM])
+    else:
+      self.chunk_size = DEFAULT_CHUNK_SIZE
+
+    # use llama index sentence splitter for chunking
+    if CHUNKING_CLASS_PARAM in self.params \
+        and self.params[CHUNKING_CLASS_PARAM] == "SentenceWindowNodeParser":
+      self.doc_parser = SentenceWindowNodeParser.from_defaults(
+        window_size=CHUNK_SENTENCE_PADDING,
+        include_metadata=True,
+        window_metadata_key="window_text",
+        original_text_metadata_key="text",
+      )
+    else:
+      self.doc_parser = SentenceSplitter(chunk_size=self.chunk_size)
+
 
   @classmethod
   def downloads_bucket_name(cls, q_engine_name: str) -> str:
@@ -194,12 +215,9 @@ def chunk_document(self, doc_name: str, doc_url: str,
        doc_url: remote url of document
        doc_filepath: local file path of document
     Returns:
-       tuple of 
-          list of text chunks or None if the document could not be processed
-          list of embedding chunks or None
+       list of text chunks or None if the document could not be processed
     """
 
-    embed_chunks = None
     text_chunks = None
 
     Logger.info(f"generating index data for {doc_name}")
@@ -219,28 +237,30 @@ def chunk_document(self, doc_name: str, doc_url: str,
     if doc_text_list is not None:
       # clean text of escape and other unprintable chars
       doc_text_list = [self.clean_text(x) for x in doc_text_list]
+
       # combine text from all pages to try to avoid small chunks
       # when there is just title text on a page, for example
       doc_text = "\n".join(doc_text_list)
+
       # llama-index base class that is used by all parsers
       doc = Document(text=doc_text)
+
       # a node = a chunk of a page
       chunks = self.doc_parser.get_nodes_from_documents([doc])
+
       # remove any empty chunks
-      chunks = [c for c in chunks if c.metadata["text"].strip() != ""]
-      # this is a sentence parser with overlap --
-      # each text chunk will include the specified
-      # number of sentences before and after the current sentence
-      embed_chunks = [c.metadata["text"] for c in chunks]
-      text_chunks = [c.metadata["window_text"] for c in chunks]
+      chunks = [c for c in chunks if c.text.strip() != ""]
+
+      # get text chunks
+      text_chunks = [c.text for c in chunks]
 
       if all(element == "" for element in text_chunks):
         Logger.warning(f"All extracted pages from {doc_name} are empty.")
         self.docs_not_processed.append(doc_url)
       else:
         Logger.info(f"generated {len(text_chunks)} text chunks for {doc_name}")
 
-    return text_chunks, embed_chunks
+    return text_chunks
 
   def chunk_document_multimodal(self,
                            doc_name: str,
@@ -385,19 +405,16 @@ def extract_contextual_text(self, doc_name: str, doc_filepath: str, \
     Returns:
       str containing the contextual_text of a multimodal doc
     """
-    #chunk_document returns 2 outputs, text_chunks and contextual_text.
-    #Each element of text_chunks has the same info as its corresponding
-    #element in contextual_text, but is padded with adjacent sentences
-    #before and after. Use the 2nd output here (contextual_text).
-    _, contextual_text = self.chunk_document(doc_name,
+    # Chunk the text of the document into a list of strings
+    contextual_text = self.chunk_document(doc_name,
                                           doc_url, doc_filepath)
 
     # Format text if not None
     if contextual_text is not None:
       contextual_text = [string.strip() for string in contextual_text]
       contextual_text = " ".join(contextual_text)
 
-      #TODO: Consider all characters in my_contextual_text,
+      #TODO: Consider all characters in contextual_text,
       #not just the first 1024
       contextual_text = contextual_text[0:1023]