Merge branch 'main' into main

Author: sridhar-daxa

pebblo/app/config/config.py

@@ -22,11 +22,14 @@ def get_default_config_values():
     # set default config value
     conf_obj = Config(
         daemon=DaemonConfig(host="localhost", port=8000),
-        reports=ReportConfig(format="pdf", renderer="xhtml2pdf", cacheDir="~/.pebblo"),
-        logging=LoggingConfig(),
-        classifier=ClassifierConfig(
-            mode=ClassificationMode.ALL.value, anonymizeSnippets=False
+        reports=ReportConfig(
+            format="pdf",
+            renderer="xhtml2pdf",
+            cacheDir="~/.pebblo",
+            anonymizeSnippets=False,
         ),
+        logging=LoggingConfig(),
+        classifier=ClassifierConfig(mode=ClassificationMode.ALL.value),
         storage=StorageConfig(type="file", db=None),
         # for now, a default storage type is FILE, but in the next release DB will be the default storage type.
     )
@@ -37,7 +40,7 @@ def load_config(path: Optional[str]) -> Tuple[dict, Config]:
     try:
         if not path:
             # If Path does not exist in command, set default config value
-            get_default_config_values()
+            return get_default_config_values()
 
         # If Path exist, set config value
         try:

pebblo_safeloader/langchain/textloader_postgress/.env.sample

@@ -1,11 +1,11 @@
 # OpenAI credentials
 OPENAI_API_KEY=<YOUR OPENAI API KEY>
 
-# Pebblo configuration
-PEBBLO_CLOUD_URL=<PEBBLO CLOUD URL>
-PEBBLO_API_KEY=<YOUR PEBBLO API KEY>
-PEBBLO_CLASSIFIER_URL="http://localhost:8000/"
-
 # Postgres configuration
 PG_CONNECTION_STRING = "postgresql://<USERNAME>:<PASSWORD>@<HOST>:<PORT>/<DATABASE-NAME>"
 
+# Pebblo configuration
+PEBBLO_CLASSIFIER_URL="http://localhost:8000/"
+# Optional (only if you are using Pebblo Cloud)
+PEBBLO_CLOUD_URL=<PEBBLO CLOUD URL>
+PEBBLO_API_KEY=<YOUR PEBBLO API KEY>

pebblo_safeloader/langchain/textloader_postgress/README.md

@@ -0,0 +1,62 @@
+# Pebblo Text Loader
+
+This is a sample application that demonstrates how to use the `Pebblo Text Loader` to load the text data
+with the `Pebblo Safe Loader` into `Postgres` Vector Database.
+
+\* This solution uses predefined text data and metadata from the utility functions to demonstrate the loading of
+in-memory text data using Pebblo Safe Loader. Real-world applications can use this solution to load text data from
+various sources.
+
+**PebbloTextLoader**: PebbloTextLoader is a loader for text data. Since PebbloSafeLoader is a wrapper around document
+loaders, this loader is used to load text data directly into Documents.
+
+**This solution uses:**
+
+- PostgreSQL 15.7
+- langchain-community from daxa-ai/langchain branch(pebblo-0.1.19)
+
+### Instructions
+
+1. Create Python virtual-env
+
+```console
+$ python3 -m venv .venv
+$ source .venv/bin/activate
+```
+
+2. Install dependencies
+
+```console
+$ pip3 install -r requirements.txt
+```
+
+3. Install langchain-community from the branch `pebblo-0.1.19`
+
+```console
+$ git clone https://github.com/daxa-ai/langchain.git
+$ cd langchain
+$ git fetch && git checkout pebblo-0.1.19
+$ cd libs/community
+$ pip3 install langchain-community .
+```
+
+4. Copy the `.env.sample` file to `.env` and populate the necessary environment variable. The `.env` file should look
+   like this:
+
+```console
+$ cat .env
+# OpenAI credentials
+OPENAI_API_KEY=<YOUR OPENAI API KEY>
+
+# Postgres configuration
+PG_CONNECTION_STRING = "postgresql://<USERNAME>:<PASSWORD>@<HOST>:<PORT>/<DATABASE-NAME>"
+```
+
+5. Run Pebblo Safe Loader sample app
+
+```console
+$ python3 pebblo_safeload.py
+```
+
+6. Retrieve the Pebblo PDF report in `$HOME/.pebblo/pebblo-safe-loader-text-loader/pebblo_report.pdf` file path on the
+   system where `Pebblo Server` is running.

pebblo_safeloader/langchain/textloader_postgress/pebblo_safeload.py

@@ -48,6 +48,7 @@ def __init__(self, collection_name: str):
             description="Identity & Semantic enabled SafeLoader app using Pebblo",  # Description (Optional)
             load_semantic=True,
             api_key=PEBBLO_API_KEY,
+            anonymize_snippets=True,
         )
         self.documents = self.loader.load()
         unique_identities = set()

pebblo_safeloader/langchain/textloader_postgress/requirements.txt

@@ -2,7 +2,7 @@ python-dotenv==1.0.0
 tiktoken # OpenAI tokenizer
 
 langchain-openai>=0.1.7 # For OpenAI LLM and OpenAIEmbeddings
-langchain-community>=0.2.16,<0.3 # for PebbloSafeLoader, PebbloRetrievalQA
+#langchain-community>=0.2.16,<0.3 # for PebbloSafeLoader, PebbloRetrievalQA
 
 psycopg2-binary # For Postgres VectorStore
 langchain-postgres # For Postgres VectorStore

pebblo_safeloader/langchain/textloader_postgress/util.py

@@ -40,8 +40,12 @@ def get_data(
     if metadatas:
         # Metadata(source: fake news web url) for each text
         _metadata_list = [
-            {"source": f"https://www.acme.org/news/{i}"}
-            for i in range(1, len(texts) + 1)
+            {
+                "source": f"https://www.acme.org/news/{i + 1}",
+                "owner": "Joe Smith",
+                "size": f"{len(texts[i])}",
+            }
+            for i in range(len(texts))
         ]
     else:
         _metadata_list = None