Merge pull request #5 from whyhow-ai/unstructured-integration

Author: tomsmoker

.python-version

@@ -0,0 +1 @@
+3.10

README.md

@@ -6,6 +6,7 @@ Our goal is to provide a familiar, spreadsheet-like interface for business users
 
 [![MIT License](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE)
 [![GitHub issues](https://img.shields.io/github/issues/whyhow-ai/knowledge-table)](https://github.com/whyhow-ai/knowledge-table/issues)
+[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
 
 For a limited demo, check out the [Knowledge Table Demo](https://knowledge-table-demo.whyhow.ai/).
 
@@ -98,10 +99,26 @@ The frontend can be accessed at `http://localhost:3000`, and the backend can be
 
 4. **Install the dependencies:**
 
+   For basic installation:
    ```sh
    pip install .
    ```
 
+   For installation with Unstructured support:
+   ```sh
+   pip install .[unstructured]
+   ```
+
+   For installation with development tools:
+   ```sh
+   pip install .[dev]
+   ```
+
+   For full installation (including Unstructured and dev tools):
+   ```sh
+   pip install .[full]
+   ```
+
 5. **Start the backend:**
 
    ```sh
@@ -150,6 +167,27 @@ The frontend can be accessed at `http://localhost:3000`, and the backend can be
 
 ---
 
+## Development
+
+To set up the project for development:
+
+1. Clone the repository
+2. Install the project with development dependencies:
+   ```sh
+   pip install .[dev]
+   ```
+3. Run tests:
+   ```sh
+   pytest
+   ```
+4. Run linters:
+   ```sh
+   flake8
+   black .
+   isort .
+   ```
+---
+
 ## Features
 
 ### Available in this repo:
@@ -205,6 +243,29 @@ Knowledge Table is built to be flexible and customizable, allowing you to extend
 
 ---
 
+## Optional Integrations
+
+### Unstructured API
+
+Knowledge Table offers optional integration with the Unstructured API for enhanced document processing capabilities. This integration allows for more advanced parsing and extraction from various document types.
+
+To use the Unstructured API integration:
+
+1. Sign up for an API key at [Unstructured.io](https://www.unstructured.io/).
+2. Set the `UNSTRUCTURED_API_KEY` environment variable in the `.env` file, or with your API key:
+   ```
+   export UNSTRUCTURED_API_KEY=your_api_key_here
+   ```
+3. Install the project with Unstructured support:
+   ```
+   pip install .[unstructured]
+   ```
+
+When the `UNSTRUCTURED_API_KEY` is set, Knowledge Table will automatically use the Unstructured API for document processing. If the key is not set or if there's an issue with the Unstructured API, the system will fall back to the default document loaders.
+
+Note: Usage of the Unstructured API may incur costs based on your plan with Unstructured.io.
+---
+
 ## Roadmap
 
 - [ ] Support for more LLMs

backend/.env.sample

@@ -17,4 +17,9 @@ MILVUS_DB_PASSWORD={your-milvus-password}
 # -------------------------
 # QUERY CONFIG
 # -------------------------
-QUERY_TYPE=hybrid
+QUERY_TYPE=hybrid
+
+# -------------------------
+# UNSTRUCTURED CONFIG
+# -------------------------
+UNSTRUCTURED_API_KEY={your-unstructured-api-key}

backend/CHANGELOG.md

@@ -5,18 +5,31 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [Unreleased]
+## [0.1.1] - 2024-10-08
 
 ### Added
 
 - Added git workflows
 - Added issue templates
+- Integrated Unstructured API for enhanced document processing
+- Optional dependency groups in pyproject.toml for flexible installation
+- New `unstructured_loader` function for processing documents with Unstructured API
+- Error handling for Unstructured API import and usage
+
 
 ### Changed
 
+- Updated `upload_document` function to use Unstructured API when available
+- Modified project structure to support optional Unstructured integration
+- Updated installation instructions in README to reflect new dependency options
+
+
+### Fixed
+
 - Fixed issues for mypy, flake8, isort, black
+- Improved error handling in document processing pipeline
 
-## [v0.1.0]
+## [0.1.0]
 
 ### Added
 

backend/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 name = "knowledge-table-api"
 authors = [{name = "Foo Bar"}]
 description = "A Python API for the Knowledge Table service"
-keywords = ["knowlege", "api"]
+keywords = ["knowledge", "api"]
 classifiers = ["Programming Language :: Python :: 3"]
 requires-python = ">=3.10"
 dependencies = [
@@ -93,15 +93,20 @@ dependencies = [
     "typing_extensions==4.12.2",
     "tzdata==2024.1",
     "ujson==5.10.0",
+    "unstructured",
     "urllib3==2.2.3",
     "uuid==1.30",
     "uvicorn==0.30.6",
-    "whyhow==0.1.7",
+    "whyhow",
     "yarl==1.11.1",
 ]
 dynamic = ["version"]
 
 [project.optional-dependencies]
+unstructured = [
+    "unstructured[all-docs]",
+]
+
 dev = [
     "bandit[toml]",
     "black",
@@ -117,6 +122,22 @@ dev = [
     "pytest-html",
 ]
 
+full = [
+    "unstructured[all-docs]",
+    "bandit[toml]",
+    "black",
+    "flake8",
+    "flake8-docstrings",
+    "isort",
+    "mypy",
+    "pydocstyle[toml]",
+    "pytest-asyncio",
+    "pytest-cov",
+    "pytest-httpx",
+    "pytest",
+    "pytest-html",
+]
+
 [project.scripts]
 knowledge-table-locate = "knowledge_table_api.main:locate"
 

backend/src/knowledge_table_api/__init__.py

@@ -1,3 +1,3 @@
 """Knowledge Table API package."""
 
-__version__ = "v0.1.0"
+__version__ = "v0.1.1"

backend/src/knowledge_table_api/routers/document.py

@@ -1,6 +1,8 @@
 """Document router."""
 
-from fastapi import APIRouter, File, UploadFile
+from typing import Dict
+
+from fastapi import APIRouter, File, HTTPException, UploadFile, status
 
 from knowledge_table_api.models.document import Document
 from knowledge_table_api.services.document import upload_document
@@ -9,35 +11,91 @@
 router = APIRouter(tags=["Document"], prefix="/document")
 
 
-@router.post("", response_model=Document)
+@router.post("", response_model=Document, status_code=status.HTTP_201_CREATED)
 async def upload_document_endpoint(
     file: UploadFile = File(...),
-) -> Document | dict[str, str]:
-    """Upload a document and process it."""
+) -> Document:
+    """
+    Upload a document and process it.
+
+    Parameters
+    ----------
+    file : UploadFile
+        The file to be uploaded and processed.
+
+    Returns
+    -------
+    Document
+        The processed document information.
+
+    Raises
+    ------
+    HTTPException
+        If the file name is missing or if an error occurs during processing.
+    """
     if file.filename is None:
-        return {"message": "File name is missing"}
-    document_id = await upload_document(
-        file.content_type, file.filename, await file.read()
-    )
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail="File name is missing",
+        )
+
+    try:
+        document_id = await upload_document(
+            file.content_type, file.filename, await file.read()
+        )
+    except ValueError as ve:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST, detail=str(ve)
+        )
+    except Exception as e:
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e)
+        )
+
     if document_id is None:
-        return {"message": "An error occurred while processing the document"}
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail="An error occurred while processing the document",
+        )
 
+    # TODO: Fetch actual document details from a database
     document = Document(
         id=document_id,
         name=file.filename,
-        author="author_name",
-        tag="document_tag",
-        page_count=10,
+        author="author_name",  # TODO: Determine this dynamically
+        tag="document_tag",  # TODO: Determine this dynamically
+        page_count=10,  # TODO: Determine this dynamically
     )
     return document
 
 
-@router.delete("/{document_id}", response_model=dict)
-async def delete_document_endpoint(document_id: str) -> dict[str, str]:
-    """Delete a document."""
-    delete_document_response = await delete_document(document_id)
+@router.delete("/{document_id}", response_model=Dict[str, str])
+async def delete_document_endpoint(document_id: str) -> Dict[str, str]:
+    """
+    Delete a document.
+
+    Parameters
+    ----------
+    document_id : str
+        The ID of the document to be deleted.
+
+    Returns
+    -------
+    Dict[str, str]
+        A dictionary containing the deletion status and message.
+
+    Raises
+    ------
+    HTTPException
+        If an error occurs during the deletion process.
+    """
+    try:
+        delete_document_response = await delete_document(document_id)
+    except Exception as e:
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e)
+        )
 
-    # Delete the document
     return {
         "id": document_id,
         "status": delete_document_response["status"],