ci: run integration test on more python version (#2400) #2441
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Integration Tests | |
| on: | |
| push: | |
| branches: [ main ] | |
| pull_request: | |
| branches: [ main ] | |
| paths: | |
| - 'llama_stack/**' | |
| - 'tests/integration/**' | |
| - 'uv.lock' | |
| - 'pyproject.toml' | |
| - 'requirements.txt' | |
| - '.github/workflows/integration-tests.yml' # This workflow | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| test-matrix: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| # Listing tests manually since some of them currently fail | |
| # TODO: generate matrix list from tests/integration when fixed | |
| test-type: [agents, inference, datasets, inspect, scoring, post_training, providers, tool_runtime] | |
| client-type: [library, http] | |
| python-version: ["3.10", "3.11", "3.12"] | |
| fail-fast: false # we want to run all tests regardless of failure | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| - name: Install dependencies | |
| uses: ./.github/actions/setup-runner | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Setup ollama | |
| uses: ./.github/actions/setup-ollama | |
| - name: Build Llama Stack | |
| run: | | |
| uv run llama stack build --template ollama --image-type venv | |
| - name: Start Llama Stack server in background | |
| if: matrix.client-type == 'http' | |
| env: | |
| INFERENCE_MODEL: "meta-llama/Llama-3.2-3B-Instruct" | |
| run: | | |
| LLAMA_STACK_LOG_FILE=server.log nohup uv run llama stack run ./llama_stack/templates/ollama/run.yaml --image-type venv & | |
| - name: Wait for Llama Stack server to be ready | |
| if: matrix.client-type == 'http' | |
| run: | | |
| echo "Waiting for Llama Stack server..." | |
| for i in {1..30}; do | |
| if curl -s http://localhost:8321/v1/health | grep -q "OK"; then | |
| echo "Llama Stack server is up!" | |
| exit 0 | |
| fi | |
| sleep 1 | |
| done | |
| echo "Llama Stack server failed to start" | |
| cat server.log | |
| exit 1 | |
| - name: Verify Ollama status is OK | |
| if: matrix.client-type == 'http' | |
| run: | | |
| echo "Verifying Ollama status..." | |
| ollama_status=$(curl -s -L http://127.0.0.1:8321/v1/providers/ollama|jq --raw-output .health.status) | |
| echo "Ollama status: $ollama_status" | |
| if [ "$ollama_status" != "OK" ]; then | |
| echo "Ollama health check failed" | |
| exit 1 | |
| fi | |
| - name: Check Storage and Memory Available Before Tests | |
| if: ${{ always() }} | |
| run: | | |
| free -h | |
| df -h | |
| - name: Run Integration Tests | |
| env: | |
| INFERENCE_MODEL: "meta-llama/Llama-3.2-3B-Instruct" | |
| run: | | |
| if [ "${{ matrix.client-type }}" == "library" ]; then | |
| stack_config="ollama" | |
| else | |
| stack_config="http://localhost:8321" | |
| fi | |
| uv run pytest -s -v tests/integration/${{ matrix.test-type }} --stack-config=${stack_config} \ | |
| -k "not(builtin_tool or safety_with_image or code_interpreter or test_rag)" \ | |
| --text-model="meta-llama/Llama-3.2-3B-Instruct" \ | |
| --embedding-model=all-MiniLM-L6-v2 | |
| - name: Check Storage and Memory Available After Tests | |
| if: ${{ always() }} | |
| run: | | |
| free -h | |
| df -h | |
| - name: Write ollama logs to file | |
| if: ${{ always() }} | |
| run: | | |
| sudo journalctl -u ollama.service > ollama.log | |
| - name: Upload all logs to artifacts | |
| if: ${{ always() }} | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 | |
| with: | |
| name: logs-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.client-type }}-${{ matrix.test-type }}-${{ matrix.python-version }} | |
| path: | | |
| *.log | |
| retention-days: 1 |