diff --git a/.github/workflows/build_docker.yml b/.github/workflows/build_docker.yml index b29ab2ed89..fbce968697 100644 --- a/.github/workflows/build_docker.yml +++ b/.github/workflows/build_docker.yml @@ -11,45 +11,68 @@ on: release: types: - published + workflow_dispatch: permissions: {} jobs: build: + name: Build image permissions: contents: read # to fetch code (actions/checkout) packages: write # to push docker image - - name: Build image + strategy: + matrix: + image: + - backend + - database + - rabbitmq runs-on: ubuntu-latest steps: - - name: Checkout main - uses: actions/checkout@v2 - - name: Run the build - run: | - set -ex - # use that here since the variable are not present before start, so can't be in env - export LOGIN=$GITHUB_REPOSITORY_OWNER + - name: Checkout repository + uses: actions/checkout@v4 - echo $PASSWORD | docker login $REGISTRY -u $LOGIN --password-stdin + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + id: setup-buildx - for i in docker/* ; do - CONTAINER=$(basename $i) - echo "Building $CONTAINER" - export IMAGE=$LOGIN/augur_$CONTAINER - DOCKERFILE=${i}/Dockerfile + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + if: github.event_name != 'pull_request' + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} - docker build . -f $DOCKERFILE --tag $REGISTRY/$IMAGE:latest - if [[ $GITHUB_EVENT_NAME == 'release' ]]; then - TAG=$(basename $GITHUB_REF) - docker tag $REGISTRY/$IMAGE:latest $REGISTRY/$IMAGE:$TAG - docker push $REGISTRY/$IMAGE:latest - docker push $REGISTRY/$IMAGE:$TAG - elif [[ $GITHUB_EVENT_NAME == 'push' ]]; then - docker tag $REGISTRY/$IMAGE:latest $REGISTRY/$IMAGE:devel-latest - docker push $REGISTRY/$IMAGE:devel-latest - fi - done + - name: Set container metadata + uses: docker/metadata-action@v5 + id: meta env: - REGISTRY: ghcr.io - PASSWORD: ${{ secrets.GITHUB_TOKEN }} + DOCKER_METADATA_ANNOTATIONS_LEVELS: index,manifest + with: + annotations: | + org.opencontainers.image.title=augur_${{ matrix.image}} + labels: | + org.opencontainers.image.title=augur_${{ matrix.image}} + images: ghcr.io/${{ github.repository_owner }}/augur_${{ matrix.image }} + # Pushes to the dev branch update the *:devel-latest tag + # Releases update the *:latest tag and the *: tag + tags: | + type=raw,value=devel-latest,enable=${{ (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.ref == 'refs/heads/dev' }} + type=raw,value=latest,enable=${{ github.event_name == 'release' }} + type=raw,value=${{ github.event.release.tag_name }},enable=${{ github.event_name == 'release' }} + + - name: Build and push + id: push + uses: docker/build-push-action@v6 + with: + annotations: ${{ steps.meta.outputs.annotations }} + context: . + file: ./docker/${{ matrix.image }}/Dockerfile + labels: ${{ steps.meta.outputs.labels }} + platforms: linux/amd64 + # Only push if we've tagged the image in the metadata step + push: ${{ github.event_name != 'pull_request' && steps.meta.outputs.tags != '' }} + tags: ${{ steps.meta.outputs.tags }} + cache-from: type=gha,scope=container-${{ matrix.image }} + cache-to: type=gha,scope=container-${{ matrix.image }},mode=max diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml deleted file mode 100644 index 3f19c12343..0000000000 --- a/.github/workflows/docker-image.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: Docker Image CI - -on: - push: - branches: [ "main" ] - pull_request: - branches: [ "main" ] - -jobs: - - build: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - - name: Build the Docker image - run: | - ls -l - docker build . --file docker/backend/Dockerfile --tag my-image-name:$(date +%s) diff --git a/.gitignore b/.gitignore index 2c29307851..f3039acfa3 100644 --- a/.gitignore +++ b/.gitignore @@ -198,3 +198,4 @@ sendgrid.env *sendgrid*.env ./sendgrid.env sendgrid.env +notebooks/topic_modeling/output/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6368d58bf5..6fed03731b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -92,7 +92,7 @@ git push origin master ### Augur - [Stable documentation (`main` branch)](https://oss-augur.readthedocs.io/en/main/) - [Nightly/developer build documentation (`dev` branch)](https://oss-augur.readthedocs.io/en/dev/) (warning: this is should be considered an unstable branch and should not be used for production) -- [Live Augur demo](http://zephyr.osshealth.io/) +- [Live Augur demo](https://ai.chaoss.io) ### CHAOSS - [Website](https://chaoss.community/) diff --git a/README.md b/README.md index 50db605109..33e41bcf15 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Augur NEW Release v0.85.1 +# Augur NEW Release v0.86.0 Augur is primarily a data engineering tool that makes it possible for data scientists to gather open source software community data - less data carpentry for everyone else! The primary way of looking at Augur data is through [8Knot](https://github.com/oss-aspen/8knot), a public instance of 8Knot is available [here](https://metrix.chaoss.io) - this is tied to a public instance of [Augur](https://ai.chaoss.io). @@ -11,7 +11,7 @@ We follow the [First Timers Only](https://www.firsttimersonly.com/) philosophy o ## NEW RELEASE ALERT! **If you want to jump right in, the updated docker, docker-compose and bare metal installation instructions are available [here](docs/new-install.md)**. -Augur is now releasing a dramatically improved new version to the ```main``` branch. It is also available [here](https://github.com/chaoss/augur/releases/tag/v0.85.1). +Augur is now releasing a dramatically improved new version to the ```main``` branch. It is also available [here](https://github.com/chaoss/augur/releases/tag/v0.86.0). - The `main` branch is a stable version of our new architecture, which features: diff --git a/Vagrantfile b/Vagrantfile deleted file mode 100644 index 95de27ce02..0000000000 --- a/Vagrantfile +++ /dev/null @@ -1,108 +0,0 @@ -$script = <<-'SCRIPT' -set -euxo pipefail - -sudo apt-get -y update -sudo apt-get -y install --no-install-recommends \ - build-essential zlib1g-dev libncurses5-dev libgdbm-dev libnss3-dev libssl-dev libsqlite3-dev libreadline-dev libffi-dev curl libbz2-dev \ - git gcc gfortran \ - python3 python3-pip python3.8-venv \ - postgresql postgresql-contrib \ - libomp-dev \ - golang libgomp1 -sudo pg_ctlcluster 12 main start - -go get -u github.com/boyter/scc/ - -# # install Go -# installGo() ( -# cd "$(mktemp -d)" -# wget https://golang.org/dl/go1.16.5.linux-amd64.tar.gz -# rm -rf /usr/local/go && tar -C /usr/local -xzf go1.16.5.linux-amd64.tar.gz -# ) -# sudo installGo -# export PATH=$PATH:/usr/local/go/bin - - -########################################################################################## -# see: https://oss-augur.readthedocs.io/en/master/getting-started/database.html -cat < /tmp/init.psql -CREATE DATABASE augur; -CREATE USER augur WITH ENCRYPTED PASSWORD 'password'; -GRANT ALL PRIVILEGES ON DATABASE augur TO augur; -EOF -sudo -u postgres psql -U postgres -f /tmp/init.psql - - -########################################################################################## -# see: https://oss-augur.readthedocs.io/en/master/getting-started/installation.html - -mkdir -p "$HOME/augur/" "$HOME/augur/logs/" "$HOME/augur/repos/" -cat < "$HOME/augur/config.json" -{ - "Database": { - "host": "localhost", - "password": "password" - }, - "Server": { - "host": "0.0.0.0" - }, - "Logging": { - "logs_directory": "$HOME/augur/logs/", - "log_level": "INFO", - "verbose": 0, - "quiet": 0, - "debug": 1 - }, - "Workers": { - "facade_worker": { - "repo_directory": "$HOME/augur/repos/", - "switch": 1 - }, - "github_worker": { - "switch": 1 - }, - "insight_worker": { - "switch": 1 - }, - "linux_badge_worker": { - "switch": 1 - }, - "pull_request_worker": { - "switch": 1 - }, - "repo_info_worker": { - "switch": 1 - }, - "release_worker": { - "switch": 1 - } - } -} -EOF - - -python3 -m venv $HOME/.virtualenvs/augur_env -source $HOME/.virtualenvs/augur_env/bin/activate -pip install wheel - -cd /vagrant -python setup.py bdist_wheel -make clean -make install-dev - -augur config init --rc-config-file "$HOME/config.json" -augur db create-schema -augur backend start" - -SCRIPT - -Vagrant.configure("2") do |config| - config.vm.box = "ubuntu/focal64" - - config.vm.provider "virtualbox" do |v| - v.memory = 20480 - v.cpus = 4 - end - - config.vm.provision "shell", privileged: false, inline: $script -end diff --git a/augur/application/db/data_parse.py b/augur/application/db/data_parse.py index 71bde0aa21..6f20fa35a7 100644 --- a/augur/application/db/data_parse.py +++ b/augur/application/db/data_parse.py @@ -291,7 +291,7 @@ def extract_pr_event_data(event: dict, pr_id: int, gh_src_id: int, platform_id: 'action': event['event'], 'action_commit_hash': None, 'created_at': event['created_at'], - 'issue_event_src_id': gh_src_id, + 'issue_event_src_id': event["id"], 'node_id': event['node_id'], 'node_url': event['url'], 'tool_source': tool_source, diff --git a/augur/application/schema/alembic/versions/31_update_pr_events_unique.py b/augur/application/schema/alembic/versions/31_update_pr_events_unique.py new file mode 100644 index 0000000000..b55b60a09a --- /dev/null +++ b/augur/application/schema/alembic/versions/31_update_pr_events_unique.py @@ -0,0 +1,83 @@ +"""Update pr events unique + +Revision ID: 31 +Revises: 30 +Create Date: 2025-03-08 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy import text +from augur.application.db import create_database_engine, get_database_string + + +# revision identifiers, used by Alembic. +revision = '31' +down_revision = '30' +branch_labels = None +depends_on = None + + + # conn = op.get_bind() + # conn.execute(text(""" + # UPDATE pull_request_events + # SET issue_event_src_id = substring(node_url FROM '.*/([0-9]+)$')::BIGINT; + # """)) + + +def upgrade(): + + connection_string = get_database_string() + engine = create_database_engine(connection_string) + + with engine.connect() as conn: + + result = conn.execute(text("SELECT COUNT(*) FROM pull_request_events WHERE issue_event_src_id=pr_platform_event_id")) + total_rows = result.scalar() + if total_rows != 0: + print(f"Rows needing updated: {total_rows}") + print(f"0.0% complete") + total_updated = 0 + + while True: + result = conn.execute(text(""" + WITH cte AS ( + SELECT pr_event_id + FROM pull_request_events + WHERE issue_event_src_id=pr_platform_event_id + LIMIT 250000 + ) + UPDATE pull_request_events + SET issue_event_src_id = substring(node_url FROM '.*/([0-9]+)$')::BIGINT + FROM cte + WHERE pull_request_events.pr_event_id = cte.pr_event_id + RETURNING 1; + """)) + + conn.commit() + + rows_updated = result.rowcount + total_updated += rows_updated + + if rows_updated == 0: + print(f"Update complete") + break + + percentage_updated = (total_updated / total_rows) * 100 + + print(f"{percentage_updated:.1f}% complete ({total_rows-total_updated} rows left)") + + print("Creating (repo_id, issue_event_src_id) index") + op.create_unique_constraint('pr_events_repo_id_event_src_id_unique', 'pull_request_events', ['repo_id', 'issue_event_src_id'], schema='augur_data') + + +def downgrade(): + op.drop_constraint('pr_events_repo_id_event_src_id_unique', 'pull_request_events', schema='augur_data', type_='unique') + + print("Please run in background. This downgrade will take a very *very* long time") + conn = op.get_bind() + conn.execute(text(""" + UPDATE pull_request_events + SET issue_event_src_id = pr_platform_event_id + WHERE issue_event_src_id <> pr_platform_event_id; + """)) \ No newline at end of file diff --git a/augur/tasks/data_analysis/clustering_worker/setup.py b/augur/tasks/data_analysis/clustering_worker/setup.py index a197b21568..bd591554c3 100644 --- a/augur/tasks/data_analysis/clustering_worker/setup.py +++ b/augur/tasks/data_analysis/clustering_worker/setup.py @@ -26,7 +26,7 @@ def read(filename): 'Flask-Login==0.5.0', 'Flask-WTF==1.0.0', 'requests==2.32.0', - 'psycopg2-binary==2.9.3', + 'psycopg2-binary==2.9.9', #'sklearn==0.0.0', 'scikit-learn==1.5.0', 'numpy==1.26.0', diff --git a/augur/tasks/data_analysis/contributor_breadth_worker/setup.py b/augur/tasks/data_analysis/contributor_breadth_worker/setup.py index 805edfb36b..70a4cd6312 100644 --- a/augur/tasks/data_analysis/contributor_breadth_worker/setup.py +++ b/augur/tasks/data_analysis/contributor_breadth_worker/setup.py @@ -27,7 +27,7 @@ def read(filename): 'Flask-Login==0.5.0', 'Flask-WTF==1.0.0', 'requests==2.32.0', - 'psycopg2-binary==2.9.3' + 'psycopg2-binary==2.9.9' ], entry_points={ 'console_scripts': [ diff --git a/augur/tasks/data_analysis/discourse_analysis/setup.py b/augur/tasks/data_analysis/discourse_analysis/setup.py index ca936a6000..67cb4dd7e7 100644 --- a/augur/tasks/data_analysis/discourse_analysis/setup.py +++ b/augur/tasks/data_analysis/discourse_analysis/setup.py @@ -26,7 +26,7 @@ def read(filename): 'Flask-Login==0.5.0', 'Flask-WTF==1.0.0', 'requests==2.32.0', - 'psycopg2-binary==2.9.3', + 'psycopg2-binary==2.9.9', 'click==8.0.3', 'scipy>=1.10.0', 'nltk==3.6.6', diff --git a/augur/tasks/data_analysis/insight_worker/setup.py b/augur/tasks/data_analysis/insight_worker/setup.py index 92d663e3ae..b28b4bf5ab 100644 --- a/augur/tasks/data_analysis/insight_worker/setup.py +++ b/augur/tasks/data_analysis/insight_worker/setup.py @@ -27,7 +27,7 @@ def read(filename): 'Flask-Login==0.5.0', 'Flask-WTF==1.0.0', 'requests==2.32.0', - 'psycopg2-binary==2.9.3', + 'psycopg2-binary==2.9.9', 'click==8.0.3', 'scipy>=1.10.0', 'sklearn==0.0', diff --git a/augur/tasks/data_analysis/message_insights/setup.py b/augur/tasks/data_analysis/message_insights/setup.py index 2f86701619..bd136fdfec 100644 --- a/augur/tasks/data_analysis/message_insights/setup.py +++ b/augur/tasks/data_analysis/message_insights/setup.py @@ -28,7 +28,7 @@ def read(filename): 'Flask-Login==0.5.0', 'Flask-WTF==1.0.0', 'requests==2.32.0', - 'psycopg2-binary==2.9.3', + 'psycopg2-binary==2.9.9', 'click==8.0.3', 'scipy>=1.10.0', 'scikit-learn==1.5.0', #0.24.2', diff --git a/augur/tasks/data_analysis/pull_request_analysis_worker/setup.py b/augur/tasks/data_analysis/pull_request_analysis_worker/setup.py index 63ccbec1de..391b368f32 100644 --- a/augur/tasks/data_analysis/pull_request_analysis_worker/setup.py +++ b/augur/tasks/data_analysis/pull_request_analysis_worker/setup.py @@ -26,7 +26,7 @@ def read(filename): 'Flask-Login==0.5.0', 'Flask-WTF==1.0.0', 'requests==2.32.0', - 'psycopg2-binary==2.9.3', + 'psycopg2-binary==2.9.9', 'sklearn==0.0', 'nltk==3.6.6', 'numpy==1.26.0', diff --git a/augur/tasks/git/util/facade_worker/facade_worker/analyzecommit.py b/augur/tasks/git/util/facade_worker/facade_worker/analyzecommit.py index 1e2e04ac12..5f8bcd5772 100644 --- a/augur/tasks/git/util/facade_worker/facade_worker/analyzecommit.py +++ b/augur/tasks/git/util/facade_worker/facade_worker/analyzecommit.py @@ -32,6 +32,7 @@ import sqlalchemy as s from augur.application.db.lib import execute_sql, fetchall_data_from_sql_text +from augur.tasks.init import get_rabbitmq_conn_string def analyze_commit(logger, repo_id, repo_loc, commit): diff --git a/augur/tasks/git/util/facade_worker/facade_worker/utilitymethods.py b/augur/tasks/git/util/facade_worker/facade_worker/utilitymethods.py index 40f3a29e0e..caae6c02ba 100644 --- a/augur/tasks/git/util/facade_worker/facade_worker/utilitymethods.py +++ b/augur/tasks/git/util/facade_worker/facade_worker/utilitymethods.py @@ -26,7 +26,7 @@ # repos. It also rebuilds analysis data, checks any changed affiliations and # aliases, and caches data for display. import subprocess -from subprocess import check_output +from subprocess import check_output, CalledProcessError import os import sqlalchemy as s from augur.application.db.models import * @@ -154,7 +154,15 @@ def get_repo_commit_count(logger, facade_helper, repo_git): if count_branches(repo_loc) == 0: return 0 - check_commit_count_cmd = check_output(["git", "--git-dir", repo_loc, "rev-list", "--count", "HEAD"]) + try: + check_commit_count_cmd = check_output( + ["git", "--git-dir", repo_loc, "rev-list", "--count", "HEAD"], + stderr=subprocess.PIPE) + except CalledProcessError as e: + logger.error(f"Ran into {e}: {e.output} {e.stderr} \n With return code {e.returncode}") + raise e + + commit_count = int(check_commit_count_cmd) return commit_count diff --git a/augur/tasks/git/util/facade_worker/setup.py b/augur/tasks/git/util/facade_worker/setup.py index e2a1af8b75..ef65f223a2 100644 --- a/augur/tasks/git/util/facade_worker/setup.py +++ b/augur/tasks/git/util/facade_worker/setup.py @@ -27,7 +27,7 @@ def read(filename): 'Flask-Login==0.5.0', 'Flask-WTF==1.0.0', 'requests==2.32.0', - 'psycopg2-binary==2.9.3', + 'psycopg2-binary==2.9.9', 'click==8.0.3', 'XlsxWriter==1.3.7' ], diff --git a/augur/tasks/github/events.py b/augur/tasks/github/events.py index 5c79691c26..654dc68c6a 100644 --- a/augur/tasks/github/events.py +++ b/augur/tasks/github/events.py @@ -78,7 +78,7 @@ def _insert_issue_events(self, events): bulk_insert_dicts(self._logger, events, IssueEvent, issue_event_natural_keys) def _insert_pr_events(self, events): - pr_event_natural_keys = ["node_id"] + pr_event_natural_keys = ["repo_id", "issue_event_src_id"] bulk_insert_dicts(self._logger, events, PullRequestEvent, pr_event_natural_keys) def _insert_contributors(self, contributors): diff --git a/augur/tasks/github/util/github_data_access.py b/augur/tasks/github/util/github_data_access.py index a648f990c6..0eda1c8462 100644 --- a/augur/tasks/github/util/github_data_access.py +++ b/augur/tasks/github/util/github_data_access.py @@ -5,6 +5,8 @@ from urllib.parse import urlparse, parse_qs, urlencode from keyman.KeyClient import KeyClient +GITHUB_RATELIMIT_REMAINING_CAP = 50 + class RatelimitException(Exception): @@ -110,6 +112,13 @@ def make_request(self, url, method="GET", timeout=100): response.raise_for_status() + try: + if "X-RateLimit-Remaining" in response.headers and int(response.headers["X-RateLimit-Remaining"]) < GITHUB_RATELIMIT_REMAINING_CAP: + raise RatelimitException(response) + except ValueError: + self.logger.warning(f"X-RateLimit-Remaining was not an integer. Value: {response.headers['X-RateLimit-Remaining']}") + + return response def make_request_with_retries(self, url, method="GET", timeout=100): @@ -148,7 +157,7 @@ def __handle_github_ratelimit_response(self, response): f'\n\n\n\nSleeping for {retry_after} seconds due to secondary rate limit issue.\n\n\n\n') time.sleep(retry_after) - elif "X-RateLimit-Remaining" in headers and int(headers["X-RateLimit-Remaining"]) == 0: + elif "X-RateLimit-Remaining" in headers and int(headers["X-RateLimit-Remaining"]) < GITHUB_RATELIMIT_REMAINING_CAP: current_epoch = int(time.time()) epoch_when_key_resets = int(headers["X-RateLimit-Reset"]) key_reset_time = epoch_when_key_resets - current_epoch diff --git a/docker-compose.yml b/docker-compose.yml index e077260ed7..81452bd09d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,14 +10,16 @@ services: - "POSTGRES_PASSWORD=${AUGUR_DB_PASSWORD:-augur}" - "PGDATA=/var/lib/postgresql/data/pgdata" ports: - - "127.0.0.1:${AUGUR_DB_PORT:-5432}:5432" + - "${AUGUR_DB_PORT:-5432}:5432" volumes: - augurpostgres:/var/lib/postgresql/data + networks: + - augur redis: image: "redis:alpine" - ports: - - 6379:6379 + networks: + - augur rabbitmq: image: augur-rabbitmq @@ -28,12 +30,8 @@ services: - RABBIT_MQ_DEFAULT_USER=${AUGUR_RABBITMQ_USERNAME:-augur} - RABBIT_MQ_DEFAULT_PASSWORD=${AUGUR_RABBITMQ_PASSWORD:-password123} - RABBIT_MQ_DEFAULT_VHOST=${AUGUR_RABBITMQ_VHOST:-augur_vhost} - # ports for amqp connections / management api - ports: - - 5671:5671 - - 5672:5672 - - 15671:15671 - - 15672:15672 + networks: + - augur augur: image: augur-new:latest @@ -45,8 +43,8 @@ services: restart: unless-stopped ports: - 5002:5000 - extra_hosts: - - "host.docker.internal:host-gateway" #Be able to ping services on the local machine + #extra_hosts: + # - "host.docker.internal:host-gateway" #Be able to ping services on the local machine environment: - "AUGUR_DB=postgresql+psycopg2://${AUGUR_DB_USER:-augur}:${AUGUR_DB_PASSWORD:-augur}@augur-db:5432/augur" - "AUGUR_DB_SCHEMA_BUILD=1" @@ -61,6 +59,30 @@ services: - augur-db - redis - rabbitmq + networks: + - augur + + # Flower is a UI that helps more easily monitor running tasks for celery workers. + # This wont show you every error from the entire augur system, but it should show most of them. + # uncomment the section below to use flower + #flower: + # image: augur-new:latest + # restart: unless-stopped + # command: + # [ "celery", "-A", "augur.tasks.init.celery_app.celery_app", "flower", "--max-tasks=1000000" ] + # ports: + # - 5555:5555 + # environment: + # - "AUGUR_DB=postgresql+psycopg2://${AUGUR_DB_USER:-augur}:${AUGUR_DB_PASSWORD:-augur}@augur-db:5432/augur" + # - REDIS_CONN_STRING=redis://redis:6379 + # - RABBITMQ_CONN_STRING=amqp://${AUGUR_RABBITMQ_USERNAME:-augur}:${AUGUR_RABBITMQ_PASSWORD:-password123}@rabbitmq:5672/${AUGUR_RABBITMQ_VHOST:-augur_vhost} + # depends_on: + # - augur + # - augur-db + # - redis + # - rabbitmq + # networks: + # - augur volumes: facade: @@ -68,4 +90,5 @@ volumes: augurpostgres: driver: local - +networks: + augur: \ No newline at end of file diff --git a/docker/backend/Dockerfile b/docker/backend/Dockerfile index df6e952b05..959d3952f5 100644 --- a/docker/backend/Dockerfile +++ b/docker/backend/Dockerfile @@ -32,7 +32,7 @@ RUN ./scripts/docker/install-golang-deps.sh FROM python:3.11-slim-bullseye LABEL maintainer="outdoors@acm.org" -LABEL version="0.85.1" +LABEL version="0.86.0" ENV DEBIAN_FRONTEND=noninteractive ENV PATH="/usr/bin/:/usr/local/bin:/usr/lib:${PATH}" @@ -120,8 +120,8 @@ RUN set -x \ RUN ./scripts/docker/install-workers-deps.sh -COPY --from=golang "/root/scc" "./scc" -COPY --from=golang "/root/scorecard/scorecard" "./scorecard" +COPY --from=golang "/root/scc" "/root/scc" +COPY --from=golang "/root/scorecard/scorecard" "/root/scorecard" # RUN ./scripts/install/workers.sh diff --git a/docker/database/Dockerfile b/docker/database/Dockerfile index a1a59b0c89..e5825ca7bf 100644 --- a/docker/database/Dockerfile +++ b/docker/database/Dockerfile @@ -2,7 +2,7 @@ FROM postgres:16 LABEL maintainer="outdoors@acm.org" -LABEL version="0.85.1" +LABEL version="0.86.0" ENV POSTGRES_DB "test" ENV POSTGRES_USER "augur" diff --git a/docker/rabbitmq/Dockerfile b/docker/rabbitmq/Dockerfile index b4ec1948ba..9b0b3c98c3 100644 --- a/docker/rabbitmq/Dockerfile +++ b/docker/rabbitmq/Dockerfile @@ -1,7 +1,7 @@ FROM rabbitmq:3.12-management-alpine LABEL maintainer="574/augur@simplelogin.com" -LABEL version="0.85.1" +LABEL version="0.86.0" ARG RABBIT_MQ_DEFAULT_USER=augur ARG RABBIT_MQ_DEFAULT_PASSWORD=password123 diff --git a/docs/dev-osx-install.md b/docs/dev-osx-install.md index c70bac7714..aa5b67b7fa 100644 --- a/docs/dev-osx-install.md +++ b/docs/dev-osx-install.md @@ -31,7 +31,7 @@ export PKG_CONFIG_PATH="/opt/homebrew/opt/openblas/lib/pkgconfig" ## Git Platform Requirements (Things to have setup prior to initiating installation.) 1. Obtain a GitHub Access Token: https://github.com/settings/tokens -2. Obtain a GitLab Access Token: https://gitlab.com/-/profile/personal_access_tokens +2. Obtain a GitLab Access Token: https://gitlab.com/-/user_settings/personal_access_tokens ### Fork and Clone Augur 1. Fork https://github.com/chaoss/augur diff --git a/docs/new-install-ubuntu-python-3.10.md b/docs/new-install-ubuntu-python-3.10.md index 3cd7a5a38a..6fa2ee78fc 100644 --- a/docs/new-install-ubuntu-python-3.10.md +++ b/docs/new-install-ubuntu-python-3.10.md @@ -5,7 +5,7 @@ We default to this version of Ubuntu for the moment because Augur does not yet s ## Git Platform Requirements (Things to have setup prior to initiating installation.) 1. Obtain a GitHub Access Token: https://github.com/settings/tokens -2. Obtain a GitLab Access Token: https://gitlab.com/-/profile/personal_access_tokens +2. Obtain a GitLab Access Token: https://gitlab.com/-/user_settings/personal_access_tokens ### Fork and Clone Augur 1. Fork https://github.com/chaoss/augur diff --git a/docs/new-install.md b/docs/new-install.md index 9ec8960857..017cb41322 100644 --- a/docs/new-install.md +++ b/docs/new-install.md @@ -5,7 +5,7 @@ We default to this version of Ubuntu for the moment because Augur does not yet s ## Git Platform Requirements (Things to have setup prior to initiating installation.) 1. Obtain a GitHub Access Token: https://github.com/settings/tokens -2. Obtain a GitLab Access Token: https://gitlab.com/-/profile/personal_access_tokens +2. Obtain a GitLab Access Token: https://gitlab.com/-/user_settings/personal_access_tokens ### Fork and Clone Augur 1. Fork https://github.com/chaoss/augur diff --git a/docs/source/getting-started/dev-osx-install.rst b/docs/source/getting-started/dev-osx-install.rst index 05285da207..05d16c29be 100644 --- a/docs/source/getting-started/dev-osx-install.rst +++ b/docs/source/getting-started/dev-osx-install.rst @@ -47,8 +47,7 @@ Git Platform Requirements (Things to have setup prior to initiating installation ---------------------------------------------------------------------------------- 1. Obtain a GitHub Access Token: https://github.com/settings/tokens -2. Obtain a GitLab Access Token: - https://gitlab.com/-/profile/personal_access_tokens +2. Obtain a GitLab Access Token: https://gitlab.com/-/user_settings/personal_access_tokens Fork and Clone Augur ~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/source/getting-started/installation.rst b/docs/source/getting-started/installation.rst index 5c2e2f62f7..4e367958c1 100644 --- a/docs/source/getting-started/installation.rst +++ b/docs/source/getting-started/installation.rst @@ -216,11 +216,6 @@ your installation of Python 3: on most systems, this is ``python3``, but yours m # run the install script $ make install -.. code-block:: bash - - # If you want to develop with Augur, use this command instead - $ make install-dev - If you think something went wrong, check the log files in ``logs/``. If you want to try again, you can use ``make clean`` to delete any build files before running ``make install`` again. MacOS users: diff --git a/docs/source/getting-started/new-install.rst b/docs/source/getting-started/new-install.rst index 3fd45f4a97..b10aa0b41e 100644 --- a/docs/source/getting-started/new-install.rst +++ b/docs/source/getting-started/new-install.rst @@ -9,8 +9,7 @@ Git Platform Requirements (Things to have setup prior to initiating installation ---------------------------------------------------------------------------------- 1. Obtain a GitHub Access Token: https://github.com/settings/tokens -2. Obtain a GitLab Access Token: - https://gitlab.com/-/profile/personal_access_tokens +2. Obtain a GitLab Access Token: https://gitlab.com/-/user_settings/personal_access_tokens Fork and Clone Augur ~~~~~~~~~~~~~~~~~~~~ diff --git a/gsoc-ideas.md b/gsoc-ideas.md index cea5dff355..3bc2748811 100644 --- a/gsoc-ideas.md +++ b/gsoc-ideas.md @@ -5,6 +5,34 @@ [Micro-tasks and place for questions](https://github.com/chaoss/augur/issues/1640) +## Microtasks + +For becoming familiar with Augur, you can start by reading some documentation. You can find useful information at in the links, below. Grimoirelab also has a set of installation instructions and documentation here: https://chaoss.github.io/grimoirelab-tutorial/ + +#### GSoC Students : +Once you're familiar with Augur, you can have a look at the following microtasks. + +Microtask 0: + Download and configure Augur, creating a dev environment using the general cautions noted here: +Augur + https://oss-augur.readthedocs.io/en/dev/getting-started/installation.html and the full documentation here: + https://oss-augur.readthedocs.io/en/dev/development-guide/toc.html +Grimoirelab + https://chaoss.github.io/grimoirelab-tutorial/ + +Microstask 1: + Work on any Augur or Grimoirelab Issue that's Open + +Microtask 2: + Identify new issues you encounter during installation. + +Microstask 3: + Explore data presently captured, develop an experimental visualization using tools of your choice. If Jupyter Notebooks against an Augur database/API endpoint collection, use https://github.com/chaoss/augur-community-reports for development. + +Microtask 4: + Anything you want to show us. Even if you find bugs in our documentation and want to issue a PR for those! + + This project will add GenSIM logic, and other capabilities to the Clustering Worker inside of Augur Software, and be extended into a generalized Open Source Software Conversational Topic Modeling Instrument. CHOASS/augur has several workers that store machine learning information derived from computational linguistic analysis of data in the `message` table. The message table includes messages from issue, pull request, pull request review, and email messages. They are related to their origin with bridge tables like `pull_request_message_ref`. The ML/CL workers are all run against all the messages, regardless of origin. diff --git a/gsoc/microtasks/INSTALL_ISSUES_MAC_M1.md b/gsoc/microtasks/INSTALL_ISSUES_MAC_M1.md new file mode 100644 index 0000000000..28215527a5 --- /dev/null +++ b/gsoc/microtasks/INSTALL_ISSUES_MAC_M1.md @@ -0,0 +1,147 @@ + +# Microtask 2: Installation Issues Encountered on macOS 14.4 (M1) + +This document outlines the issues I encountered while installing [CHAOSS Augur](https://github.com/chaoss/augur) on a **macOS 14.4 M1** machine using **Python 3.9.18** and **Augur v0.86.0**, along with the solutions I applied to resolve them. + +--- + +### Issue 1: pyenv under Rosetta 2 +**Problem:** +Using `pyenv` to install Python 3.8.x failed due to Rosetta architecture issues: +``` +Error: Cannot install under Rosetta 2 in ARM default prefix (/opt/homebrew)! +``` + +**Solution:** +Force Homebrew and pyenv to run natively on ARM: +```bash +arch -arm64 pyenv install 3.8.17 +``` +Or with full flags: +```bash +CFLAGS="-I$(brew --prefix zlib)/include -I$(brew --prefix bzip2)/include" \ +LDFLAGS="-L$(brew --prefix zlib)/lib -L$(brew --prefix bzip2)/lib" \ +PYTHON_CONFIGURE_OPTS="--enable-framework" \ +arch -arm64 pyenv install 3.8.17 +``` + +--- + +### Issue 2: Python 3.8 not compatible with macOS 14 +**Problem:** +Even after installing Python 3.8, some packages like `numpy==1.26.0` and `tensorflow` failed to compile or install properly. + +**Solution:** +Upgrade to Python 3.9: +```bash +arch -arm64 pyenv install 3.9.18 +pyenv global 3.9.18 +``` + +--- + +### Issue 3: `make install-dev` does not exist +**Problem:** +The official documentation suggests: +```bash +make install-dev +``` +But the Makefile doesn’t have this target. + +**Solution:** +Run the regular install command: +```bash +make install +``` + +--- + +### Issue 4: `.git-credentials` permission error +**Problem:** +During configuration, Augur attempts to create `.git-credentials` in `/`, which fails: +``` +touch: /.git-credentials: Read-only file system +``` + +**Solution:** +Create a Facade repo directory manually before running the install: +```bash +mkdir -p ~/augur/repos +``` +Then use the full path when prompted: +``` +Facade worker directory: /Users//augur/repos +``` + +--- + +### Issue 5: `h5py` version conflict with TensorFlow +**Problem:** +`augur` installs `h5py==3.10.0` but `tensorflow>=2.19.0` requires `h5py>=3.11.0`: +``` +tensorflow 2.19.0 requires h5py>=3.11.0, but you have h5py 3.10.0 +``` + +**Solution:** +Manually install the required version: +```bash +pip install 'h5py>=3.11.0' +``` +Warning: This conflicts with Augur’s `setup.py`. It may be necessary to fork and patch `requirements.txt`. + +--- + +### Issue 6: Invalid CLI commands in docs (`augur db init`) +**Problem:** +Documentation suggests running: +```bash +augur db init +``` +But this returns: +``` +Error: No such command 'init' +``` + +**Solution:** +Use the actual CLI options: +```bash +augur db init-database # if database doesn't exist +augur db create-schema # to create schema +augur db upgrade-db-version # to apply migrations +``` + +--- + +### Issue 7: NLTK download prompt +**Problem:** +During installation, Augur prompts: +``` +Would you like to install required NLTK word lists for machine learning workers? +``` + +**Solution:** +Accept prompt by typing `y`, or manually install later with: +```python +import nltk +nltk.download('all') +``` + +--- + +## Summary + +These issues are frequently encountered during Augur installation on Apple Silicon Macs. Resolving and documenting them helps improve onboarding and reduces developer friction. + +--- + +**System Info:** +- macOS 14.4 (M1) +- Python 3.9.18 (via pyenv) +- Augur v0.86.0 +- RabbitMQ, PostgreSQL, Redis installed via Homebrew +- Virtualenv: `augur39` + +--- + +Maintained by [Xiaoha](mailto:placeholder@email.com) +``` \ No newline at end of file diff --git a/metadata.py b/metadata.py index 184caa4f70..740db52f3d 100644 --- a/metadata.py +++ b/metadata.py @@ -5,8 +5,8 @@ __short_description__ = "Python 3 package for free/libre and open-source software community metrics, models & data collection" -__version__ = "0.85.1" -__release__ = "v0.85.1 (Eurovision: Fire Saga)" +__version__ = "0.86.0" +__release__ = "v0.86.0 (Pod People)" __license__ = "MIT" __copyright__ = "University of Missouri, University of Nebraska-Omaha, CHAOSS, Sean Goggins, Brian Warner & Augurlabs 2025" diff --git a/notebooks/topic_modeling/assets/hdp_topic_vis_preview.png b/notebooks/topic_modeling/assets/hdp_topic_vis_preview.png new file mode 100644 index 0000000000..98e9b63266 Binary files /dev/null and b/notebooks/topic_modeling/assets/hdp_topic_vis_preview.png differ diff --git a/notebooks/topic_modeling/augur_topic_modeling_demo.ipynb b/notebooks/topic_modeling/augur_topic_modeling_demo.ipynb new file mode 100644 index 0000000000..6f9cbfb900 --- /dev/null +++ b/notebooks/topic_modeling/augur_topic_modeling_demo.ipynb @@ -0,0 +1,358 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "01fb09d6-4f45-4339-b31a-2d26f8e92383", + "metadata": {}, + "outputs": [], + "source": [ + "# Topic Modeling Demo for CHAOSS Augur Message Data\n", + "\n", + " # 1. Environment and Dependencies\n", + "import pandas as pd\n", + "import psycopg2\n", + "import spacy\n", + "import gensim\n", + "import gensim.corpora as corpora\n", + "from nltk.corpus import stopwords\n", + "from gensim.models import HdpModel\n", + "import pyLDAvis.gensim_models as gensimvis\n", + "import pyLDAvis\n", + "import matplotlib.pyplot as plt\n", + "import warnings\n", + "warnings.filterwarnings(\"ignore\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bfdd2953-a690-426a-bfe5-ad030db77634", + "metadata": {}, + "outputs": [], + "source": [ + "# 2. Database connection configuration (localhost, user=augur, default port)\n", + "conn = psycopg2.connect(\n", + " dbname=\"augur\",\n", + " user=\"augur\",\n", + " password=\"augur\",\n", + " host=\"localhost\",\n", + " port=\"5432\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cc636a2e-6df3-42c8-a6a4-a90eaf4d149a", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Total messages retrieved: 43\n" + ] + } + ], + "source": [ + "# 3. Data extraction: Get messages from message table with repo_id=24441\n", + "sql = \"\"\"\n", + "SELECT msg_id, msg_text, msg_timestamp \n", + "FROM augur_data.message\n", + "WHERE repo_id = 24441\n", + " AND LENGTH(msg_text) > 30\n", + " AND msg_timestamp BETWEEN '2021-08-03' AND '2023-12-31'\n", + "\"\"\"\n", + "\n", + "messages_df = pd.read_sql_query(sql, conn)\n", + "print(f\"Total messages retrieved: {len(messages_df)}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a05c726e-79d9-42b9-a50d-bc7a55644652", + "metadata": {}, + "outputs": [], + "source": [ + "# 4. Text preprocessing: spaCy tokenization + stopword removal\n", + "txts = messages_df['msg_text'].astype(str).tolist()\n", + "nlp = spacy.load(\"en_core_web_sm\")\n", + "stop_words = set(stopwords.words(\"english\"))\n", + "\n", + "# Basic preprocessing function\n", + "def preprocess(text):\n", + " doc = nlp(text.lower())\n", + " return [token.lemma_ for token in doc if token.is_alpha and token.lemma_ not in stop_words and len(token) > 2]\n", + "\n", + "texts_tokenized = [preprocess(t) for t in txts]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b2754db5-ea32-45c9-8e9f-045f61593e39", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Training HDP model...\n" + ] + } + ], + "source": [ + "# 5. Build Gensim dictionary and corpus\n", + "id2word = corpora.Dictionary(texts_tokenized)\n", + "corpus = [id2word.doc2bow(text) for text in texts_tokenized]\n", + "\n", + "# 6. Train HDP model to automatically infer the number of topics\n", + "print(\"\\nTraining HDP model...\")\n", + "hdp_model = HdpModel(corpus=corpus, id2word=id2word)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54da5d8b-a0cf-43a1-b964-c6986b23b496", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Top 10 topics and keywords:\n", + "\n", + "Topic 0\n", + "['first', 'operate', 'pull', 'blob', 'review', 'locally', 'write', 'full', 'code', 'approval']\n", + "\n", + "Topic 1\n", + "['retweet', 'event', 'detail', 'please', 'guide', 'new', 'check', 'complete', 'octo', 'solution']\n", + "\n", + "Topic 2\n", + "['approve', 'trigger', 'open', 'btw', 'link', 'share', 'add', 'approval', 'issue', 'tweet']\n", + "\n", + "Topic 3\n", + "['michaelclifford', 'assign', 'inactivity', 'seem', 'approve', 'full', 'wonder', 'since', 'documentation', 'requirement']\n", + "\n", + "Topic 4\n", + "['great', 'additional', 'well', 'add', 'review', 'ready', 'thank', 'open', 'requirement', 'lifecycle']\n", + "\n", + "Topic 5\n", + "['qualify', 'review', 'inactivity', 'full', 'opinion', 'proof', 'hence', 'process', 'oindrillac', 'perfect']\n", + "\n", + "Topic 6\n", + "['plan', 'bot', 'detail', 'complete', 'quote', 'another', 'lifecycle', 'add', 'blob', 'approvalnotifier']\n", + "\n", + "Topic 7\n", + "['version', 'complete', 'bypass', 'well', 'safe', 'please', 'stale', 'operate', 'approval', 'remove']\n", + "\n", + "Topic 8\n", + "['cancel', 'michaelclifford', 'solution', 'idea', 'prow', 'describe', 'manually', 'also', 'quick', 'new']\n", + "\n", + "Topic 9\n", + "['version', 'count', 'since', 'contributor', 'accept', 'quote', 'community', 'shortend', 'michaelclifford', 'plan']\n" + ] + } + ], + "source": [ + "# 7. Print the top topics and keywords\n", + "hdp_topics = hdp_model.show_topics(num_topics=10, num_words=10, formatted=False)\n", + "print(\"\\nTop 10 topics and keywords:\")\n", + "for topic_id, words in hdp_topics:\n", + " print(f\"\\nTopic {topic_id}\")\n", + " print([word for word, _ in words])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ac9903be-0ad1-418e-b36c-1cb63979b116", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Preparing visualization...\n", + "✔️ Visualization saved as hdp_topics_augur.html\n" + ] + } + ], + "source": [ + "# 8. Visualize using pyLDAvis\n", + "print(\"\\nPreparing visualization...\")\n", + "vis_data = gensimvis.prepare(hdp_model, corpus, id2word)\n", + "pyLDAvis.display(vis_data)\n", + "\n", + "# Optionally save as HTML for later viewing\n", + "pyLDAvis.save_html(vis_data, \"notebooks/topic_modeling/output/hdp_topics_augur.html\")\n", + "print(\"✔️ Visualization saved as hdp_topics_augur.html\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5a51dbe8-8340-44c6-877c-5e854dfcc575", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
dominant_topicmsg_timestampmsg_text
002021-11-04 13:33:03[APPROVALNOTIFIER] This PR is **APPROVED**\\n\\n...
102022-04-22 20:18:18[APPROVALNOTIFIER] This PR is **APPROVED**\\n\\n...
212022-03-11 20:12:43/lgtm 👍 \\r\\n/hold\\r\\n@schwesig do you want to ...
322021-10-11 15:28:44> What about we also link to the https://githu...
422022-01-28 14:13:27for documentation purposes - https://github.co...
542022-03-11 22:43:14@oindrillac great tweet, thank you for making ...
6152021-10-16 15:01:08Issues go stale after 90d of inactivity.\\nMark...
7152021-10-26 18:04:43Issues go stale after 90d of inactivity.\\nMark...
81282022-03-11 22:07:33@oindrillac Please change and do what is neede...
\n", + "
" + ], + "text/plain": [ + " dominant_topic msg_timestamp \\\n", + "0 0 2021-11-04 13:33:03 \n", + "1 0 2022-04-22 20:18:18 \n", + "2 1 2022-03-11 20:12:43 \n", + "3 2 2021-10-11 15:28:44 \n", + "4 2 2022-01-28 14:13:27 \n", + "5 4 2022-03-11 22:43:14 \n", + "6 15 2021-10-16 15:01:08 \n", + "7 15 2021-10-26 18:04:43 \n", + "8 128 2022-03-11 22:07:33 \n", + "\n", + " msg_text \n", + "0 [APPROVALNOTIFIER] This PR is **APPROVED**\\n\\n... \n", + "1 [APPROVALNOTIFIER] This PR is **APPROVED**\\n\\n... \n", + "2 /lgtm 👍 \\r\\n/hold\\r\\n@schwesig do you want to ... \n", + "3 > What about we also link to the https://githu... \n", + "4 for documentation purposes - https://github.co... \n", + "5 @oindrillac great tweet, thank you for making ... \n", + "6 Issues go stale after 90d of inactivity.\\nMark... \n", + "7 Issues go stale after 90d of inactivity.\\nMark... \n", + "8 @oindrillac Please change and do what is neede... " + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Assign dominant topic to each message\n", + "doc_topics = [max(hdp_model[doc], key=lambda x: x[1])[0] if hdp_model[doc] else -1 for doc in corpus]\n", + "messages_df['dominant_topic'] = doc_topics\n", + "\n", + "# Show 2 representative comments for each topic\n", + "representative_samples = messages_df.groupby('dominant_topic').apply(lambda x: x.sample(min(2, len(x)))).reset_index(drop=True)\n", + "representative_samples[['dominant_topic', 'msg_timestamp', 'msg_text']]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8f4c9672-801f-4342-a46c-4db87d9d0782", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3.9 (Augur)", + "language": "python", + "name": "augur39" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.18" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/podman-compose.yml b/podman-compose.yml deleted file mode 100644 index 9970f81325..0000000000 --- a/podman-compose.yml +++ /dev/null @@ -1,68 +0,0 @@ -#SPDX-License-Identifier: MIT -version: '3' -services: - augur-db: - image: postgres:14 - restart: unless-stopped - environment: - - "POSTGRES_DB=augur" - - "POSTGRES_USER=${AUGUR_DB_USER:-augur}" - - "POSTGRES_PASSWORD=${AUGUR_DB_PASSWORD:-augur}" - - "PGDATA=/var/lib/postgresql/data/pgdata" - ports: - - "${AUGUR_DB_PORT:-5432}:5432" - volumes: - - augurpostgres:/var/lib/postgresql/data - - redis: - image: "redis:alpine" - ports: - - 6379:6379 - - rabbitmq: - image: augur-rabbitmq - build: - context: . - dockerfile: ./docker/rabbitmq/Dockerfile - args: - - RABBIT_MQ_DEFAULT_USER=${AUGUR_RABBITMQ_USERNAME:-augur} - - RABBIT_MQ_DEFAULT_PASSWORD=${AUGUR_RABBITMQ_PASSWORD:-password123} - - RABBIT_MQ_DEFAULT_VHOST=${AUGUR_RABBITMQ_VHOST:-augur_vhost} - # ports for amqp connections / management api - ports: - - 5671:5671 - - 5672:5672 - - 15671:15671 - - 15672:15672 - - augur: - image: augur-new:latest - build: - context: . - dockerfile: ./docker/backend/Dockerfile - volumes: - - facade:/augur/facade - restart: unless-stopped - ports: - - 5002:5000 - environment: - - "AUGUR_DB=postgresql+psycopg2://${AUGUR_DB_USER:-augur}:${AUGUR_DB_PASSWORD:-augur}@augur-db:5432/augur" - - "AUGUR_DB_SCHEMA_BUILD=1" - - "AUGUR_GITHUB_API_KEY=${AUGUR_GITHUB_API_KEY}" - - "AUGUR_GITLAB_API_KEY=${AUGUR_GITLAB_API_KEY}" - - "AUGUR_GITHUB_USERNAME=${AUGUR_GITHUB_USERNAME}" - - "AUGUR_GITLAB_USERNAME=${AUGUR_GITLAB_USERNAME}" - - REDIS_CONN_STRING=redis://redis:6379 - - RABBITMQ_CONN_STRING=amqp://${AUGUR_RABBITMQ_USERNAME:-augur}:${AUGUR_RABBITMQ_PASSWORD:-password123}@rabbitmq:5672/${AUGUR_RABBITMQ_VHOST:-augur_vhost} - depends_on: - - augur-db - - redis - - rabbitmq - -volumes: - facade: - driver: local - augurpostgres: - driver: local - - diff --git a/setup.py b/setup.py index 8591f483e2..a356fef858 100644 --- a/setup.py +++ b/setup.py @@ -52,7 +52,7 @@ "pandas==1.5.3", # 1.4.3 "numpy==1.26.0", # 1.23.2 "requests==2.32.0", # 2.28.1 - "psycopg2-binary==2.9.3", #2.9.3 what is pscopg-binary 3.0.16 + "psycopg2-binary==2.9.9", #2.9.3 what is pscopg-binary 3.0.16 "click==8.0.3", # 8.1.3 "psutil==5.8.0", # 5.9.1 "gunicorn==22.0.0", # 20.1.0