feat: add workflow upsert-github-repositories

Author: UlisesGascon

src/cli/workflows.js

@@ -1,10 +1,16 @@
 const inquirer = require('inquirer').default
-const { updateGithubOrgs } = require('../workflows')
+const debug = require('debug')('cli:workflows')
+const { updateGithubOrgs, upsertGithubRepositories } = require('../workflows')
 const { logger } = require('../utils')
 
 const commandList = [{
   name: 'update-github-orgs',
-  description: 'Check the organizations stored and update the information with the GitHub API.'
+  description: 'Check the organizations stored and update the information with the GitHub API.',
+  workflow: updateGithubOrgs
+}, {
+  name: 'upsert-github-repositories',
+  description: 'Check the organizations stored and update/create the information related to the repositories with the GitHub API.',
+  workflow: upsertGithubRepositories
 }]
 
 const validCommandNames = commandList.map(({ name }) => name)
@@ -32,9 +38,9 @@ async function runWorkflowCommand (knex, options = {}) {
       }
     ])
 
-  if (answers.name === 'update-github-orgs') {
-    await updateGithubOrgs(knex)
-  }
+  const command = commandList.find(({ name }) => name === answers.name)
+  debug(`Running workflow: ${command.name}`)
+  await command.workflow(knex)
 
   return answers
 }

src/workflows/index.js

@@ -3,7 +3,7 @@ const { simplifyObject } = require('@ulisesgascon/simplify-object')
 const { github } = require('../providers')
 const { initializeStore } = require('../store')
 const { logger } = require('../utils')
-const { validateGithubOrg } = require('../schemas')
+const { validateGithubOrg, validateGithubListOrgRepos, validateGithubRepository } = require('../schemas')
 
 const mapOrgData = (data) => {
   const mappedData = simplifyObject(data, {
@@ -37,6 +37,44 @@ const mapOrgData = (data) => {
   return mappedData
 }
 
+const mapRepoData = (data) => {
+  const mappedData = simplifyObject(data, {
+    include: [
+      'node_id', 'name', 'full_name',
+      'html_url', 'description', 'fork', 'url',
+      'git_url', 'ssh_url', 'clone_url', 'svn_url',
+      'homepage', 'size', 'stargazers_count', 'watchers_count',
+      'language', 'has_issues', 'has_projects', 'has_downloads',
+      'has_wiki', 'has_pages', 'has_discussions', 'forks_count',
+      'mirror_url', 'archived', 'disabled', 'open_issues_count',
+      'allow_forking', 'is_template', 'web_commit_signoff_required',
+      'topics', 'visibility', 'default_branch', 'allow_squash_merge',
+      'allow_merge_commit', 'allow_rebase_merge', 'allow_auto_merge',
+      'delete_branch_on_merge', 'allow_update_branch', 'use_squash_pr_title_as_default',
+      'squash_merge_commit_message', 'squash_merge_commit_title', 'merge_commit_message',
+      'merge_commit_title', 'network_count', 'subscribers_count'
+    ]
+  })
+  // Additional fields that have different names in the database
+  mappedData.github_repo_id = data.id
+  mappedData.github_created_at = data.created_at
+  mappedData.github_updated_at = data.updated_at
+  mappedData.github_archived_at = data.archived_at
+  // license
+  mappedData.license_key = data.license?.key
+  mappedData.license_name = data.license?.name
+  mappedData.license_spdx_id = data.license?.spdx_id
+  mappedData.license_url = data.license?.url
+  mappedData.license_node_id = data.license?.node_id
+  // Security and analysis
+  mappedData.secret_scanning_status = data.security_and_analysis?.secret_scanning?.status
+  mappedData.secret_scanning_push_protection_status = data.security_and_analysis?.secret_scanning_push_protection?.status
+  mappedData.dependabot_security_updates_status = data.security_and_analysis?.dependabot_security_updates?.status
+  mappedData.secret_scanning_non_provider_patterns_status = data.security_and_analysis?.secret_scanning_non_provider_patterns?.status
+  mappedData.secret_scanning_validity_checks_status = data.security_and_analysis?.secret_scanning_validity_checks?.status
+  return mappedData
+}
+
 const updateGithubOrgs = async (knex) => {
   const { getAllGithubOrganizations, updateGithubOrganization } = initializeStore(knex)
   const organizations = await getAllGithubOrganizations()
@@ -58,6 +96,39 @@ const updateGithubOrgs = async (knex) => {
   logger.log('GitHub organizations updated successfully')
 }
 
+const upsertGithubRepositories = async (knex) => {
+  const { getAllGithubOrganizations, upsertGithubRepository } = initializeStore(knex)
+  const organizations = await getAllGithubOrganizations()
+  debug('Checking stored organizations')
+  if (organizations.length === 0) {
+    throw new Error('No organizations found. Please add organizations/projects before running this workflow.')
+  }
+
+  debug('Fetching repositories for organizations from GitHub API')
+
+  await Promise.all(organizations.map(async (org) => {
+    debug(`Fetching repositories for org (${org.login})`)
+    const repoList = await github.fetchOrgReposListByLogin(org.login)
+    debug(`Got ${repoList.length} repositories for org (${org.login})`)
+    debug('Validating data')
+    validateGithubListOrgRepos(repoList)
+    debug(`Enriching all repositories for org (${org.login})`)
+
+    // Enrich and upsert each repository in parallel
+    await Promise.all(repoList.map(async (repo) => {
+      debug(`Enriching repository (${repo.full_name})`)
+      const repoData = await github.fetchRepoByFullName(repo.full_name)
+      debug(`Validating repository (${repo.full_name})`)
+      validateGithubRepository(repoData)
+      debug(`Transforming repository (${repo.full_name}) data`)
+      const mappedData = mapRepoData(repoData)
+      debug(`Upserting repository (${repo.full_name})`)
+      await upsertGithubRepository(mappedData, org.id)
+    }))
+  }))
+}
+
 module.exports = {
-  updateGithubOrgs
+  updateGithubOrgs,
+  upsertGithubRepositories
 }