Start capturing metadata for GitHub API requests

Author: mikebroberts

src/app/domain/github/crawler/crawlInstallation.ts

@@ -12,14 +12,18 @@ export async function crawlInstallation(
   lookbackDays: number
 ) {
   logger.info(`Crawling Installation for ${installation.accountLogin}`)
-  await crawlUsers(appState, installation)
-  const repos = await crawlRepositories(appState, installation)
+  const githubInstallationClient = appState.githubClient.clientForInstallation(installation.installationId)
+  await crawlUsers(appState, installation, githubInstallationClient)
+  logger.info('Github Metadata after crawling users', { ...githubInstallationClient.meta() })
+  const repos = await crawlRepositories(appState, installation, githubInstallationClient)
   // Eventually consider doing some parallelization here (or move back to step function) but
   // need to be careful since GitHub gets twitchy about concurrent requests to the API
   // Their "best practice" doc says don't do it, but their rate limit doc says it's supported
   // Only really need to care if things start getting slow
   for (const repo of repos) {
-    await crawlPushes(appState, installation, repo)
-    await crawlWorkflowRunEvents(appState, installation, repo, lookbackDays)
+    await crawlPushes(appState, installation, repo, githubInstallationClient)
+    await crawlWorkflowRunEvents(appState, installation, repo, lookbackDays, githubInstallationClient)
   }
+
+  logger.info('Github Metadata after crawls', { ...githubInstallationClient.meta() })
 }

src/app/domain/github/crawler/crawlPushes.ts

@@ -4,18 +4,17 @@ import { isRawGithubPushEventEvent } from '../../types/rawGithub/RawGithubAPIPus
 import { fromRawGithubPushEventEvent, GithubPush } from '../../types/GithubPush'
 import { processPushes } from '../githubPush'
 import { GithubInstallation } from '../../types/GithubInstallation'
-import { logger } from '../../../util/logging'
+import { GithubInstallationClient } from '../../../outboundInterfaces/githubInstallationClient'
 
 // TOEventually - only get all pushes back to lookback in crawl configuration, however GitHub doesn't keep
 // them around for very long
 export async function crawlPushes(
   appState: AppState,
   // the owner ID on repo isn't sufficient when we are crawling public repos from other accounts
   installation: GithubInstallation,
-  repo: GithubRepositorySummary
+  repo: GithubRepositorySummary,
+  githubClient: GithubInstallationClient
 ) {
-  logger.info(`Crawling Pushes for ${installation.accountLogin}/${repo.name}`)
-  const githubClient = appState.githubClient.clientForInstallation(installation.installationId)
   const allEventsForRepo = await githubClient.listMostRecentEventsForRepo(repo.ownerName, repo.name)
   const rawPushes = allEventsForRepo.filter(isRawGithubPushEventEvent)
   // TODO - this comment was from pre-step-functions version. Is there something that can be improved now

src/app/domain/github/crawler/crawlRepositories.ts

@@ -3,11 +3,12 @@ import { GithubInstallation } from '../../types/GithubInstallation'
 import { GithubInstallationClient } from '../../../outboundInterfaces/githubInstallationClient'
 import { processRawRepositories, toRepositorySummary } from '../githubRepository'
 import { ORGANIZATION_ACCOUNT_TYPE, USER_ACCOUNT_TYPE } from '../../types/GithubAccountType'
-import { logger } from '../../../util/logging'
 
-export async function crawlRepositories(appState: AppState, installation: GithubInstallation) {
-  logger.info(`Crawling Repositories for ${installation.accountLogin}`)
-  const githubClient = appState.githubClient.clientForInstallation(installation.installationId)
+export async function crawlRepositories(
+  appState: AppState,
+  installation: GithubInstallation,
+  githubClient: GithubInstallationClient
+) {
   const latestRawRepositories = await readRawRepositories(installation, githubClient)
   const repos = await processRawRepositories(appState, latestRawRepositories)
   return repos.map(toRepositorySummary)

src/app/domain/github/crawler/crawlRunEvents.ts

@@ -3,17 +3,16 @@ import { GithubRepositorySummary } from '../../types/GithubRepository'
 import { dateTimeAddDays } from '../../../util/dateAndTime'
 import { processRawRunEvents } from '../githubWorkflowRunEvent'
 import { GithubInstallation } from '../../types/GithubInstallation'
-import { logger } from '../../../util/logging'
+import { GithubInstallationClient } from '../../../outboundInterfaces/githubInstallationClient'
 
 export async function crawlWorkflowRunEvents(
   appState: AppState,
   // the owner ID on repo isn't sufficient when we are crawling public repos from other accounts
   installation: GithubInstallation,
   repo: GithubRepositorySummary,
-  lookbackDays: number
+  lookbackDays: number,
+  githubClient: GithubInstallationClient
 ) {
-  logger.info(`Crawling Run Events for ${installation.accountLogin}/${repo.name}`)
-  const githubClient = appState.githubClient.clientForInstallation(installation.installationId)
   const startTime = `${dateTimeAddDays(appState.clock.now(), -1 * lookbackDays).toISOString()}`
 
   const recentRunEvents = await githubClient.listWorkflowRunsForRepo(

src/app/domain/github/crawler/crawlUsers.ts

@@ -3,14 +3,13 @@ import { GithubInstallation } from '../../types/GithubInstallation'
 import { GithubInstallationClient } from '../../../outboundInterfaces/githubInstallationClient'
 import { processRawUsers } from '../githubUser'
 import { ORGANIZATION_ACCOUNT_TYPE, USER_ACCOUNT_TYPE } from '../../types/GithubAccountType'
-import { logger } from '../../../util/logging'
 
-export async function crawlUsers(appState: AppState, installation: GithubInstallation) {
-  logger.info(`Crawling Users for ${installation.accountLogin}`)
-  const latestRawUsers = await readRawUsers(
-    installation,
-    appState.githubClient.clientForInstallation(installation.installationId)
-  )
+export async function crawlUsers(
+  appState: AppState,
+  installation: GithubInstallation,
+  githubClient: GithubInstallationClient
+) {
+  const latestRawUsers = await readRawUsers(installation, githubClient)
   await processRawUsers(appState, latestRawUsers, installation)
 }
 

src/app/outboundInterfaces/githubInstallationClient.ts

@@ -4,7 +4,6 @@ import { RawGithubWorkflowRunEvent } from '../domain/types/rawGithub/RawGithubWo
 import { RawGithubRepository } from '../domain/types/rawGithub/RawGithubRepository'
 import { RawGithubUser } from '../domain/types/rawGithub/RawGithubUser'
 import { RawGithubEvent } from '../domain/types/rawGithub/RawGithubEvent'
-import { logger } from '../util/logging'
 
 export interface GithubInstallationClient {
   listWorkflowRunsForRepo(owner: string, repo: string, created?: string): Promise<RawGithubWorkflowRunEvent[]>
@@ -19,6 +18,17 @@ export interface GithubInstallationClient {
   listMostRecentEventsForRepo(owner: string, repo: string): Promise<RawGithubEvent[]>
 
   getUser(username: string): Promise<RawGithubUser>
+
+  meta(): GithubInstallationClientMeta
+}
+
+export interface GithubInstallationClientMeta {
+  // For now assuming all API calls are to the "core" resource, and so I don't differentiate
+  // by x-ratelimit-used header (see https://docs.github.com/en/rest/rate-limit/rate-limit?apiVersion=2022-11-28#get-rate-limit-status-for-the-authenticated-user)
+  readonly ratelimit: number
+  readonly ratelimitUsed: number
+  readonly ratelimitRemaining: number
+  readonly ratelimitResetTimestamp: number
 }
 
 export function createRealGithubInstallationClient(
@@ -38,41 +48,82 @@ export function createRealGithubInstallationClient(
       installationId
     }
   })
+  const rateLimitData: {
+    ratelimit: number
+    ratelimitRemaining: number
+    ratelimitUsed: number
+    ratelimitResetTimestamp: number
+  } = {
+    ratelimit: 0,
+    ratelimitRemaining: 0,
+    ratelimitUsed: 0,
+    ratelimitResetTimestamp: 0
+  }
+
+  function updateRateLimits(headers: OctokitResponseHeaders) {
+    rateLimitData.ratelimit = Number(headers['x-ratelimit-limit'] ?? 0)
+    rateLimitData.ratelimitRemaining = Number(headers['x-ratelimit-remaining'] ?? 0)
+    rateLimitData.ratelimitUsed = Number(headers['x-ratelimit-used'] ?? 0)
+    rateLimitData.ratelimitResetTimestamp = Number(headers['x-ratelimit-reset'] ?? 0)
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  async function processOctokitIterator<T>(iterator: any) {
+    const results: T[] = []
+    for await (const response of iterator) {
+      results.push(...response.data)
+      updateRateLimits(response.headers)
+    }
+    return results
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  function processOctokitResponse(response: any) {
+    updateRateLimits(response.headers)
+    return response.data
+  }
 
   return {
     async listWorkflowRunsForRepo(owner: string, repo: string, created?: string) {
-      // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-      // @ts-ignore
-      return await octokit.paginate(octokit.actions.listWorkflowRunsForRepo, {
-        owner,
-        repo,
-        created
-      })
+      return processOctokitIterator(
+        octokit.paginate.iterator(octokit.actions.listWorkflowRunsForRepo, {
+          owner,
+          repo,
+          created
+        })
+      )
     },
     // TOMaybe - consider other options here, e.g. type, sort
     async listOrganizationRepositories(org: string): Promise<RawGithubRepository[]> {
-      // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-      // @ts-ignore
-      return await octokit.paginate(octokit.repos.listForOrg, { org })
+      return processOctokitIterator(octokit.paginate.iterator(octokit.repos.listForOrg, { org }))
     },
     async listInstallationRepositories(): Promise<RawGithubRepository[]> {
-      // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-      // @ts-ignore
-      return await octokit.paginate(octokit.apps.listReposAccessibleToInstallation, {})
+      return processOctokitIterator(
+        octokit.paginate.iterator(octokit.apps.listReposAccessibleToInstallation, {})
+      )
     },
     async listOrganizationMembers(org: string): Promise<RawGithubUser[]> {
-      // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-      // @ts-ignore
-      return await octokit.paginate(octokit.orgs.listMembers, { org })
+      return processOctokitIterator(octokit.paginate.iterator(octokit.orgs.listMembers, { org }))
     },
     // For now, hard code page size to 10
     // GitHub doesn't retain these for long - so anything older than a few days won't be returned
     async listMostRecentEventsForRepo(owner: string, repo: string): Promise<RawGithubEvent[]> {
-      logger.debug(`List recent ${owner} ${repo}`)
-      return (await octokit.activity.listRepoEvents({ owner, repo, per_page: 10 })).data
+      return processOctokitResponse(await octokit.activity.listRepoEvents({ owner, repo, per_page: 10 }))
     },
     async getUser(username: string): Promise<RawGithubUser> {
-      return (await octokit.users.getByUsername({ username })).data
+      return processOctokitResponse(await octokit.users.getByUsername({ username }))
+    },
+    meta(): GithubInstallationClientMeta {
+      return rateLimitData
     }
   }
 }
+
+export type OctokitResponseHeaders = {
+  etag?: string
+  'last-modified'?: string
+  'x-ratelimit-limit'?: string
+  'x-ratelimit-remaining'?: string
+  'x-ratelimit-reset'?: string
+  'x-ratelimit-used'?: string
+}

test/local/functional/domain/github/crawler/crawlPushes.test.ts

@@ -25,7 +25,6 @@ test('repo-crawler-for-personal-account-installation', async () => {
   // A
   const appState = new FakeAppState()
   const githubInstallationClient = new FakeGithubInstallationClient()
-  appState.githubClient.fakeClientsForInstallation.addResponse(48093071, githubInstallationClient)
   githubInstallationClient.stubMostRecentEventsForRepo.addResponse(
     {
       owner: 'cicada-test-user',
@@ -35,7 +34,7 @@ test('repo-crawler-for-personal-account-installation', async () => {
   )
 
   // A
-  await crawlPushes(appState, testPersonalInstallation, testPersonalTestRepo)
+  await crawlPushes(appState, testPersonalInstallation, testPersonalTestRepo, githubInstallationClient)
 
   // A
   expectPutsLength(appState).toEqual(2)
@@ -57,7 +56,7 @@ test('repo-crawler-for-org-installation', async () => {
   )
 
   // A
-  await crawlPushes(appState, testOrgInstallation, testOrgTestRepoOne)
+  await crawlPushes(appState, testOrgInstallation, testOrgTestRepoOne, githubInstallationClient)
 
   // A
   expectPutsLength(appState).toEqual(2)

test/local/functional/domain/github/crawler/crawlRepositories.test.ts

@@ -21,11 +21,10 @@ test('repository-crawler-for-personal-account-installation', async () => {
   // A
   const appState = new FakeAppState()
   const githubInstallationClient = new FakeGithubInstallationClient()
-  appState.githubClient.fakeClientsForInstallation.addResponse(48093071, githubInstallationClient)
   githubInstallationClient.stubInstallationRepositories = [example_personal_account_repo]
 
   // A
-  await crawlRepositories(appState, testPersonalInstallation)
+  await crawlRepositories(appState, testPersonalInstallation, githubInstallationClient)
 
   // A
   expectBatchWritesLength(appState).toEqual(1)
@@ -36,11 +35,10 @@ test('repository-crawler-for-org-installation', async () => {
   // A
   const appState = new FakeAppState()
   const githubInstallationClient = new FakeGithubInstallationClient()
-  appState.githubClient.fakeClientsForInstallation.addResponse(48133709, githubInstallationClient)
   githubInstallationClient.stubOrganizationRepositories.addResponse('cicada-test-org', example_org_repos)
 
   // A
-  await crawlRepositories(appState, testOrgInstallation)
+  await crawlRepositories(appState, testOrgInstallation, githubInstallationClient)
 
   // A
   expectBatchWritesLength(appState).toEqual(1)

test/local/functional/domain/github/crawler/crawlRunEvents.test.ts

@@ -27,7 +27,6 @@ test('repo-crawler-for-personal-account-installation', async () => {
   // A
   const appState = new FakeAppState()
   const githubInstallationClient = new FakeGithubInstallationClient()
-  appState.githubClient.fakeClientsForInstallation.addResponse(48093071, githubInstallationClient)
   githubInstallationClient.stubWorkflowRunsForRepo.addResponse(
     {
       owner: 'cicada-test-user',
@@ -38,7 +37,13 @@ test('repo-crawler-for-personal-account-installation', async () => {
   )
 
   // A
-  await crawlWorkflowRunEvents(appState, testPersonalInstallation, testPersonalTestRepo, 10)
+  await crawlWorkflowRunEvents(
+    appState,
+    testPersonalInstallation,
+    testPersonalTestRepo,
+    10,
+    githubInstallationClient
+  )
 
   // A
   expectPutsLength(appState).toEqual(3)
@@ -51,7 +56,6 @@ test('repo-crawler-for-org-installation', async () => {
   // A
   const appState = new FakeAppState()
   const githubInstallationClient = new FakeGithubInstallationClient()
-  appState.githubClient.fakeClientsForInstallation.addResponse(48133709, githubInstallationClient)
   githubInstallationClient.stubWorkflowRunsForRepo.addResponse(
     {
       owner: 'cicada-test-org',
@@ -62,7 +66,13 @@ test('repo-crawler-for-org-installation', async () => {
   )
 
   // A
-  await crawlWorkflowRunEvents(appState, testOrgInstallation, testOrgTestRepoOne, 10)
+  await crawlWorkflowRunEvents(
+    appState,
+    testOrgInstallation,
+    testOrgTestRepoOne,
+    10,
+    githubInstallationClient
+  )
 
   // A
   expectPutsLength(appState).toEqual(3)

test/local/functional/domain/github/crawler/crawlUsers.test.ts

@@ -29,11 +29,10 @@ test('user-crawler-for-personal-account-installation', async () => {
   // A
   const appState = new FakeAppState()
   const githubInstallationClient = new FakeGithubInstallationClient()
-  appState.githubClient.fakeClientsForInstallation.addResponse(48093071, githubInstallationClient)
   githubInstallationClient.stubUsers.addResponse('cicada-test-user', example_personal_account_user)
 
   // A
-  await crawlUsers(appState, testPersonalInstallation)
+  await crawlUsers(appState, testPersonalInstallation, githubInstallationClient)
 
   // A
   expectBatchWritesLength(appState).toEqual(2)
@@ -47,7 +46,6 @@ test('user-crawler-for-org-installation', async () => {
   // A
   const appState = new FakeAppState()
   const githubInstallationClient = new FakeGithubInstallationClient()
-  appState.githubClient.fakeClientsForInstallation.addResponse(48133709, githubInstallationClient)
   githubInstallationClient.stubOrganizationMembers.addResponse('cicada-test-org', example_org_users)
 
   stubQueryAccountMembershipsByAccount(appState, [
@@ -57,7 +55,7 @@ test('user-crawler-for-org-installation', async () => {
   ])
 
   // A
-  await crawlUsers(appState, testOrgInstallation)
+  await crawlUsers(appState, testOrgInstallation, githubInstallationClient)
 
   // A
   expectBatchWritesLength(appState).toEqual(3)