refactor openapi scripts (github#34803)

Author: rachmari

.github/workflows/openapi-decorate.yml

@@ -1,7 +1,7 @@
 name: Sync OpenAPI schema
 
-# **What it does**: Once a day, this workflow syncs the dereferenced files from github/rest-api-description and creates a pull request if there are updates to any of the static files we generate from the OpenAPI.
-# **Why we have it**: So we can consume OpenAPI changes.
+# **What it does**: Once a day, this workflow syncs the REST, Webhooks, and GitHub Apps automated pipelines with the github/rest-api-description repository, and creates a pull request if there are updates to any of the data files we generate from the OpenAPI.
+# **Why we have it**: So we can automate updates to REST, Webhooks, and GitHub Apps documentation
 # **Who does it impact**: Anyone making OpenAPI changes in `github/github`, and wanting to get them published on the docs site.
 
 on:
@@ -49,19 +49,17 @@ jobs:
 
       - uses: ./.github/actions/node-npm-setup
 
-      - name: Copy dereferenced OpenAPI files
+      - name: Get the rest-api-description SHA being synced
         id: rest-api-description
         run: |
-          mkdir ./src/rest/data/dereferenced
-          find rest-api-description/descriptions-next -type f -name "*.deref.json" -exec sh -c 'cp $1 ./src/rest/data/dereferenced' sh {} \;
           cd rest-api-description
           OPENAPI_COMMIT_SHA=$(git rev-parse HEAD)
           echo "OPENAPI_COMMIT_SHA=$OPENAPI_COMMIT_SHA" >> $GITHUB_OUTPUT
           echo "Copied files from github/rest-api-description repo. Commit SHA: $OPENAPI_COMMIT_SHA"
 
-      - name: Decorate the dereferenced OpenAPI schemas
+      - name: Sync the REST, Webhooks, and GitHub Apps schemas
         run: |
-          src/rest/scripts/update-files.js --decorate-only --open-source
+          src/rest/scripts/update-files.js --source-repo rest-api-description --output rest github-apps webhooks rest-redirects
           git status
           echo "Deleting the cloned github/rest-api-description repo..."
           rm -rf rest-api-description

lib/all-versions.js

@@ -26,7 +26,7 @@ const plans = [
     latestRelease: latestNonNumberedRelease,
     nonEnterpriseDefault: true, // permanent way to refer to this plan if the name changes
     hasNumberedReleases: false,
-    openApiBaseName: 'api.github.com', // used for REST
+    openApiBaseName: 'fpt', // used for REST
     miscBaseName: 'dotcom', // used for GraphQL and webhooks
   },
   {
@@ -59,7 +59,7 @@ const plans = [
     // It lets us do semantic comparison internally while only exposing `@latest` in the UI.
     internalLatestRelease: '3.4',
     hasNumberedReleases: false,
-    openApiBaseName: 'github.ae',
+    openApiBaseName: 'ghae',
     miscBaseName: 'ghae',
   },
 ]

lib/redirects/static/client-side-rest-api-redirects.json

@@ -1,10 +1,12 @@
 import express from 'express'
+import path from 'path'
 
 import { readCompressedJsonFileFallbackLazily } from '../lib/read-json-file.js'
 import { defaultCacheControl } from './cache-control.js'
+import { REST_DATA_DIR } from '../src/rest/lib/index.js'
 
 const clientSideRestAPIRedirects = readCompressedJsonFileFallbackLazily(
-  './lib/redirects/static/client-side-rest-api-redirects.json'
+  path.join(REST_DATA_DIR, 'client-side-rest-api-redirects.json')
 )
 
 const router = express.Router()

middleware/anchor-redirect.js

@@ -609,13 +609,6 @@ Run this script to pull openAPI files from github/github, dereference them, and
 
 
 
----
-
-
-### [`rest/utils/rest-api-overrides.json`](rest/utils/rest-api-overrides.json)
-
-
-
 ---
 
 

script/README.md

@@ -893,7 +893,6 @@
       "requestPath": "/installation/token"
     }
   ],
-  "billing": [],
   "branches": [
     {
       "slug": "list-branches",
@@ -1716,7 +1715,6 @@
       "requestPath": "/emojis"
     }
   ],
-  "gists": [],
   "git": [
     {
       "slug": "create-a-blob",
@@ -2493,7 +2491,6 @@
       "requestPath": "/users/{username}/orgs"
     }
   ],
-  "packages": [],
   "pages": [
     {
       "slug": "get-a-github-pages-site",

src/github-apps/data/api.github.com.2022-11-28/server-to-server-rest.json renamed to src/github-apps/data/fpt-2022-11-28/server-to-server-rest.json

@@ -1133,8 +1133,6 @@
       "requestPath": "/emojis"
     }
   ],
-  "enterprise-admin": [],
-  "gists": [],
   "git": [
     {
       "slug": "create-a-blob",
@@ -1545,7 +1543,6 @@
       "requestPath": "/repos/{owner}/{repo}/stats/punch_card"
     }
   ],
-  "migrations": [],
   "orgs": [
     {
       "slug": "list-organizations",

src/github-apps/data/github.ae/server-to-server-rest.json renamed to src/github-apps/data/ghae/server-to-server-rest.json

@@ -1845,8 +1845,6 @@
       "requestPath": "/emojis"
     }
   ],
-  "enterprise-admin": [],
-  "gists": [],
   "git": [
     {
       "slug": "create-a-blob",
@@ -2677,7 +2675,6 @@
       "requestPath": "/users/{username}/orgs"
     }
   ],
-  "packages": [],
   "pages": [
     {
       "slug": "get-a-github-enterprise-cloud-pages-site",

src/github-apps/data/ghec.2022-11-28/server-to-server-rest.json renamed to src/github-apps/data/ghec-2022-11-28/server-to-server-rest.json

@@ -1439,7 +1439,6 @@
       "requestPath": "/repos/{owner}/{repo}/pre-receive-hooks/{pre_receive_hook_id}"
     }
   ],
-  "gists": [],
   "git": [
     {
       "slug": "create-a-blob",
@@ -1850,8 +1849,6 @@
       "requestPath": "/repos/{owner}/{repo}/stats/punch_card"
     }
   ],
-  "migrations": [],
-  "oauth-authorizations": [],
   "orgs": [
     {
       "slug": "list-organizations",

src/github-apps/data/ghes-3.4/server-to-server-rest.json

@@ -1547,7 +1547,6 @@
       "requestPath": "/repos/{owner}/{repo}/pre-receive-hooks/{pre_receive_hook_id}"
     }
   ],
-  "gists": [],
   "git": [
     {
       "slug": "create-a-blob",
@@ -1958,8 +1957,6 @@
       "requestPath": "/repos/{owner}/{repo}/stats/punch_card"
     }
   ],
-  "migrations": [],
-  "oauth-authorizations": [],
   "orgs": [
     {
       "slug": "list-organizations",

src/github-apps/data/ghes-3.5/server-to-server-rest.json

@@ -1389,7 +1389,6 @@
       "requestPath": "/repos/{owner}/{repo}/dependabot/secrets/{secret_name}"
     }
   ],
-  "dependency-graph": [],
   "deploy-keys": [
     {
       "slug": "list-deploy-keys",
@@ -1644,7 +1643,6 @@
       "requestPath": "/scim/v2/Users/{scim_user_id}"
     }
   ],
-  "gists": [],
   "git": [
     {
       "slug": "create-a-blob",
@@ -2055,8 +2053,6 @@
       "requestPath": "/repos/{owner}/{repo}/stats/punch_card"
     }
   ],
-  "migrations": [],
-  "oauth-authorizations": [],
   "orgs": [
     {
       "slug": "list-organizations",

src/github-apps/data/ghes-3.6/server-to-server-rest.json

@@ -1693,7 +1693,6 @@
       "requestPath": "/scim/v2/Users/{scim_user_id}"
     }
   ],
-  "gists": [],
   "git": [
     {
       "slug": "create-a-blob",
@@ -2104,8 +2103,6 @@
       "requestPath": "/repos/{owner}/{repo}/stats/punch_card"
     }
   ],
-  "migrations": [],
-  "oauth-authorizations": [],
   "orgs": [
     {
       "slug": "list-organizations",

src/github-apps/data/ghes-3.7/server-to-server-rest.json

@@ -1923,7 +1923,6 @@
       "requestPath": "/scim/v2/Users/{scim_user_id}"
     }
   ],
-  "gists": [],
   "git": [
     {
       "slug": "create-a-blob",
@@ -2334,8 +2333,6 @@
       "requestPath": "/repos/{owner}/{repo}/stats/punch_card"
     }
   ],
-  "migrations": [],
-  "oauth-authorizations": [],
   "orgs": [
     {
       "slug": "list-organizations",

src/github-apps/data/ghes-3.8/server-to-server-rest.json

@@ -0,0 +1,10 @@
+{
+  "api-versions": {
+    "fpt": [
+      "2022-11-28"
+    ],
+    "ghec": [
+      "2022-11-28"
+    ]
+  }
+}

src/github-apps/data/meta.json

@@ -1,3 +1,5 @@
+import path from 'path'
+
 import { readCompressedJsonFileFallback } from '../../../lib/read-json-file.js'
 import { getOpenApiVersion } from '../../../lib/all-versions.js'
 
@@ -7,13 +9,12 @@ export const ENABLED_APPS_FILENAME = 'server-to-server-rest.json'
 const enabledForApps = new Map()
 
 export async function getEnabledForApps(docsVersion, apiVersion) {
-  const openApiVersion = getOpenApiVersion(docsVersion) + (apiVersion ? `.${apiVersion}` : '')
+  const openApiVersion = getOpenApiVersion(docsVersion) + (apiVersion ? `-${apiVersion}` : '')
   if (!enabledForApps.has(openApiVersion)) {
     // The `readCompressedJsonFileFallback()` function
     // will check for both a .br and .json extension.
-    const data = readCompressedJsonFileFallback(
-      `${ENABLED_APPS_DIR}/${openApiVersion}/${ENABLED_APPS_FILENAME}`
-    )
+    const appDataPath = path.join(ENABLED_APPS_DIR, openApiVersion, ENABLED_APPS_FILENAME)
+    const data = readCompressedJsonFileFallback(appDataPath)
     enabledForApps.set(openApiVersion, data)
   }
 

src/github-apps/lib/index.js

@@ -0,0 +1,73 @@
+#!/usr/bin/env node
+
+import { existsSync } from 'fs'
+import mkdirp from 'mkdirp'
+import { readFile, writeFile } from 'fs/promises'
+import path from 'path'
+import { slug } from 'github-slugger'
+
+import { getOverrideCategory } from '../../rest/scripts/utils/operation.js'
+import { ENABLED_APPS_DIR, ENABLED_APPS_FILENAME } from '../lib/index.js'
+
+// Creates the src/github-apps/data files used for
+// https://docs.github.com/en/rest/overview/endpoints-available-for-github-apps
+export async function syncGitHubAppsData(sourceDirectory, sourceSchemas) {
+  for (const schemaName of sourceSchemas) {
+    const data = JSON.parse(await readFile(path.join(sourceDirectory, schemaName), 'utf8'))
+
+    // Because the information used on the apps page doesn't require any
+    // rendered content we can parse the dereferenced files directly
+    const enabledForApps = {}
+    for (const [requestPath, operationsAtPath] of Object.entries(data.paths)) {
+      for (const [verb, operation] of Object.entries(operationsAtPath)) {
+        // We only want to process operations that are
+        // server-to-server GitHub App enabled
+        if (!operation['x-github'].enabledForGitHubApps) continue
+
+        const schemaCategory = operation['x-github'].category
+        const schemaSubcategory = operation['x-github'].subcategory
+        const { category, subcategory } = getOverrideCategory(
+          operation.operationId,
+          schemaCategory,
+          schemaSubcategory
+        )
+
+        if (!enabledForApps[category]) {
+          enabledForApps[category] = []
+        }
+        enabledForApps[category].push({
+          slug: slug(operation.summary),
+          subcategory,
+          verb,
+          requestPath,
+        })
+      }
+    }
+
+    if (Object.keys(enabledForApps).length === 0) {
+      throw new Error(
+        `Generating GitHub Apps data failed for ${sourceDirectory}/${schemaName}. The generated data file was empty.`
+      )
+    }
+
+    // Sort the operations by category for readability
+    const sortedOperations = Object.keys(enabledForApps)
+      .sort()
+      .reduce((acc, key) => {
+        acc[key] = enabledForApps[key]
+        return acc
+      }, {})
+
+    const versionName = path.basename(schemaName, '.json')
+    const targetDirectory = path.join(ENABLED_APPS_DIR, versionName)
+
+    // When a new version is added, we need to create the directory for it
+    if (!existsSync(targetDirectory)) {
+      await mkdirp(targetDirectory)
+    }
+
+    const targetPath = path.join(targetDirectory, ENABLED_APPS_FILENAME)
+    await writeFile(targetPath, JSON.stringify(sortedOperations, null, 2))
+    console.log(`✅ Wrote ${targetPath}`)
+  }
+}