Merge pull request github#16143 from github/hidden-docs-alt

Early-access docs V2

Author: sarahs

.env.example

@@ -1,6 +1,3 @@
 ALGOLIA_API_KEY=
 ALGOLIA_APPLICATION_ID=
 ALLOW_TRANSLATION_COMMITS=
-EARLY_ACCESS_HOSTNAME=
-EARLY_ACCESS_SHARED_SECRET=
-GITHUB_TOKEN=

.github/workflows/test.yml

@@ -27,7 +27,7 @@ jobs:
         with:
           cancel_others: 'false'
           github_token: ${{ github.token }}
-          paths: '[".github/workflows/test.yml",".node-version", ".npmrc", "app.json", "content/**", "data/**","lib/**", "Dockerfile", "feature-flags.json", "Gemfile", "Gemfile.lock", "middleware/**", "node_modules/**","package.json", "package-lock.json", "server.js", "tests/**", "translations/**", "Procfile", "webpack.config.js"]'
+          paths: '[".github/workflows/test.yml", ".node-version", ".npmrc", "app.json", "content/**", "data/**","lib/**", "Dockerfile", "feature-flags.json", "Gemfile", "Gemfile.lock", "middleware/**", "node_modules/**","package.json", "package-lock.json", "server.js", "tests/**", "translations/**", "Procfile", "webpack.config.js"]'
 
   test:
     needs: see_if_should_skip
@@ -44,6 +44,9 @@ jobs:
       - if: ${{ needs.see_if_should_skip.outputs.should_skip != 'true' }}
         name: Check out repo
         uses: actions/checkout@5a4ac9002d0be2fb38bd78e4b4dbde5606d7042f
+        with:
+          # Enables cloning the Early Access repo later with the relevant PAT
+          persist-credentials: 'false'
 
       - if: ${{ needs.see_if_should_skip.outputs.should_skip != 'true' }}
         name: Setup node
@@ -70,19 +73,19 @@ jobs:
         name: Install dependencies
         run: npm ci
 
-      - if: ${{ needs.see_if_should_skip.outputs.should_skip != 'true' }}
-        name: Run build script
+      - name: Clone early access
+        if: ${{ needs.see_if_should_skip.outputs.should_skip != 'true' && github.repository == 'github/docs-internal' }}
+        run: npm run heroku-postbuild
+        env:
+          DOCUBOT_REPO_PAT: ${{ secrets.DOCUBOT_REPO_PAT }}
+          GIT_BRANCH: ${{ github.ref }}
+
+      - name: Run build script
+        if: ${{ needs.see_if_should_skip.outputs.should_skip != 'true' && github.repository != 'github/docs-internal' }}
         run: npm run build
 
       - if: ${{ needs.see_if_should_skip.outputs.should_skip != 'true' }}
         name: Run tests
         run: npx jest tests/${{ matrix.test-group }}/
         env:
           NODE_OPTIONS: '--max_old_space_size=4096'
-
-      - name: Send Slack notification if workflow fails
-        uses: rtCamp/action-slack-notify@e17352feaf9aee300bf0ebc1dfbf467d80438815
-        if: failure() && github.ref == 'early-access'
-        env:
-          SLACK_WEBHOOK: ${{ secrets.DOCS_ALERTS_SLACK_WEBHOOK }}
-          SLACK_MESSAGE: 'Tests are failing on the `early-access` branch.'

.gitignore

@@ -1,9 +1,17 @@
 .algolia-cache
 .DS_Store
 .env
-node_modules
+/node_modules/
 npm-debug.log
-coverage
+coverage/
 .linkinator
-broken_links.md
+/assets/images/early-access
+/content/early-access
+/data/early-access
 dist
+
+# blc: broken link checker
+blc_output.log
+blc_output_internal.log
+/dist/
+broken_links.md

content/rest/overview/api-previews.md

@@ -2,7 +2,6 @@
 title: API previews
 intro: You can use API previews to try out new features and provide feedback before these features become official.
 redirect_from:
-  - /early-access/
   - /v3/previews
 versions:
   free-pro-team: '*'
@@ -60,7 +59,7 @@ Create, list, update, and delete environments for pre-receive hooks.
 {% if enterpriseServerVersions contains currentVersion and currentVersion ver_lt "[email protected]" %}
 ### Integrations
 
-Manage [integrations](/early-access/integrations/) through the API.
+Manage [integrations](/v3/integrations) through the API.
 
 **Custom media type:** `machine-man-preview`
 **Announced:** [2016-09-14](https://developer.github.com/changes/2016-09-14-Integrations-Early-Access/)

crowdin.yml

@@ -3,17 +3,19 @@ files:
     translation: /translations/%locale%/%original_path%/%original_file_name%
     ignore:
       - '/content/README.md'
+      - '/content/early-access'
   - source: /data/**/*.yml
     translation: /translations/%locale%/%original_path%/%original_file_name%
   - source: /data/**/*.md
     translation: /translations/%locale%/%original_path%/%original_file_name%
     ignore:
-      - 'data/README.md'
-      - 'data/reusables/README.md'
-      - 'data/variables/product.yml'
-      - 'data/variables/README.md'
-      - 'data/graphql'
-      - 'data/products.yml'
+      - '/data/README.md'
+      - '/data/reusables/README.md'
+      - '/data/variables/product.yml'
+      - '/data/variables/README.md'
+      - '/data/early-access'
+      - '/data/graphql'
+      - '/data/products.yml'
 
 # These end up as env vars used by the GitHub Actions workflow
 project_id_env: CROWDIN_PROJECT_ID

includes/article.html

@@ -67,7 +67,7 @@ <h3 id="in-this-article" class="f5 mb-2"><a class="link-gray-dark" href="#in-thi
 
     <div class="d-block border-top border-gray-light mt-4 markdown-body">
       {% include helpfulness %}
-      {% include contribution %}
+      {% unless page.hidden %}{% include contribution %}{% endunless %}
     </div>
   </article>
 </main>

includes/breadcrumbs.html

@@ -1,6 +1,10 @@
 <nav class="breadcrumbs f5" aria-label="Breadcrumb">
   {% for breadcrumb in breadcrumbs %}
+  {% if page.hidden %}
+  <span class="d-inline-block">{{breadcrumb[1].title}}</span>
+  {% else %}
   <a title="{{ breadcrumb[0]}}: {{breadcrumb[1].title}}" href="/{{currentLanguage}}{{breadcrumb[1].href}}" class="d-inline-block {% if breadcrumb[1].href == currentPathWithoutLanguage %}text-gray-light{% endif %}">
     {{breadcrumb[1].title}}</a>
+  {% endif %}
   {% endfor %}
 </nav>

includes/header.html

@@ -1,5 +1,7 @@
 <div class="border-bottom border-gray-light no-print">
+  {% unless error == '404' %}
   {% include header-notification %}
+  {% endunless %}
 
   <header class="container-xl px-3 px-md-6 pt-3 pb-2 position-relative d-flex flex-justify-between width-full {% if error == '404' %} d-md-none {% endif %}">
 

includes/sidebar-specific-product.html

@@ -9,7 +9,9 @@
 {% assign product = siteTree[currentLanguage][currentVersion].products[currentProduct] %}
 {% include all-products-link %}
 <li title="{{product.title}}" class="sidebar-product mb-2">
+  {% unless page.hidden %}
   <a href="/{{currentLanguage}}{{product.href}}" class="pl-4 pr-5 pb-1 f4">{{ product.title }}</a>
+  {% endunless %}
 </li>
 <ul class="sidebar-categories list-style-none">
   {% for category in product.categories %}

lib/algolia/find-indexable-pages.js

@@ -3,8 +3,10 @@ const loadPages = require('../pages')
 module.exports = async function findIndexablePages () {
   const allPages = await loadPages()
   const indexablePages = allPages
-    // exclude pages that are part of WIP products
-    .filter(page => !page.parentProduct || !page.parentProduct.wip)
+    // exclude hidden pages
+    .filter(page => !page.hidden)
+    // exclude pages that are part of WIP or hidden products
+    .filter(page => !page.parentProduct || !page.parentProduct.wip || page.parentProduct.hidden)
     // exclude index homepages
     .filter(page => !page.relativePath.endsWith('index.md'))
 

lib/all-products.js

@@ -14,14 +14,32 @@ const productsYml = yaml.load(fs.readFileSync(productsFile, 'utf8'))
 const sortedProductIds = productsYml.productsInOrder
 
 const contentProductIds = fs.readdirSync(contentDir, { withFileTypes: true })
+  .map(entry => {
+    // `fs.readdir` provides file entries based on `fs.lstat`, which doesn't
+    // resolve symbolic links to their target file/directory. We need to take
+    // an extra step here to resolve the Early Access symlinked directory.
+    const { name } = entry
+    if (entry.isSymbolicLink()) {
+      entry = fs.statSync(path.join(contentDir, entry.name))
+      entry.name = name
+    }
+    return entry
+  })
   .filter(entry => entry.isDirectory())
   .map(entry => entry.name)
 
-assert(difference(sortedProductIds, contentProductIds).length === 0)
-assert(difference(contentProductIds, sortedProductIds).length === 0)
+// require the content/<subdir> list to match the list in data/products.yml,
+// with the exception of content/early-access, which lives in a separate private repo
+const publicContentProductIds = contentProductIds.filter(id => id !== 'early-access')
+assert(difference(sortedProductIds, publicContentProductIds).length === 0)
+assert(difference(publicContentProductIds, sortedProductIds).length === 0)
 
 const internalProducts = {}
 
+// add optional early access content dir to sorted products list if present
+const earlyAccessId = contentProductIds.find(id => id === 'early-access')
+if (earlyAccessId) sortedProductIds.push(earlyAccessId)
+
 sortedProductIds.forEach(productId => {
   const relPath = productId
   const dir = slash(path.join('content', relPath))
@@ -36,7 +54,8 @@ sortedProductIds.forEach(productId => {
     href,
     dir,
     toc,
-    wip: data.wip || false
+    wip: data.wip || false,
+    hidden: data.hidden || false
   }
 
   internalProducts[productId].versions = applicableVersions

lib/fetch-early-access-paths.js

@@ -38,11 +38,9 @@ const schema = {
     mapTopic: {
       type: 'boolean'
     },
-    // The `hidden` frontmatter property is no longer used, but leaving it here
-    // with an enum of `[false]` will help us catch any possible regressions.
+    // allow hidden articles under `early-access`
     hidden: {
-      type: 'boolean',
-      enum: [false]
+      type: 'boolean'
     },
     layout: {
       type: ['string', 'boolean'],

lib/frontmatter.js

@@ -25,7 +25,7 @@ module.exports = {
   searchPath: /\/search(?:\/)?(\?)/,
   ymd: /^\d{4}-\d{2}-\d{2}$/,
   hasLiquid: /[{{][{%]/,
-  dataReference: /{% ?data\s(?:reusables|variables|ui)\..*?%}/gm,
+  dataReference: /{% ?data\s(?:early-access\.)?(?:reusables|variables|ui)\..*?%}/gm,
   imagePath: /\/?assets\/images\/.*?\.(png|svg|gif|pdf|ico|jpg|jpeg)/gi,
   homepagePath: /^\/\w{2}$/, // /en, /ja, /cn
   multipleSlashes: /^\/{2,}/,

lib/patterns.js

@@ -1,24 +1,22 @@
 const statsd = require('./statsd')
-const fetchEarlyAccessPaths = require('./fetch-early-access-paths')
 const loadPages = require('./pages')
 const loadRedirects = require('./redirects/precompile')
 const loadSiteData = require('./site-data')
 const loadSiteTree = require('./site-tree')
 
 // For local caching
-let pages, site, redirects, siteTree, earlyAccessPaths
+let pages, site, redirects, siteTree
 
 function isFullyWarmed () {
-  return Boolean(pages && site && earlyAccessPaths && redirects && siteTree)
+  return Boolean(pages && site && redirects && siteTree)
 }
 
 function getWarmedCache () {
   return {
     pages,
     site,
     redirects,
-    siteTree,
-    earlyAccessPaths
+    siteTree
   }
 }
 
@@ -29,12 +27,11 @@ async function warmServer () {
     console.log('Priming context information...')
   }
 
-  if (!pages || !site || !earlyAccessPaths) {
+  if (!pages || !site) {
     // Promise.all is used to load multiple things in parallel
-    [pages, site, earlyAccessPaths] = await Promise.all([
+    [pages, site] = await Promise.all([
       pages || loadPages(),
-      site || loadSiteData(),
-      earlyAccessPaths || fetchEarlyAccessPaths()
+      site || loadSiteData()
     ])
   }
 

lib/warm-server.js

@@ -2,7 +2,7 @@ const languages = require('../lib/languages')
 const enterpriseServerReleases = require('../lib/enterprise-server-releases')
 const allVersions = require('../lib/all-versions')
 const allProducts = require('../lib/all-products')
-const activeProducts = Object.values(allProducts).filter(product => !product.wip)
+const activeProducts = Object.values(allProducts).filter(product => !product.wip && !product.hidden)
 const { getVersionStringFromPath, getProductStringFromPath, getPathWithoutLanguage } = require('../lib/path-utils')
 const productNames = require('../lib/product-names')
 const warmServer = require('../lib/warm-server')
@@ -12,7 +12,7 @@ const featureFlags = Object.keys(require('../feature-flags'))
 // Note that additional middleware in middleware/index.js adds to this context object
 module.exports = async function contextualize (req, res, next) {
   // Ensure that we load some data only once on first request
-  const { site, redirects, pages, siteTree, earlyAccessPaths } = await warmServer()
+  const { site, redirects, pages, siteTree } = await warmServer()
   req.context = {}
 
   // make feature flag environment variables accessible in layouts
@@ -33,7 +33,6 @@ module.exports = async function contextualize (req, res, next) {
   req.context.currentPath = req.path
   req.context.query = req.query
   req.context.languages = languages
-  req.context.earlyAccessPaths = earlyAccessPaths
   req.context.productNames = productNames
   req.context.enterpriseServerReleases = enterpriseServerReleases
   req.context.enterpriseServerVersions = Object.keys(allVersions).filter(version => version.startsWith('enterprise-server@'))

middleware/context.js

@@ -0,0 +1,27 @@
+module.exports = function earlyAccessContext (req, res, next) {
+  if (process.env.NODE_ENV === 'production') {
+    return next(404)
+  }
+
+  // Get a list of all hidden pages per version
+  const earlyAccessPageLinks = req.context.pages
+    .filter(page => page.hidden)
+    // Do not include early access landing page
+    .filter(page => page.relativePath !== 'early-access/index.md')
+    // Create Markdown links
+    .map(page => {
+      return page.permalinks.map(permalink => `- [${permalink.title}](${permalink.href})`)
+    })
+    .flat()
+    // Get links for the current version
+    .filter(link => link.includes(req.context.currentVersion))
+    .sort()
+
+  // Add to the rendering context
+  // This is only used in the separate EA repo on local development
+  req.context.earlyAccessPageLinks = earlyAccessPageLinks.length
+    ? earlyAccessPageLinks.join('\n')
+    : '_None for this version!_'
+
+  return next()
+}