feat: add sitemaps for pillar pages and jobs

.github/workflows/ci.yml

@@ -82,11 +82,11 @@ jobs:
       - run: pnpm install --frozen-lockfile
       - name: Build
         env:
-          NEXT_PUBLIC_FRONTEND_URL: https://app.dev.jobstash.xyz
-          NEXT_PUBLIC_MW_URL: https://middleware.dev.jobstash.xyz
-          NEXT_PUBLIC_PRIVY_APP_ID: clyyn5fpe00j7nnq78b6dua71
+          NEXT_PUBLIC_FRONTEND_URL: https://jobstash.xyz
+          NEXT_PUBLIC_MW_URL: https://middleware.jobstash.xyz
+          NEXT_PUBLIC_PRIVY_APP_ID: clyr78r8l05a16wqnojin5hbz
           NEXT_PUBLIC_ALLOW_INDEXING: false
-          NEXT_PUBLIC_GA_MEASUREMENT_ID: G-VBP2TL6RJ0
+          NEXT_PUBLIC_GA_MEASUREMENT_ID: G-PQSHG9DB44
           SESSION_SECRET: ci-build-dummy-secret-not-for-prod!
           PRIVY_APP_SECRET: ci-build-dummy-secret
           R2_ENDPOINT: https://placeholder.test

package.json

@@ -1,6 +1,6 @@
 {
   "name": "webapp",
-  "version": "2.30.0",
+  "version": "2.31.0",
   "private": true,
   "scripts": {
     "dev": "next dev",

src/app/sitemap.ts

@@ -0,0 +1,52 @@
+import type { MetadataRoute } from 'next';
+
+import { clientEnv } from '@/lib/env/client';
+import { fetchSitemapJobs } from '@/features/jobs/server/data';
+import { fetchPillarSitemapSlugs } from '@/features/pillar/server/data';
+
+const FRONTEND_URL = clientEnv.FRONTEND_URL;
+
+export const generateSitemaps = () => [{ id: 0 }, { id: 1 }, { id: 2 }];
+
+export default async function sitemap({
+  id,
+}: {
+  id: number;
+}): Promise<MetadataRoute.Sitemap> {
+  if (id === 0) return staticSitemap();
+  if (id === 1) return pillarSitemap();
+  return jobSitemap();
+}
+
+function staticSitemap(): MetadataRoute.Sitemap {
+  return [
+    {
+      url: FRONTEND_URL,
+      lastModified: new Date(),
+      changeFrequency: 'hourly',
+      priority: 1.0,
+    },
+  ];
+}
+
+async function pillarSitemap(): Promise<MetadataRoute.Sitemap> {
+  const slugs = await fetchPillarSitemapSlugs();
+
+  return slugs.map(({ slug, lastModified }) => ({
+    url: `${FRONTEND_URL}/${slug}`,
+    lastModified: new Date(lastModified),
+    changeFrequency: 'hourly' as const,
+    priority: 0.8,
+  }));
+}
+
+async function jobSitemap(): Promise<MetadataRoute.Sitemap> {
+  const jobs = await fetchSitemapJobs();
+
+  return jobs.map((job) => ({
+    url: `${FRONTEND_URL}${job.href}`,
+    lastModified: job.lastModified,
+    changeFrequency: 'hourly' as const,
+    priority: 0.6,
+  }));
+}

src/features/jobs/server/data/fetch-sitemap-jobs.ts

@@ -0,0 +1,32 @@
+import 'server-only';
+
+import {
+  sitemapJobsResponseDto,
+  type SitemapJobDto,
+} from '@/features/jobs/server/dtos/sitemap-job.dto';
+import { clientEnv } from '@/lib/env/client';
+import { slugify } from '@/lib/server/utils';
+
+export const fetchSitemapJobs = async () => {
+  const url = `${clientEnv.MW_URL}/v2/search/sitemap/jobs`;
+  const response = await fetch(url, {
+    cache: 'force-cache',
+    next: { revalidate: 3600 },
+  });
+  if (!response.ok) throw new Error('Failed to fetch sitemap jobs');
+
+  const json = await response.json();
+  const parsed = sitemapJobsResponseDto.safeParse(json);
+  if (!parsed.success) throw new Error('Failed to parse sitemap jobs');
+
+  return parsed.data.data.map((job) => ({
+    href: createSitemapJobHref(job),
+    lastModified: new Date(job.timestamp),
+  }));
+};
+
+const createSitemapJobHref = (job: SitemapJobDto) => {
+  const title = job.title ?? 'Open Role';
+  const orgText = job.organizationName ? `-${job.organizationName}` : '';
+  return `/${slugify(`${title}${orgText}`)}/${job.shortUUID}`;
+};

src/features/jobs/server/data/index.ts

@@ -1,3 +1,4 @@
 export { fetchJobListPage } from './fetch-job-list-page';
 export { fetchJobDetails } from './fetch-job-details';
 export { fetchJobDetailsStaticParams } from './fetch-job-details-static-params';
+export { fetchSitemapJobs } from './fetch-sitemap-jobs';

src/features/jobs/server/dtos/sitemap-job.dto.ts

@@ -0,0 +1,15 @@
+import 'server-only';
+
+import { z } from 'zod';
+
+export const sitemapJobDto = z.object({
+  shortUUID: z.string(),
+  title: z.string().nullable(),
+  organizationName: z.string().nullable(),
+  timestamp: z.number(),
+});
+export type SitemapJobDto = z.infer<typeof sitemapJobDto>;
+
+export const sitemapJobsResponseDto = z.object({
+  data: sitemapJobDto.array().optional().default([]),
+});

src/features/pillar/server/data/fetch-pillar-sitemap-slugs.ts

@@ -0,0 +1,27 @@
+import 'server-only';
+
+import { pillarSitemapSlugsDto } from '@/features/pillar/server/dtos/pillar-sitemap-slugs.dto';
+import { clientEnv } from '@/lib/env/client';
+
+const LIMIT_LENGTH = 255;
+
+export const fetchPillarSitemapSlugs = async () => {
+  const url = `${clientEnv.MW_URL}/v2/search/sitemap/pillars`;
+  const response = await fetch(url, {
+    cache: 'force-cache',
+    next: { revalidate: 3600 },
+  });
+  if (!response.ok) throw new Error('Failed to fetch pillar sitemap slugs');
+
+  const json = await response.json();
+  const parsed = pillarSitemapSlugsDto.safeParse(json);
+  if (!parsed.success) throw new Error('Failed to parse pillar sitemap slugs');
+
+  return parsed.data.data.filter((item) => {
+    const isSafe = item.slug.length <= LIMIT_LENGTH;
+    if (!isSafe) {
+      console.warn(`[fetchPillarSitemapSlugs] Slug too long: ${item.slug}`);
+    }
+    return isSafe;
+  });
+};

src/features/pillar/server/data/fetch-pillar-static-params.ts

@@ -7,7 +7,7 @@ import { clientEnv } from '@/lib/env/client';
 const LIMIT_LENGTH = 255;
 
 export const fetchPillarStaticParams = async () => {
-  const url = `${clientEnv.MW_URL}/search/pillar/slugs?nav=jobs`;
+  const url = `${clientEnv.MW_URL}/v2/search/pillar/slugs`;
   const response = await fetch(url, {
     cache: 'force-cache',
     next: { revalidate: 3600 },
@@ -18,14 +18,13 @@ export const fetchPillarStaticParams = async () => {
   const parsed = pillarSlugsDto.safeParse(json);
   if (!parsed.success) throw new Error('Failed to parse static pillar slugs');
 
-  // Skip slugs that are too long
-  const filteredSlugs = parsed.data.filter((slug) => {
-    const isSafe = slug.length <= LIMIT_LENGTH;
-    if (!isSafe) {
-      console.warn(`[fetchStaticPillarSlugs] Slug is too long: ${slug}`);
-    }
-    return isSafe;
-  });
-
-  return filteredSlugs.map((slug) => ({ slug: getFrontendSlug(slug) }));
+  return parsed.data
+    .filter((slug) => {
+      const isSafe = slug.length <= LIMIT_LENGTH;
+      if (!isSafe) {
+        console.warn(`[fetchPillarStaticParams] Slug is too long: ${slug}`);
+      }
+      return isSafe;
+    })
+    .map((slug) => ({ slug: getFrontendSlug(slug) }));
 };

src/features/pillar/server/data/index.ts

@@ -1,2 +1,3 @@
 export { fetchPillarPageStatic } from './fetch-pillar-page-static';
 export { fetchPillarStaticParams } from './fetch-pillar-static-params';
+export { fetchPillarSitemapSlugs } from './fetch-pillar-sitemap-slugs';

src/features/pillar/server/dtos/pillar-sitemap-slugs.dto.ts

@@ -0,0 +1,13 @@
+import 'server-only';
+
+import { z } from 'zod';
+
+const pillarSitemapSlugDto = z.object({
+  slug: z.string(),
+  lastModified: z.string(),
+});
+
+export const pillarSitemapSlugsDto = z.object({
+  data: pillarSitemapSlugDto.array(),
+});
+export type PillarSitemapSlugsDto = z.infer<typeof pillarSitemapSlugsDto>;