Rewrite /category in middleware (#579)

This pr extends the middleware to rewrite `/blog/category/:path*` to our
Webflow Landing Page. Additionally, `sitemap.xml` got improved & fixed
by caching it statically on build time.

apps/web/src/app/sitemap.ts

@@ -3,6 +3,14 @@ import { XMLParser } from 'fast-xml-parser'
 import path from 'path'
 import { replaceUrls } from '@/utils/replaceUrls'
 import { getPageForSitemap } from '@/utils/sitemap'
+import {
+  landingPageHostname,
+  landingPageFramerHostname,
+  blogFramerHostname,
+  changelogFramerHostname,
+} from '@/app/hostnames'
+
+export const dynamic = 'force-static'
 
 type ChangeFrequency =
   | 'always'
@@ -22,17 +30,17 @@ type Site = {
 
 const sites: Site[] = [
   {
-    sitemapUrl: 'https://e2b-landing-page.framer.website/sitemap.xml',
+    sitemapUrl: `https://${landingPageHostname}/sitemap.xml`,
     priority: 1.0,
     changeFrequency: 'daily',
   },
   {
-    sitemapUrl: 'https://e2b-blog.framer.website/sitemap.xml',
+    sitemapUrl: `https://${blogFramerHostname}/sitemap.xml`,
     priority: 0.9,
     changeFrequency: 'daily',
   },
   {
-    sitemapUrl: 'https://e2b-changelog.framer.website/sitemap.xml',
+    sitemapUrl: `https://${changelogFramerHostname}/sitemap.xml`,
     priority: 0.2,
     changeFrequency: 'weekly',
   },
@@ -54,7 +62,7 @@ type Sitemap = {
 async function getXmlData(url: string): Promise<Sitemap> {
   const parser = new XMLParser()
 
-  const response = await fetch(url, { cache: 'no-cache' })
+  const response = await fetch(url)
 
   if (!response.ok) {
     return { urlset: { url: [] } }
@@ -64,29 +72,73 @@ async function getXmlData(url: string): Promise<Sitemap> {
 
   return parser.parse(text) as Sitemap
 }
-async function getSitemap(
-  site: Site,
-): Promise<MetadataRoute.Sitemap> {
+async function getSitemap(site: Site): Promise<MetadataRoute.Sitemap> {
   const data = await getXmlData(site.sitemapUrl)
 
   if (!data) {
     return []
   }
 
+  const normalizeUrl = (inputUrl: string, pathname: string) => {
+    // First normalize the URL format
+    let normalizedUrl = inputUrl
+      .replace(/^www\./, '') // Remove www. prefix
+      .replace(/https:\/\/https:\/\//, 'https://') // Fix double https://
+      .replace(/^https:\/\/www\./, 'https://') // Remove www. after https://
+
+    // Parse the URL to work with its components
+    const urlObj = new URL(normalizedUrl)
+
+    // Normalize category URLs to include /blog prefix
+    if (pathname.startsWith('/category/')) {
+      urlObj.pathname = `/blog${pathname}`
+    }
+
+    // Convert back to string for further processing
+    normalizedUrl = urlObj.toString()
+
+    // Apply replaceUrls after initial normalization
+    normalizedUrl = replaceUrls(normalizedUrl, urlObj.pathname)
+
+    // Ensure all URLs use e2b.dev domain
+    // Handle both www. and non-www variants
+    const hostnames = [
+      landingPageFramerHostname,
+      landingPageHostname,
+      changelogFramerHostname,
+      blogFramerHostname,
+    ]
+
+    for (const hostname of hostnames) {
+      normalizedUrl = normalizedUrl
+        .replace(`www.${hostname}`, 'e2b.dev')
+        .replace(hostname, 'e2b.dev')
+    }
+
+    // Final cleanup for any remaining double https:// or www.
+    return normalizedUrl
+      .replace(/https:\/\/https:\/\//, 'https://')
+      .replace(/^https:\/\/www\./, 'https://')
+  }
+
   if (Array.isArray(data.urlset.url)) {
     return data.urlset.url.map((line) => {
       const url = new URL(line.loc)
+      const pathname = url.pathname
+
       return {
-        url: replaceUrls(line.loc, url.pathname),
+        url: normalizeUrl(line.loc, pathname),
         priority: line?.priority || site.priority,
         changeFrequency: line?.changefreq || site.changeFrequency,
       }
     })
   } else {
     const url = new URL(data.urlset.url.loc)
+    const pathname = url.pathname
+
     return [
       {
-        url: replaceUrls(data.urlset.url.loc, url.pathname),
+        url: normalizeUrl(data.urlset.url.loc, pathname),
         priority: data.urlset.url?.priority || site.priority,
         changeFrequency: data.urlset.url?.changefreq || site.changeFrequency,
       },
@@ -97,14 +149,25 @@ async function getSitemap(
 export default async function sitemap(): Promise<MetadataRoute.Sitemap> {
   let mergedSitemap: MetadataRoute.Sitemap = []
 
-
-  const dashboardPath = path.join(process.cwd(), 'src', 'app', '(dashboard)', 'dashboard')
-  const dashboardPages = getPageForSitemap(dashboardPath, 'https://e2b.dev/dashboard/', 0.5)
+  const dashboardPath = path.join(
+    process.cwd(),
+    'src',
+    'app',
+    '(dashboard)',
+    'dashboard'
+  )
+  const dashboardPages = getPageForSitemap(
+    dashboardPath,
+    'https://e2b.dev/dashboard/',
+    0.5
+  )
 
   const docsDirectory = path.join(process.cwd(), 'src', 'app', '(docs)', 'docs')
-  const docsPages = getPageForSitemap(docsDirectory, 'https://e2b.dev/docs/', 0.5).filter(
-    (page) => !page.url.startsWith('https://e2b.dev/docs/api/'),
-  )
+  const docsPages = getPageForSitemap(
+    docsDirectory,
+    'https://e2b.dev/docs/',
+    0.5
+  ).filter((page) => !page.url.startsWith('https://e2b.dev/docs/api/'))
 
   mergedSitemap = mergedSitemap.concat(dashboardPages, docsPages)
 
@@ -113,5 +176,14 @@ export default async function sitemap(): Promise<MetadataRoute.Sitemap> {
     mergedSitemap = mergedSitemap.concat(...urls)
   }
 
-  return mergedSitemap.sort((a, b) => a.url.localeCompare(b.url))
+  // Deduplicate URLs, keeping the entry with highest priority
+  const urlMap = new Map<string, MetadataRoute.Sitemap[0]>()
+  for (const entry of mergedSitemap) {
+    const existing = urlMap.get(entry.url)
+    if (!existing || (existing.priority || 0) < (entry.priority || 0)) {
+      urlMap.set(entry.url, entry)
+    }
+  }
+
+  return Array.from(urlMap.values()).sort((a, b) => a.url.localeCompare(b.url))
 }

apps/web/src/middleware.ts

@@ -1,9 +1,6 @@
 import { NextRequest, NextResponse } from 'next/server'
 import { replaceUrls } from '@/utils/replaceUrls'
-import {
-  landingPageHostname,
-  landingPageFramerHostname,
-} from '@/app/hostnames'
+import { landingPageHostname, landingPageFramerHostname } from '@/app/hostnames'
 
 export async function middleware(req: NextRequest): Promise<NextResponse> {
   if (req.method !== 'GET') return NextResponse.next()
@@ -42,7 +39,13 @@ export async function middleware(req: NextRequest): Promise<NextResponse> {
   }
 
   if (url.pathname.startsWith('/blog')) {
+    const segments = url.pathname.split('/')
+
     url.hostname = landingPageHostname
+
+    if (segments[2] === 'category') {
+      url.pathname = segments.slice(2).join('/')
+    }
   }
 
   // TODO: Not on the new landing page hosting yet
@@ -78,4 +81,3 @@ export const config = {
     '/cookbook/:path*',
   ],
 }
-

package.json

@@ -6,7 +6,8 @@
     "publish": "changeset publish && pnpm run -r postPublish",
     "test": "pnpm test --recursive --if-present",
     "dev:web": "pnpm --prefix apps/web run dev",
-    "dev:build": "pnpm --prefix apps/web run build",
+    "build:web": "pnpm --prefix apps/web run build",
+    "start:web": "pnpm --prefix apps/web run start",
     "rm-node-modules": "find . -name 'node_modules' -type d -prune -exec rm -rf '{}' +",
     "pnpm-install-hack": "cd packages/js-sdk && sed -i '' 's/\"version\": \".*\"/\"version\": \"9.9.9\"/g' package.json && cd ../.. && pnpm i && git checkout -- packages/js-sdk/package.json",
     "generate-sdk-reference": "pnpm --if-present --recursive run generate-sdk-reference"