Merge pull request #137 from techulus/develop

Add sitemap to robots.txt

Author: arjunkomath

apps/page/pages/api/robots.ts

@@ -1,10 +1,12 @@
 import type { NextApiRequest, NextApiResponse } from "next";
 import { fetchRenderData, translateHostToPageIdentifier } from "../../lib/data";
 
-const ALLOW = `User-agent: *
+const getAllowRobots = (hostname: string) => `User-agent: *
 Allow: /
 Disallow: /api/
-Disallow: /_next/`;
+Disallow: /_next/
+
+Sitemap: https://${hostname}/sitemap.xml`;
 
 const DISALLOW = `User-agent: *
 Disallow: /`;
@@ -16,6 +18,7 @@ async function handler(
   const hostname = String(req?.headers?.host);
 
   const { domain, page: url_slug } = translateHostToPageIdentifier(hostname);
+  const pageUrl = domain ?? `${url_slug}.changes.page`;
 
   res.setHeader("Content-Type", "text/plain; charset=UTF-8");
 
@@ -27,7 +30,9 @@ async function handler(
     if (!page) throw new Error("Page not found");
     if (!settings) throw new Error("Settings not found");
 
-    res.status(200).send(settings?.hide_search_engine ? DISALLOW : ALLOW);
+    res
+      .status(200)
+      .send(settings?.hide_search_engine ? DISALLOW : getAllowRobots(pageUrl));
   } catch (e: unknown) {
     console.log("robots.txt [Error]", e);
     res.status(200).send(DISALLOW);