Rename endpoints to be the same as job IDs

api/_common/middleware.js

@@ -27,7 +27,6 @@ const commonMiddleware = (handler) => {
         });
       }
     } catch (error) {
-      console.log(error);
       callback(null, {
         statusCode: 500,
         body: JSON.stringify({ error: error.message }),

api/read-robots-txt.js

@@ -1,45 +0,0 @@
-const axios = require('axios');
-
-exports.handler = async function(event, context) {
-  const siteURL = (event.queryStringParameters || event.query).url;
-
-  if (!siteURL) {
-    return {
-      statusCode: 400,
-      body: JSON.stringify({ error: 'Missing url query parameter' }),
-    };
-  }
-
-  let parsedURL;
-  try {
-    parsedURL = new URL(siteURL);
-  } catch (error) {
-    return {
-      statusCode: 400,
-      body: JSON.stringify({ error: 'Invalid url query parameter' }),
-    };
-  }
-
-  const robotsURL = `${parsedURL.protocol}//${parsedURL.hostname}/robots.txt`;
-
-  try {
-    const response = await axios.get(robotsURL);
-
-    if (response.status === 200) {
-      return {
-        statusCode: 200,
-        body: response.data,
-      };
-    } else {
-      return {
-        statusCode: response.status,
-        body: JSON.stringify({ error: 'Failed to fetch robots.txt', statusCode: response.status }),
-      };
-    }
-  } catch (error) {
-    return {
-      statusCode: 500,
-      body: JSON.stringify({ error: `Error fetching robots.txt: ${error.message}` }),
-    };
-  }
-};

api/robots-txt.js

@@ -0,0 +1,66 @@
+const axios = require('axios');
+const middleware = require('./_common/middleware');
+
+const parseRobotsTxt = (content) => {
+  const lines = content.split('\n');
+  const rules = [];
+
+  lines.forEach(line => {
+    line = line.trim();  // This removes trailing and leading whitespaces
+
+    let match = line.match(/^(Allow|Disallow):\s*(\S*)$/i);
+    if (match) {
+      const rule = {
+        lbl: match[1],
+        val: match[2],
+      };
+      
+      rules.push(rule);
+    } else {
+      match = line.match(/^(User-agent):\s*(\S*)$/i);
+      if (match) {
+        const rule = {
+          lbl: match[1],
+          val: match[2],
+        };
+        
+        rules.push(rule);
+      }
+    }
+  });
+  return { robots: rules };
+}
+
+const handler = async function(url) {
+  let parsedURL;
+  try {
+    parsedURL = new URL(url);
+  } catch (error) {
+    return {
+      statusCode: 400,
+      body: JSON.stringify({ error: 'Invalid url query parameter' }),
+    };
+  }
+
+  const robotsURL = `${parsedURL.protocol}//${parsedURL.hostname}/robots.txt`;
+
+  try {
+    const response = await axios.get(robotsURL);
+
+    if (response.status === 200) {
+      return parseRobotsTxt(response.data);
+    } else {
+      return {
+        statusCode: response.status,
+        body: JSON.stringify({ error: 'Failed to fetch robots.txt', statusCode: response.status }),
+      };
+    }
+  } catch (error) {
+    return {
+      statusCode: 500,
+      body: JSON.stringify({ error: `Error fetching robots.txt: ${error.message}` }),
+    };
+  }
+};
+
+exports.handler = middleware(handler);