Update Dockerfile (#43)

Updates the Dockerfile with changes suggested by @GWnbsp  in https://github.com/Lissy93/web-check/issues/43#issuecomment-1719212234

### Summary of Changes
1. **ARG Statements:** Introduced `ARG` statements for Node.js and Debian versions, making the Dockerfile more customizable.
2. **SHELL Command:** Changed the default shell to Bash with certain options enabled, improving robustness.
3. **Chromium Installation:** Updated Chromium installation to use Google's signing keys and repositories, aiming for more secure and up-to-date packages.
4. **Chromium Version:** Added a step to save Chromium's version into `/etc/chromium-version` for reference.
5. **Directory Creation:** Added a new directory /app/data in the container's filesystem.
6. **CMD Change:** Changed the CMD to start Node.js server (server.js) instead of using yarn serve.
7. **General Cleanup and Comments:** Code has been refactored for better readability and detailed comments have been added for clarity.
8. **Dependency Installation:** Kept yarn install and the removal of the Yarn cache, but the command is more streamlined.
9. **Other Minor Changes:**
  - Added flags like `-qq` and `--no-install-recommends` for quieter and optimized installation.
  - Enhanced cleanup with `rm -rf /var/lib/apt/lists/*.`

.env

@@ -2,11 +2,24 @@
 # Be sure to uncomment any line you populate
 # Everything is optional, but some features won't work without external API access
 
-# GOOGLE_CLOUD_API_KEY=''
-# SHODAN_API_KEY=''
-# REACT_APP_SHODAN_API_KEY=''
-# WHO_API_KEY=''
-# REACT_APP_WHO_API_KEY=''
-# SECURITY_TRAILS_API_KEY=''
-# BUILT_WITH_API_KEY=''
-# CI=false
+# API Keys for external services (backend)
+GOOGLE_CLOUD_API_KEY=''
+TORRENT_IP_API_KEY=''
+SECURITY_TRAILS_API_KEY=''
+BUILT_WITH_API_KEY=''
+URL_SCAN_API_KEY=''
+TRANCO_USERNAME=''
+TRANCO_API_KEY=''
+CLOUDMERSIVE_API_KEY=''
+
+# API Keys for external services (frontend)
+REACT_APP_SHODAN_API_KEY=''
+REACT_APP_WHO_API_KEY=''
+
+# Configuration settings
+# CHROME_PATH='/usr/bin/chromium' # The path the the Chromium executable
+# PORT='3000' # Port to serve the API, when running server.js
+# DISABLE_GUI='false' # Disable the GUI, and only serve the API
+# API_TIMEOUT_LIMIT='10000' # The timeout limit for API requests, in milliseconds
+# API_CORS_ORIGIN='*' # Enable CORS, by setting your allowed hostname(s) here
+# REACT_APP_API_ENDPOINT='/api' # The endpoint for the API (can be local or remote)

.github/screenshots/README.md

@@ -0,0 +1 @@
+![Screenshot](https://raw.githubusercontent.com/Lissy93/web-check/HEAD/.github/screenshots/web-check-screenshot3.png)

.github/workflows/credits.yml

@@ -0,0 +1,37 @@
+# Inserts list of community members into ./README.md
+name: 💓 Inserts Contributors & Sponsors
+on:
+  workflow_dispatch: # Manual dispatch
+  schedule:
+    - cron: '45 1 * * 0' # At 01:45 on Sunday.
+
+jobs:
+  # Job #1 - Fetches sponsors and inserts table into readme
+  insert-sponsors:
+    runs-on: ubuntu-latest
+    name: Inserts Sponsors 💓
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Updates readme with sponsors
+        uses: JamesIves/github-sponsors-readme-action@1.0.5
+        with:
+          token: ${{ secrets.BOT_TOKEN || secrets.GITHUB_TOKEN }}
+          file: .github/README.md
+
+  # Job #2 - Fetches contributors and inserts table into readme
+  insert-contributors:
+    runs-on: ubuntu-latest
+    name: Inserts Contributors 💓
+    steps:
+      - name: Updates readme with contributors
+        uses: akhilmhdh/contributors-readme-action@v2.3.4
+        env:
+          GITHUB_TOKEN: ${{ secrets.BOT_TOKEN || secrets.GITHUB_TOKEN }}
+        with:
+          image_size: 80
+          readme_path: .github/README.md
+          columns_per_row: 6
+          commit_message: 'docs: Updates contributors list'
+          committer_username: liss-bot
+          committer_email: liss-bot@d0h.co

.github/workflows/deploy-aws.yml

@@ -0,0 +1,128 @@
+name: 🚀 Deploy to AWS
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - master
+    tags:
+      - '*'
+    paths:
+      - api/**
+      - serverless.yml
+      - package.json
+      - .github/workflows/deploy-aws.yml
+
+jobs:
+  deploy-api:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v2
+
+    - name: Setup Node.js
+      uses: actions/setup-node@v1
+      with:
+        node-version: 16
+
+    - name: Cache node_modules
+      uses: actions/cache@v2
+      with:
+        path: node_modules
+        key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
+        restore-keys: |
+          ${{ runner.os }}-yarn-
+
+    - name: Create GitHub deployment for API
+      uses: chrnorm/deployment-action@releases/v1
+      id: deployment_api
+      with:
+        token: ${{ secrets.BOT_TOKEN || secrets.GITHUB_TOKEN }}
+        environment: AWS (Backend API)
+        ref: ${{ github.ref }}
+      
+    - name: Install Serverless CLI and dependencies
+      run: |
+        npm i -g serverless
+        yarn
+
+    - name: Deploy to AWS
+      env:
+        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }}
+      run: serverless deploy
+      
+    - name: Update GitHub deployment status (API)
+      if: always()
+      uses: chrnorm/deployment-status@releases/v1
+      with:
+        token: ${{ secrets.BOT_TOKEN || secrets.GITHUB_TOKEN }}
+        state: "${{ job.status }}"
+        deployment_id: ${{ steps.deployment_api.outputs.deployment_id }}
+        ref: ${{ github.ref }}
+
+  deploy-frontend:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v2
+
+    - name: Setup Node.js
+      uses: actions/setup-node@v1
+      with:
+        node-version: 16
+    
+    - name: Cache node_modules
+      uses: actions/cache@v2
+      with:
+        path: node_modules
+        key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
+        restore-keys: |
+          ${{ runner.os }}-yarn-
+
+    - name: Create GitHub deployment for Frontend
+      uses: chrnorm/deployment-action@releases/v1
+      id: deployment_frontend
+      with:
+        token: ${{ secrets.BOT_TOKEN || secrets.GITHUB_TOKEN }}
+        environment: AWS (Frontend Web UI)
+        ref: ${{ github.ref }}
+        
+    - name: Install dependencies and build
+      run: |
+        yarn install
+        yarn build
+    
+    - name: Setup AWS
+      uses: aws-actions/configure-aws-credentials@v1
+      with:
+        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        aws-region: us-east-1
+
+    - name: Upload to S3
+      env:
+        AWS_S3_BUCKET: 'web-check-frontend'
+      run: aws s3 sync ./build/ s3://$AWS_S3_BUCKET/ --delete
+
+    - name: Invalidate CloudFront cache
+      uses: chetan/invalidate-cloudfront-action@v2.4
+      env:
+        DISTRIBUTION: E30XKAM2TG9FD8
+        PATHS: '/*'
+        AWS_REGION: 'us-east-1'
+        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        
+    - name: Update GitHub deployment status (Frontend)
+      if: always()
+      uses: chrnorm/deployment-status@releases/v1
+      with:
+        token: ${{ secrets.BOT_TOKEN || secrets.GITHUB_TOKEN }}
+        state: "${{ job.status }}"
+        deployment_id: ${{ steps.deployment_frontend.outputs.deployment_id }}
+        ref: ${{ github.ref }}
+

.gitignore

@@ -1,28 +1,60 @@
-
-# Keys
+# ------------------------
+#   ENVIRONMENT SETTINGS
+# ------------------------
 .env
 
-# dependencies
-/node_modules
-/.pnp
-.pnp.js
+# ------------------------
+#       PRODUCTION
+# ------------------------
+/build/
 
-# testing
-/coverage
+# ------------------------
+#         DEPLOYMENT
+# ------------------------
+.vercel/
+.netlify/
+.webpack/
+.serverless/
 
-# production
-/build
-
-# misc
-.DS_Store
-.env.local
-.env.development.local
-.env.test.local
-.env.production.local
+# ------------------------
+#         DEPENDENCIES
+# ------------------------
+node_modules/
+.yarn/cache/
+.yarn/unplugged/
+.yarn/build-state.yml
+.yarn/install-state.gz
+.pnpm/
+.pnp.*
 
+# ------------------------
+#            LOGS
+# ------------------------
+logs/
+*.log
 npm-debug.log*
 yarn-debug.log*
 yarn-error.log*
+lerna-debug.log*
+.pnpm-debug.log*
+
+# ------------------------
+#          TESTING
+# ------------------------
+coverage/
+.nyc_output/
+
+# ------------------------
+#       OS SPECIFIC
+# ------------------------
+.DS_Store
+Thumbs.db
+
+# ------------------------
+#         EDITORS
+# ------------------------
+.idea/
+.vscode/
+*.swp
+*.swo
 
-# Local Netlify folder
-.netlify

Dockerfile

@@ -1,12 +1,48 @@
-FROM node:16-buster-slim AS base
-WORKDIR /app
-FROM base AS builder
-COPY . .
-RUN apt-get update && \
-    apt-get install -y chromium traceroute && \
-    chmod 755 /usr/bin/chromium && \
+# Specify the Node.js version to use
+ARG NODE_VERSION=16
+
+# Specify the Debian version to use, the default is "bullseye"
+ARG DEBIAN_VERSION=bullseye
+
+# Use Node.js Docker image as the base image, with specific Node and Debian versions
+FROM docker.io/library/node:${NODE_VERSION}-${DEBIAN_VERSION}
+
+# Set the container's default shell to Bash and enable some options
+SHELL ["/bin/bash", "-euo", "pipefail", "-c"]
+
+# Install Chromium browser and Download and verify Google Chrome’s signing key
+RUN apt-get update -qq --fix-missing && \
+    apt-get -qqy install --allow-unauthenticated gnupg wget && \
+    wget --quiet --output-document=- https://dl-ssl.google.com/linux/linux_signing_key.pub | gpg --dearmor > /etc/apt/trusted.gpg.d/google-archive.gpg && \
+    echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" > /etc/apt/sources.list.d/google.list && \
+    apt-get update -qq && \
+    apt-get -qqy --no-install-recommends install chromium traceroute && \
     rm -rf /var/lib/apt/lists/* 
-RUN npm install --force
-EXPOSE 8888
+
+# Run the Chromium browser's version command and redirect its output to the /etc/chromium-version file
+RUN /usr/bin/chromium --no-sandbox --version > /etc/chromium-version
+
+# Set the working directory to /app
+WORKDIR /app
+
+# Copy package.json and yarn.lock to the working directory
+COPY package.json yarn.lock ./
+
+# Run yarn install to install dependencies and clear yarn cache
+RUN yarn install && rm -rf /app/node_modules/.cache
+
+# Copy all files to working directory
+COPY . .
+RUN mkdir /app/data
+
+# Run yarn build to build the application
+RUN yarn build
+
+# Exposed container port, the default is 3000, which can be modified through the environment variable PORT
+EXPOSE ${PORT:-3000}
+
+# Set the environment variable CHROME_PATH to specify the path to the Chromium binaries
 ENV CHROME_PATH='/usr/bin/chromium'
-CMD ["npm", "run", "serve"]
+
+# Define the command executed when the container starts and start the server.js of the Node.js application
+CMD [ "node", "server.js" ]

api/_common/aws-webpack.config.js

@@ -0,0 +1,51 @@
+const path = require('path');
+const nodeExternals = require('webpack-node-externals');
+
+module.exports = {
+    target: 'node',
+    mode: 'production',
+    entry: {
+        'carbon': './api/carbon.js',
+        'cookies': './api/cookies.js',
+        'dns-server': './api/dns-server.js',
+        'dns': './api/dns.js',
+        'dnssec': './api/dnssec.js',
+        'features': './api/features.js',
+        'get-ip': './api/get-ip.js',
+        'headers': './api/headers.js',
+        'hsts': './api/hsts.js',
+        'linked-pages': './api/linked-pages.js',
+        'mail-config': './api/mail-config.js',
+        'ports': './api/ports.js',
+        'quality': './api/quality.js',
+        'redirects': './api/redirects.js',
+        'robots-txt': './api/robots-txt.js',
+        'screenshot': './api/screenshot.js',
+        'security-txt': './api/security-txt.js',
+        'sitemap': './api/sitemap.js',
+        'social-tags': './api/social-tags.js',
+        'ssl': './api/ssl.js',
+        'status': './api/status.js',
+        'tech-stack': './api/tech-stack.js',
+        'trace-route': './api/trace-route.js',
+        'txt-records': './api/txt-records.js',
+        'whois': './api/whois.js',
+    },
+    externals: [nodeExternals()],
+    output: {
+        filename: '[name].js',
+        path: path.resolve(__dirname, '.webpack'),
+        libraryTarget: 'commonjs2'
+    },
+    module: {
+        rules: [
+            {
+                test: /\.js$/,
+                use: {
+                    loader: 'babel-loader'
+                },
+                exclude: /node_modules/,
+            }
+        ]
+    }
+};

api/_common/middleware.js

@@ -0,0 +1,82 @@
+const normalizeUrl = (url) => {
+  return url.startsWith('http') ? url : `https://${url}`;
+};
+
+const headers = {
+  'Access-Control-Allow-Origin': process.env.API_CORS_ORIGIN || '*',
+  'Access-Control-Allow-Credentials': true,
+  'Content-Type': 'application/json;charset=UTF-8',
+};
+
+const commonMiddleware = (handler) => {
+  // Vercel
+  const vercelHandler = async (request, response) => {
+    const queryParams = request.query || {};
+    const rawUrl = queryParams.url;
+
+    if (!rawUrl) {
+      return response.status(500).json({ error: 'No URL specified' });
+    }
+
+    const url = normalizeUrl(rawUrl);
+
+    try {
+      const handlerResponse = await handler(url, request);
+      if (handlerResponse.body && handlerResponse.statusCode) {
+        response.status(handlerResponse.statusCode).json(handlerResponse.body);
+      } else {
+        response.status(200).json(
+          typeof handlerResponse === 'object' ? handlerResponse : JSON.parse(handlerResponse)
+        );
+      }
+    } catch (error) {
+      response.status(500).json({ error: error.message });
+    }
+  };
+
+  // Netlify
+  const netlifyHandler = async (event, context, callback) => {
+    const queryParams = event.queryStringParameters || event.query || {};
+    const rawUrl = queryParams.url;
+
+    if (!rawUrl) {
+      callback(null, {
+        statusCode: 500,
+        body: JSON.stringify({ error: 'No URL specified' }),
+        headers,
+      });
+      return;
+    }
+
+    const url = normalizeUrl(rawUrl);
+
+    try {
+      const handlerResponse = await handler(url, event, context);
+      if (handlerResponse.body && handlerResponse.statusCode) {
+        callback(null, handlerResponse);
+      } else {
+        callback(null, {
+          statusCode: 200,
+          body: typeof handlerResponse === 'object' ? JSON.stringify(handlerResponse) : handlerResponse,
+          headers,
+        });
+      }
+    } catch (error) {
+      callback(null, {
+        statusCode: 500,
+        body: JSON.stringify({ error: error.message }),
+        headers,
+      });
+    }
+  };
+
+  // The format of the handlers varies between platforms
+  // E.g. Netlify + AWS expect Lambda functions, but Vercel or Node needs standard handler
+  const platformEnv = (process.env.PLATFORM || '').toUpperCase(); // Has user set platform manually?
+
+  const nativeMode = (['VERCEL', 'NODE'].includes(platformEnv) || process.env.VERCEL || process.env.WC_SERVER);
+
+  return nativeMode ? vercelHandler : netlifyHandler;
+};
+
+module.exports = commonMiddleware;

api/archives.js

@@ -0,0 +1,84 @@
+const axios = require('axios');
+const middleware = require('./_common/middleware');
+
+const convertTimestampToDate = (timestamp) => {
+  const [year, month, day, hour, minute, second] = [
+    timestamp.slice(0, 4),
+    timestamp.slice(4, 6) - 1,
+    timestamp.slice(6, 8),
+    timestamp.slice(8, 10),
+    timestamp.slice(10, 12),
+    timestamp.slice(12, 14),
+  ].map(num => parseInt(num, 10));
+
+  return new Date(year, month, day, hour, minute, second);
+}
+
+const countPageChanges = (results) => {
+  let prevDigest = null;
+  return results.reduce((acc, curr) => {
+    if (curr[2] !== prevDigest) {
+      prevDigest = curr[2];
+      return acc + 1;
+    }
+    return acc;
+  }, -1);
+}
+
+const getAveragePageSize = (scans) => {
+    const totalSize = scans.map(scan => parseInt(scan[3], 10)).reduce((sum, size) => sum + size, 0);
+    return Math.round(totalSize / scans.length);
+};
+
+const getScanFrequency = (firstScan, lastScan, totalScans, changeCount) => {
+  const formatToTwoDecimal = num => parseFloat(num.toFixed(2));
+
+  const dayFactor = (lastScan - firstScan) / (1000 * 60 * 60 * 24);  
+  const daysBetweenScans = formatToTwoDecimal(dayFactor / totalScans);
+  const daysBetweenChanges = formatToTwoDecimal(dayFactor / changeCount);
+  const scansPerDay = formatToTwoDecimal((totalScans - 1) / dayFactor);
+  const changesPerDay = formatToTwoDecimal(changeCount / dayFactor);
+  return {
+    daysBetweenScans,
+    daysBetweenChanges,
+    scansPerDay,
+    changesPerDay,
+  };
+};
+
+const getWaybackData = async (url) => {
+  const cdxUrl = `https://web.archive.org/cdx/search/cdx?url=${url}&output=json&fl=timestamp,statuscode,digest,length,offset`;
+
+  try {
+    const { data } = await axios.get(cdxUrl);
+    
+    // Check there's data
+    if (!data || !Array.isArray(data) || data.length <= 1) {
+      return { skipped: 'Site has never before been archived via the Wayback Machine' };
+    }
+
+    // Remove the header row
+    data.shift();
+
+    // Process and return the results
+    const firstScan = convertTimestampToDate(data[0][0]);
+    const lastScan = convertTimestampToDate(data[data.length - 1][0]);
+    const totalScans = data.length;
+    const changeCount = countPageChanges(data);
+    return {
+      firstScan,
+      lastScan,
+      totalScans,
+      changeCount,
+      averagePageSize: getAveragePageSize(data),
+      scanFrequency: getScanFrequency(firstScan, lastScan, totalScans, changeCount),
+      scans: data,
+      scanUrl: url,
+    };
+  } catch (err) {
+    return { error: `Error fetching Wayback data: ${err.message}` };
+  }
+};
+
+module.exports = middleware(getWaybackData);
+module.exports.handler = middleware(getWaybackData);

api/block-lists.js

@@ -0,0 +1,105 @@
+const dns = require('dns');
+const { URL } = require('url');
+const middleware = require('./_common/middleware');
+
+const DNS_SERVERS = [
+  { name: 'AdGuard', ip: '176.103.130.130' },
+  { name: 'AdGuard Family', ip: '176.103.130.132' },
+  { name: 'CleanBrowsing Adult', ip: '185.228.168.10' },
+  { name: 'CleanBrowsing Family', ip: '185.228.168.168' },
+  { name: 'CleanBrowsing Security', ip: '185.228.168.9' },
+  { name: 'CloudFlare', ip: '1.1.1.1' },
+  { name: 'CloudFlare Family', ip: '1.1.1.3' },
+  { name: 'Comodo Secure', ip: '8.26.56.26' },
+  { name: 'Google DNS', ip: '8.8.8.8' },
+  { name: 'Neustar Family', ip: '156.154.70.3' },
+  { name: 'Neustar Protection', ip: '156.154.70.2' },
+  { name: 'Norton Family', ip: '199.85.126.20' },
+  { name: 'OpenDNS', ip: '208.67.222.222' },
+  { name: 'OpenDNS Family', ip: '208.67.222.123' },
+  { name: 'Quad9', ip: '9.9.9.9' },
+  { name: 'Yandex Family', ip: '77.88.8.7' },
+  { name: 'Yandex Safe', ip: '77.88.8.88' },
+];
+const knownBlockIPs = [
+  '146.112.61.106', // OpenDNS
+  '185.228.168.10', // CleanBrowsing
+  '8.26.56.26',     // Comodo
+  '9.9.9.9',        // Quad9
+  '208.69.38.170',  // Some OpenDNS IPs
+  '208.69.39.170',  // Some OpenDNS IPs
+  '208.67.222.222', // OpenDNS
+  '208.67.222.123', // OpenDNS FamilyShield
+  '199.85.126.10',  // Norton
+  '199.85.126.20',  // Norton Family
+  '156.154.70.22',  // Neustar
+  '77.88.8.7',      // Yandex
+  '77.88.8.8',      // Yandex
+  '::1',              // Localhost IPv6
+  '2a02:6b8::feed:0ff', // Yandex DNS
+  '2a02:6b8::feed:bad', // Yandex Safe
+  '2a02:6b8::feed:a11', // Yandex Family
+  '2620:119:35::35',    // OpenDNS
+  '2620:119:53::53',    // OpenDNS FamilyShield
+  '2606:4700:4700::1111', // Cloudflare
+  '2606:4700:4700::1001', // Cloudflare
+  '2001:4860:4860::8888', // Google DNS
+  '2a0d:2a00:1::',        // AdGuard
+  '2a0d:2a00:2::'         // AdGuard Family
+];
+
+const isDomainBlocked = async (domain, serverIP) => {
+  return new Promise((resolve) => {
+    dns.resolve4(domain, { server: serverIP }, (err, addresses) => {
+      if (!err) {
+        if (addresses.some(addr => knownBlockIPs.includes(addr))) {
+          resolve(true);
+          return;
+        }
+        resolve(false);
+        return;
+      }
+
+      dns.resolve6(domain, { server: serverIP }, (err6, addresses6) => {
+        if (!err6) {
+          if (addresses6.some(addr => knownBlockIPs.includes(addr))) {
+            resolve(true);
+            return;
+          }
+          resolve(false);
+          return;
+        }
+        if (err6.code === 'ENOTFOUND' || err6.code === 'SERVFAIL') {
+          resolve(true);
+        } else {
+          resolve(false);
+        }
+      });
+    });
+  });
+};
+
+const checkDomainAgainstDnsServers = async (domain) => {
+  let results = [];
+
+  for (let server of DNS_SERVERS) {
+    const isBlocked = await isDomainBlocked(domain, server.ip);
+    results.push({
+      server: server.name,
+      serverIp: server.ip,
+      isBlocked,
+    });
+  }
+
+  return results;
+};
+
+const handler = async (url) => {
+  const domain = new URL(url).hostname;
+  const results = await checkDomainAgainstDnsServers(domain);
+  return { blocklists: results };
+};
+
+module.exports = middleware(handler);
+module.exports.handler = middleware(handler);
+

api/carbon.js

@@ -1,14 +1,7 @@
 const https = require('https');
+const middleware = require('./_common/middleware');
 
-exports.handler = async (event, context) => {
-  const { url } = event.queryStringParameters;
-
-  if (!url) {
-    return {
-      statusCode: 400,
-      body: JSON.stringify({ error: 'url query parameter is required' }),
-    };
-  }
+const handler = async (url) => {
 
   // First, get the size of the website's HTML
   const getHtmlSize = (url) => new Promise((resolve, reject) => {
@@ -49,14 +42,11 @@ exports.handler = async (event, context) => {
     }
 
     carbonData.scanUrl = url;
-    return {
-      statusCode: 200,
-      body: JSON.stringify(carbonData),
-    };
+    return carbonData;
   } catch (error) {
-    return {
-      statusCode: 500,
-      body: JSON.stringify({ error: `Error: ${error.message}` }),
-    };
+    throw new Error(`Error: ${error.message}`);
   }
 };
+
+module.exports = middleware(handler);
+module.exports.handler = middleware(handler);

api/content-links.js

@@ -1,60 +0,0 @@
-const axios = require('axios');
-const cheerio = require('cheerio');
-const urlLib = require('url');
-
-exports.handler = async (event, context) => {
-  let url = event.queryStringParameters.url;
-  
-  // Check if url includes protocol
-  if (!url.startsWith('http://') && !url.startsWith('https://')) {
-    url = 'http://' + url;
-  }
-  
-  try {
-    const response = await axios.get(url);
-    const html = response.data;
-    const $ = cheerio.load(html);
-    const internalLinksMap = new Map();
-    const externalLinksMap = new Map();
-
-    $('a[href]').each((i, link) => {
-      const href = $(link).attr('href');
-      const absoluteUrl = urlLib.resolve(url, href);
-      
-      if (absoluteUrl.startsWith(url)) {
-        const count = internalLinksMap.get(absoluteUrl) || 0;
-        internalLinksMap.set(absoluteUrl, count + 1);
-      } else if (href.startsWith('http://') || href.startsWith('https://')) {
-        const count = externalLinksMap.get(absoluteUrl) || 0;
-        externalLinksMap.set(absoluteUrl, count + 1);
-      }
-    });
-
-    // Convert maps to sorted arrays
-    const internalLinks = [...internalLinksMap.entries()].sort((a, b) => b[1] - a[1]).map(entry => entry[0]);
-    const externalLinks = [...externalLinksMap.entries()].sort((a, b) => b[1] - a[1]).map(entry => entry[0]);
-
-    if (internalLinks.length === 0 && externalLinks.length === 0) {
-      return {
-        statusCode: 400,
-        body: JSON.stringify({
-          skipped: 'No internal or external links found. '
-            + 'This may be due to the website being dynamically rendered, using a client-side framework (like React), and without SSR enabled. '
-            + 'That would mean that the static HTML returned from the HTTP request doesn\'t contain any meaningful content for Web-Check to analyze. '
-            + 'You can rectify this by using a headless browser to render the page instead.',
-          }),
-      };
-    }
-
-    return {
-      statusCode: 200,
-      body: JSON.stringify({ internal: internalLinks, external: externalLinks }),
-    };
-  } catch (error) {
-    console.log(error);
-    return {
-      statusCode: 500,
-      body: JSON.stringify({ error: 'Failed fetching data' }),
-    };
-  }
-};

api/cookies.js

@@ -0,0 +1,58 @@
+const axios = require('axios');
+const puppeteer = require('puppeteer');
+const middleware = require('./_common/middleware');
+
+const getPuppeteerCookies = async (url) => {
+  const browser = await puppeteer.launch({
+    headless: 'new',
+    args: ['--no-sandbox', '--disable-setuid-sandbox'],
+  });
+
+  try {
+    const page = await browser.newPage();
+    const navigationPromise = page.goto(url, { waitUntil: 'networkidle2' });
+        const timeoutPromise = new Promise((_, reject) => 
+      setTimeout(() => reject(new Error('Puppeteer took too long!')), 3000)
+    );
+    await Promise.race([navigationPromise, timeoutPromise]);
+    return await page.cookies();
+  } finally {
+    await browser.close();
+  }
+};
+
+const handler = async (url) => {
+  let headerCookies = null;
+  let clientCookies = null;
+
+  try {
+    const response = await axios.get(url, {
+      withCredentials: true,
+      maxRedirects: 5,
+    });
+    headerCookies = response.headers['set-cookie'];
+  } catch (error) {
+    if (error.response) {
+      return { error: `Request failed with status ${error.response.status}: ${error.message}` };
+    } else if (error.request) {
+      return { error: `No response received: ${error.message}` };
+    } else {
+      return { error: `Error setting up request: ${error.message}` };
+    }
+  }
+
+  try {
+    clientCookies = await getPuppeteerCookies(url);
+  } catch (_) {
+    clientCookies = null;
+  }
+
+  if (!headerCookies && (!clientCookies || clientCookies.length === 0)) {
+    return { skipped: 'No cookies' };
+  }
+
+  return { headerCookies, clientCookies };
+};
+
+module.exports = middleware(handler);
+module.exports.handler = middleware(handler);

api/dns-server.js

@@ -1,11 +1,11 @@
 const dns = require('dns');
 const dnsPromises = dns.promises;
-// const https = require('https');
 const axios = require('axios');
+const middleware = require('./_common/middleware');
 
-exports.handler = async (event) => {
-  const domain = event.queryStringParameters.url.replace(/^(?:https?:\/\/)?/i, "");
+const handler = async (url) => {
   try {
+    const domain = url.replace(/^(?:https?:\/\/)?/i, "");
     const addresses = await dnsPromises.resolve4(domain);
     const results = await Promise.all(addresses.map(async (address) => {
       const hostname = await dnsPromises.reverse(address).catch(() => null);
@@ -22,6 +22,7 @@ exports.handler = async (event) => {
         dohDirectSupports,
       };
     }));
+
     // let dohMozillaSupport = false;
     // try {
     //   const mozillaList = await axios.get('https://firefox.settings.services.mozilla.com/v1/buckets/security-state/collections/onecrl/records');
@@ -29,20 +30,16 @@ exports.handler = async (event) => {
     // } catch (error) {
     //   console.error(error);
     // }
+
     return {
-      statusCode: 200,
-      body: JSON.stringify({
       domain,
       dns: results,
       // dohMozillaSupport,
-      }),
     };
   } catch (error) {
-    return {
-      statusCode: 500,
-      body: JSON.stringify({
-        error: `An error occurred while resolving DNS. ${error.message}`,
-      }),
-    };
+    throw new Error(`An error occurred while resolving DNS. ${error.message}`); // This will be caught and handled by the commonMiddleware
   }
 };
+
+module.exports = middleware(handler);
+module.exports.handler = middleware(handler);

api/dns.js

@@ -1,8 +1,9 @@
 const dns = require('dns');
 const util = require('util');
+const middleware = require('./_common/middleware');
 
-exports.handler = async function(event, context) {
-  let hostname = event.queryStringParameters.url;
+const handler = async (url) => {
+  let hostname = url;
 
   // Handle URLs by extracting hostname
   if (hostname.startsWith('http://') || hostname.startsWith('https://')) {
@@ -35,8 +36,6 @@ exports.handler = async function(event, context) {
     ]);
 
     return {
-      statusCode: 200,
-      body: JSON.stringify({
       A: a,
       AAAA: aaaa,
       MX: mx,
@@ -46,14 +45,11 @@ exports.handler = async function(event, context) {
       SOA: soa,
       SRV: srv,
       PTR: ptr
-      })
     };
   } catch (error) {
-    return {
-      statusCode: 500,
-      body: JSON.stringify({
-        error: error.message
-      })
-    };
+    throw new Error(error.message);
   }
 };
+
+module.exports = middleware(handler);
+module.exports.handler = middleware(handler);

api/dnssec.js

@@ -1,16 +1,7 @@
 const https = require('https');
+const middleware = require('./_common/middleware'); // Make sure this path is correct
 
-exports.handler = async function(event, context) {
-  let { url } = event.queryStringParameters;
-
-  if (!url) {
-    return errorResponse('URL query parameter is required.');
-  }
-
-  // Extract hostname from URL
-  const parsedUrl = new URL(url);
-  const domain = parsedUrl.hostname;
-
+const handler = async (domain) => {
   const dnsTypes = ['DNSKEY', 'DS', 'RRSIG'];
   const records = {};
 
@@ -48,22 +39,16 @@ exports.handler = async function(event, context) {
       if (dnsResponse.Answer) {
         records[type] = { isFound: true, answer: dnsResponse.Answer, response: dnsResponse.Answer };
       } else {
-        records[type] = { isFound: false, answer: null, response: dnsResponse};
+        records[type] = { isFound: false, answer: null, response: dnsResponse };
       }
     } catch (error) {
-      return errorResponse(`Error fetching ${type} record: ${error.message}`);
+      throw new Error(`Error fetching ${type} record: ${error.message}`); // This will be caught and handled by the commonMiddleware
     }
   }
 
-  return {
-    statusCode: 200,
-    body: JSON.stringify(records),
-  };
+  return records;
 };
 
-const errorResponse = (message, statusCode = 444) => {
-  return {
-    statusCode: statusCode,
-    body: JSON.stringify({ error: message }),
-  };
-};
+module.exports = middleware(handler);
+module.exports.handler = middleware(handler);
+

api/features.js

@@ -1,22 +1,15 @@
 const https = require('https');
+const middleware = require('./_common/middleware');
 
-exports.handler = async function (event, context) {
-  const { url } = event.queryStringParameters;
+const handler = async (url) => {
   const apiKey = process.env.BUILT_WITH_API_KEY;
 
-  const errorResponse = (message, statusCode = 500) => {
-    return {
-      statusCode: statusCode,
-      body: JSON.stringify({ error: message }),
-    };
-  };
-
   if (!url) {
-    return errorResponse('URL query parameter is required', 400);
+    throw new Error('URL query parameter is required');
   }
 
   if (!apiKey) {
-    return errorResponse('Missing BuiltWith API key in environment variables', 500);
+    throw new Error('Missing BuiltWith API key in environment variables');
   }
 
   const apiUrl = `https://api.builtwith.com/free1/api.json?KEY=${apiKey}&LOOKUP=${encodeURIComponent(url)}`;
@@ -46,11 +39,11 @@ exports.handler = async function (event, context) {
       req.end();
     });
 
-    return {
-      statusCode: 200,
-      body: response,
-    };
+    return response;
   } catch (error) {
-    return errorResponse(`Error making request: ${error.message}`);
+    throw new Error(`Error making request: ${error.message}`);
   }
 };
+
+module.exports = middleware(handler);
+module.exports.handler = middleware(handler);

api/find-url-ip.js

@@ -1,33 +0,0 @@
-const dns = require('dns');
-
-/* Lambda function to fetch the IP address of a given URL */
-exports.handler = function (event, context, callback) {
-  const addressParam = event.queryStringParameters.url;
-  
-  if (!addressParam) {
-    callback(null, errorResponse('Address parameter is missing.'));
-    return;
-  }
-
-  const address = decodeURIComponent(addressParam)
-    .replaceAll('https://', '')
-    .replaceAll('http://', '');
-
-  dns.lookup(address, (err, ip, family) => {
-    if (err) {
-      callback(null, errorResponse(err.message));
-    } else {
-      callback(null, {
-        statusCode: 200,
-        body: JSON.stringify({ ip, family }),
-      });
-    }
-  });
-};
-
-const errorResponse = (message, statusCode = 444) => {
-  return {
-    statusCode: statusCode,
-    body: JSON.stringify({ error: message }),
-  };
-};

api/firewall.js

@@ -0,0 +1,106 @@
+const axios = require('axios');
+const middleware = require('./_common/middleware');
+
+const hasWaf = (waf) => {
+  return {
+    hasWaf: true, waf,
+  }
+};
+
+const handler = async (url) => {
+  const fullUrl = url.startsWith('http') ? url : `http://${url}`;
+  
+  try {
+    const response = await axios.get(fullUrl);
+
+    const headers = response.headers;
+
+    if (headers['server'] && headers['server'].includes('cloudflare')) {
+      return hasWaf('Cloudflare');
+    }
+
+    if (headers['x-powered-by'] && headers['x-powered-by'].includes('AWS Lambda')) {
+      return hasWaf('AWS WAF');
+    }
+
+    if (headers['server'] && headers['server'].includes('AkamaiGHost')) {
+      return hasWaf('Akamai');
+    }
+
+    if (headers['server'] && headers['server'].includes('Sucuri')) {
+      return hasWaf('Sucuri');
+    }
+
+    if (headers['server'] && headers['server'].includes('BarracudaWAF')) {
+      return hasWaf('Barracuda WAF');
+    }
+
+    if (headers['server'] && (headers['server'].includes('F5 BIG-IP') || headers['server'].includes('BIG-IP'))) {
+      return hasWaf('F5 BIG-IP');
+    }
+
+    if (headers['x-sucuri-id'] || headers['x-sucuri-cache']) {
+      return hasWaf('Sucuri CloudProxy WAF');
+    }
+
+    if (headers['server'] && headers['server'].includes('FortiWeb')) {
+      return hasWaf('Fortinet FortiWeb WAF');
+    }
+
+    if (headers['server'] && headers['server'].includes('Imperva')) {
+      return hasWaf('Imperva SecureSphere WAF');
+    }
+
+    if (headers['x-protected-by'] && headers['x-protected-by'].includes('Sqreen')) {
+      return hasWaf('Sqreen');
+    }
+
+    if (headers['x-waf-event-info']) {
+      return hasWaf('Reblaze WAF');
+    }
+
+    if (headers['set-cookie'] && headers['set-cookie'].includes('_citrix_ns_id')) {
+      return hasWaf('Citrix NetScaler');
+    }
+
+    if (headers['x-denied-reason'] || headers['x-wzws-requested-method']) {
+      return hasWaf('WangZhanBao WAF');
+    }
+
+    if (headers['x-webcoment']) {
+      return hasWaf('Webcoment Firewall');
+    }
+
+    if (headers['server'] && headers['server'].includes('Yundun')) {
+      return hasWaf('Yundun WAF');
+    }
+
+    if (headers['x-yd-waf-info'] || headers['x-yd-info']) {
+      return hasWaf('Yundun WAF');
+    }
+
+    if (headers['server'] && headers['server'].includes('Safe3WAF')) {
+      return hasWaf('Safe3 Web Application Firewall');
+    }
+
+    if (headers['server'] && headers['server'].includes('NAXSI')) {
+      return hasWaf('NAXSI WAF');
+    }
+
+    if (headers['x-datapower-transactionid']) {
+      return hasWaf('IBM WebSphere DataPower');
+    }
+
+    return {
+      hasWaf: false,
+    }
+  } catch (error) {
+    return {
+      statusCode: 500,
+      body: JSON.stringify({ error: error.message }),
+    };
+  }
+};
+
+module.exports = middleware(handler);
+module.exports.handler = middleware(handler);

api/follow-redirects.js

@@ -1,35 +0,0 @@
-exports.handler = async (event) => {
-  const { url } = event.queryStringParameters;
-  const redirects = [url];
-
-  try {
-    const got = await import('got');
-    await got.default(url, {
-      followRedirect: true,
-      maxRedirects: 12,
-      hooks: {
-        beforeRedirect: [
-          (options, response) => {
-            redirects.push(response.headers.location);
-          },
-        ],
-      },
-    });
-
-    return {
-      statusCode: 200,
-      body: JSON.stringify({
-        redirects: redirects,
-      }),
-    };
-  } catch (error) {
-    return errorResponse(`Error: ${error.message}`);
-  }
-};
-
-const errorResponse = (message, statusCode = 444) => {
-  return {
-    statusCode: statusCode,
-    body: JSON.stringify({ error: message }),
-  };
-};

api/get-cookies.js

@@ -1,26 +0,0 @@
-const axios = require('axios');
-
-exports.handler = async function(event, context) {
-  const { url } = event.queryStringParameters;
-
-  if (!url) {
-    return {
-      statusCode: 400,
-      body: JSON.stringify({ message: 'url query string parameter is required' }),
-    };
-  }
-
-  try {
-    const response = await axios.get(url, {withCredentials: true});
-    const cookies = response.headers['set-cookie'];
-    return {
-      statusCode: 200,
-      body: JSON.stringify({ cookies }),
-    };
-  } catch (error) {
-    return {
-      statusCode: 500,
-      body: JSON.stringify({ error: error.message }),
-    };
-  }
-};

api/get-headers.js

@@ -1,31 +0,0 @@
-const axios = require('axios');
-
-exports.handler = async function(event, context) {
-  const { url } = event.queryStringParameters;
-
-  if (!url) {
-    return {
-      statusCode: 400,
-      body: JSON.stringify({ error: 'url query string parameter is required' }),
-    };
-  }
-
-  try {
-    const response = await axios.get(url, {
-      validateStatus: function (status) {
-        return status >= 200 && status < 600; // Resolve only if the status code is less than 600
-      },
-    });
-
-    return {
-      statusCode: 200,
-      body: JSON.stringify(response.headers),
-    };
-  } catch (error) {
-    console.log(error);
-    return {
-      statusCode: 500,
-      body: JSON.stringify({ error: error.message }),
-    };
-  }
-};

api/get-ip.js

@@ -0,0 +1,23 @@
+const dns = require('dns');
+const middleware = require('./_common/middleware');
+
+const lookupAsync = (address) => {
+  return new Promise((resolve, reject) => {
+    dns.lookup(address, (err, ip, family) => {
+      if (err) {
+        reject(err);
+      } else {
+        resolve({ ip, family });
+      }
+    });
+  });
+};
+
+const handler = async (url) => {
+  const address = url.replaceAll('https://', '').replaceAll('http://', '');
+  return await lookupAsync(address);
+};
+
+
+module.exports = middleware(handler);
+module.exports.handler = middleware(handler);

api/headers.js

@@ -0,0 +1,19 @@
+const axios = require('axios');
+const middleware = require('./_common/middleware');
+
+const handler = async (url, event, context) => {
+  try {
+    const response = await axios.get(url, {
+      validateStatus: function (status) {
+        return status >= 200 && status < 600; // Resolve only if the status code is less than 600
+      },
+    });
+
+    return response.headers;
+  } catch (error) {
+    throw new Error(error.message);
+  }
+};
+
+module.exports = middleware(handler);
+module.exports.handler = middleware(handler);

api/hsts.js

@@ -1,30 +1,20 @@
 const https = require('https');
+const middleware = require('./_common/middleware');
 
-exports.handler = async function(event, context) {
-  const siteURL = event.queryStringParameters.url;
-
+const handler = async (url, event, context) => {
   const errorResponse = (message, statusCode = 500) => {
     return {
       statusCode: statusCode,
       body: JSON.stringify({ error: message }),
     };
   };
-  const hstsIncompatible = (message, statusCode = 200) => {
-    return {
-      statusCode: statusCode,
-      body: JSON.stringify({ message, compatible: false }),
-    };
+  const hstsIncompatible = (message, compatible = false, hstsHeader = null ) => {
+    return { message, compatible, hstsHeader };
   };
 
-  if (!siteURL) {
-    return {
-      statusCode: 400,
-      body: JSON.stringify({ error: 'URL parameter is missing!' }),
-    };
-  }
 
   return new Promise((resolve, reject) => {
-    const req = https.request(siteURL, res => {
+    const req = https.request(url, res => {
       const headers = res.headers;
       const hstsHeader = headers['strict-transport-security'];
 
@@ -42,14 +32,7 @@ exports.handler = async function(event, context) {
         } else if (!preload) {
           resolve(hstsIncompatible(`HSTS header does not contain the preload directive.`));
         } else {
-          resolve({
-            statusCode: 200,
-            body: JSON.stringify({
-              message: "Site is compatible with the HSTS preload list!",
-              compatible: true,
-              hstsHeader: hstsHeader,
-            }),
-          });
+          resolve(hstsIncompatible(`Site is compatible with the HSTS preload list!`, true, hstsHeader));
         }
       }
     });
@@ -61,3 +44,7 @@ exports.handler = async function(event, context) {
     req.end();
   });
 };
+
+module.exports = middleware(handler);
+module.exports.handler = middleware(handler);
+

api/http-security.js

@@ -0,0 +1,26 @@
+const axios = require('axios');
+const middleware = require('./_common/middleware');
+
+const handler = async (url) => {
+  const fullUrl = url.startsWith('http') ? url : `http://${url}`;
+  
+  try {
+    const response = await axios.get(fullUrl);
+    const headers = response.headers;
+    return {
+      strictTransportPolicy: headers['strict-transport-security'] ? true : false,
+      xFrameOptions: headers['x-frame-options'] ? true : false,
+      xContentTypeOptions: headers['x-content-type-options'] ? true : false,
+      xXSSProtection: headers['x-xss-protection'] ? true : false,
+      contentSecurityPolicy: headers['content-security-policy'] ? true : false,
+    }
+  } catch (error) {
+    return {
+      statusCode: 500,
+      body: JSON.stringify({ error: error.message }),
+    };
+  }
+};
+
+module.exports = middleware(handler);
+module.exports.handler = middleware(handler);

api/legacy-rank.js

@@ -0,0 +1,71 @@
+const axios = require('axios');
+const unzipper = require('unzipper');
+const csv = require('csv-parser');
+const fs = require('fs');
+const middleware = require('./_common/middleware');
+
+// Should also work with the following sources:
+// https://www.domcop.com/files/top/top10milliondomains.csv.zip
+// https://tranco-list.eu/top-1m.csv.zip
+// https://www.domcop.com/files/top/top10milliondomains.csv.zip
+// https://radar.cloudflare.com/charts/LargerTopDomainsTable/attachment?id=525&top=1000000
+// https://statvoo.com/dl/top-1million-sites.csv.zip
+
+const FILE_URL = 'https://s3-us-west-1.amazonaws.com/umbrella-static/top-1m.csv.zip';
+const TEMP_FILE_PATH = '/tmp/top-1m.csv';
+
+const handler = async (url) => {
+  let domain = null;
+
+  try {
+    domain = new URL(url).hostname;
+  } catch (e) {
+    throw new Error('Invalid URL');
+  }
+
+// Download and unzip the file if not in cache
+if (!fs.existsSync(TEMP_FILE_PATH)) {
+  const response = await axios({
+    method: 'GET',
+    url: FILE_URL,
+    responseType: 'stream'
+  });
+
+  await new Promise((resolve, reject) => {
+    response.data
+      .pipe(unzipper.Extract({ path: '/tmp' }))
+      .on('close', resolve)
+      .on('error', reject);
+  });
+}
+
+// Parse the CSV and find the rank
+return new Promise((resolve, reject) => {
+  const csvStream = fs.createReadStream(TEMP_FILE_PATH)
+    .pipe(csv({
+      headers: ['rank', 'domain'],
+    }))
+    .on('data', (row) => {
+      if (row.domain === domain) {
+        csvStream.destroy();
+        resolve({
+          domain: domain,
+          rank: row.rank,
+          isFound: true,
+        });
+      }
+    })
+    .on('end', () => {
+      resolve({
+        skipped: `Skipping, as ${domain} is not present in the Umbrella top 1M list.`,
+        domain: domain,
+        isFound: false,
+      });
+    })
+    .on('error', reject);
+});
+};
+
+module.exports = middleware(handler);
+module.exports.handler = middleware(handler);
+

api/lighthouse-report.js

@@ -1,40 +0,0 @@
-const axios = require('axios');
-
-exports.handler = function(event, context, callback) {
-  const { url } = event.queryStringParameters;
-
-  if (!url) {
-    callback(null, {
-      statusCode: 400,
-      body: JSON.stringify({ error: 'URL param is required'}),
-    });
-  }
-
-  const apiKey = process.env.GOOGLE_CLOUD_API_KEY;
-
-  if (!apiKey) {
-    callback(null, {
-      statusCode: 500,
-      body: JSON.stringify({ error: 'API key (GOOGLE_CLOUD_API_KEY) not set'}),
-    });
-  }
-
-  const endpoint = `https://www.googleapis.com/pagespeedonline/v5/runPagespeed?url=${encodeURIComponent(url)}&category=PERFORMANCE&category=ACCESSIBILITY&category=BEST_PRACTICES&category=SEO&category=PWA&strategy=mobile&key=${apiKey}`;
-  
-  axios.get(endpoint)
-  .then(
-    (response) => {
-      callback(null, {
-        statusCode: 200,
-        body: JSON.stringify(response.data),
-      });
-    }
-  ).catch(
-    () => {
-      callback(null, {
-        statusCode: 500,
-        body: JSON.stringify({ error: 'Error running Lighthouse'}),
-      });
-    }
-  );
-};

api/linked-pages.js

@@ -0,0 +1,49 @@
+const axios = require('axios');
+const cheerio = require('cheerio');
+const urlLib = require('url');
+const middleware = require('./_common/middleware');
+
+const handler = async (url) => {
+  const response = await axios.get(url);
+  const html = response.data;
+  const $ = cheerio.load(html);
+  const internalLinksMap = new Map();
+  const externalLinksMap = new Map();
+
+  // Get all links on the page
+  $('a[href]').each((i, link) => {
+    const href = $(link).attr('href');
+    const absoluteUrl = urlLib.resolve(url, href);
+    
+    // Check if absolute / relative, append to appropriate map or increment occurrence count
+    if (absoluteUrl.startsWith(url)) {
+      const count = internalLinksMap.get(absoluteUrl) || 0;
+      internalLinksMap.set(absoluteUrl, count + 1);
+    } else if (href.startsWith('http://') || href.startsWith('https://')) {
+      const count = externalLinksMap.get(absoluteUrl) || 0;
+      externalLinksMap.set(absoluteUrl, count + 1);
+    }
+  });
+
+  // Sort by most occurrences, remove supplicates, and convert to array
+  const internalLinks = [...internalLinksMap.entries()].sort((a, b) => b[1] - a[1]).map(entry => entry[0]);
+  const externalLinks = [...externalLinksMap.entries()].sort((a, b) => b[1] - a[1]).map(entry => entry[0]);
+
+  // If there were no links, then mark as skipped and show reasons
+  if (internalLinks.length === 0 && externalLinks.length === 0) {
+    return {
+      statusCode: 400,
+      body: JSON.stringify({
+        skipped: 'No internal or external links found. '
+          + 'This may be due to the website being dynamically rendered, using a client-side framework (like React), and without SSR enabled. '
+          + 'That would mean that the static HTML returned from the HTTP request doesn\'t contain any meaningful content for Web-Check to analyze. '
+          + 'You can rectify this by using a headless browser to render the page instead.',
+        }),
+    };
+  }
+
+  return { internal: internalLinks, external: externalLinks };
+};
+
+module.exports = middleware(handler);
+module.exports.handler = middleware(handler);

api/mail-config.js

@@ -1,11 +1,11 @@
+const middleware = require('./_common/middleware');
+
 const dns = require('dns').promises;
 const URL = require('url-parse');
 
-exports.handler = async (event, context) => {
+const handler = async (url, event, context) => {
   try {
-    let domain = event.queryStringParameters.url;
-    const parsedUrl = new URL(domain);
-    domain = parsedUrl.hostname || parsedUrl.pathname;
+    const domain = new URL(url).hostname || new URL(url).pathname;
 
     // Get MX records
     const mxRecords = await dns.resolveMx(domain);
@@ -56,19 +56,13 @@ exports.handler = async (event, context) => {
     }
 
     return {
-      statusCode: 200,
-      body: JSON.stringify({
         mxRecords,
         txtRecords: emailTxtRecords,
         mailServices,
-      }),
       };
   } catch (error) {
     if (error.code === 'ENOTFOUND' || error.code === 'ENODATA') {
-      return {
-        statusCode: 200,
-        body: JSON.stringify({ skipped: 'No mail server in use on this domain' }),
-      };
+      return { skipped: 'No mail server in use on this domain' };
     } else {
       return {
         statusCode: 500,
@@ -77,3 +71,6 @@ exports.handler = async (event, context) => {
     }
   }
 };
+
+module.exports = middleware(handler);
+module.exports.handler = middleware(handler);

api/ports.js

@@ -1,4 +1,5 @@
 const net = require('net');
+const middleware = require('./_common/middleware');
 
 // A list of commonly used ports.
 const PORTS = [
@@ -12,7 +13,7 @@ async function checkPort(port, domain) {
     return new Promise((resolve, reject) => {
         const socket = new net.Socket();
 
-        socket.setTimeout(1500); // you may want to adjust the timeout
+        socket.setTimeout(1500);
 
         socket.once('connect', () => {
             socket.destroy();
@@ -33,12 +34,8 @@ async function checkPort(port, domain) {
     });
 }
 
-exports.handler = async (event, context) => {
-  const domain = event.queryStringParameters.url;
-  
-  if (!domain) {
-    return errorResponse('Missing domain parameter.');
-  }
+const handler = async (url, event, context) => {
+  const domain = url.replace(/(^\w+:|^)\/\//, '');
   
   const delay = ms => new Promise(res => setTimeout(res, ms));
   const timeout = delay(9000);
@@ -76,15 +73,12 @@ exports.handler = async (event, context) => {
     return errorResponse('The function timed out before completing.');
   }
 
-  return {
-    statusCode: 200,
-    body: JSON.stringify({ openPorts, failedPorts }),
-  };
+  return { openPorts, failedPorts };
 };
 
 const errorResponse = (message, statusCode = 444) => {
-  return {
-    statusCode: statusCode,
-    body: JSON.stringify({ error: message }),
-  };
+  return { error: message };
 };
+
+module.exports = middleware(handler);
+module.exports.handler = middleware(handler);

api/quality.js

@@ -0,0 +1,19 @@
+const axios = require('axios');
+const middleware = require('./_common/middleware');
+
+const handler = async (url, event, context) => {
+  const apiKey = process.env.GOOGLE_CLOUD_API_KEY;
+
+  if (!apiKey) {
+    throw new Error('API key (GOOGLE_CLOUD_API_KEY) not set');
+  }
+
+  const endpoint = `https://www.googleapis.com/pagespeedonline/v5/runPagespeed?url=${encodeURIComponent(url)}&category=PERFORMANCE&category=ACCESSIBILITY&category=BEST_PRACTICES&category=SEO&category=PWA&strategy=mobile&key=${apiKey}`;
+
+  const response = await axios.get(endpoint);
+  
+  return response.data;
+};
+
+module.exports = middleware(handler);
+module.exports.handler = middleware(handler);

api/rank.js

@@ -0,0 +1,26 @@
+const axios = require('axios');
+const middleware = require('./_common/middleware');
+
+const handler = async (url) => { 
+  const domain = url ? new URL(url).hostname : null;
+  if (!domain) throw new Error('Invalid URL');
+
+  try {
+    const auth = process.env.TRANCO_API_KEY ? // Auth is optional.
+      { auth: { username: process.env.TRANCO_USERNAME, password: process.env.TRANCO_API_KEY } }
+      : {};
+    const response = await axios.get(
+      `https://tranco-list.eu/api/ranks/domain/${domain}`, { timeout: 5000 }, auth,
+      );
+    if (!response.data || !response.data.ranks || response.data.ranks.length === 0) {
+      return { skipped: `Skipping, as ${domain} isn't ranked in the top 100 million sites yet.`};
+    }
+    return response.data;
+  } catch (error) {
+    return { error: `Unable to fetch rank, ${error.message}` };
+  }
+};
+
+module.exports = middleware(handler);
+module.exports.handler = middleware(handler);
+

api/read-robots-txt.js

@@ -1,45 +0,0 @@
-const axios = require('axios');
-
-exports.handler = async function(event, context) {
-  const siteURL = event.queryStringParameters.url;
-
-  if (!siteURL) {
-    return {
-      statusCode: 400,
-      body: JSON.stringify({ error: 'Missing url query parameter' }),
-    };
-  }
-
-  let parsedURL;
-  try {
-    parsedURL = new URL(siteURL);
-  } catch (error) {
-    return {
-      statusCode: 400,
-      body: JSON.stringify({ error: 'Invalid url query parameter' }),
-    };
-  }
-
-  const robotsURL = `${parsedURL.protocol}//${parsedURL.hostname}/robots.txt`;
-
-  try {
-    const response = await axios.get(robotsURL);
-
-    if (response.status === 200) {
-      return {
-        statusCode: 200,
-        body: response.data,
-      };
-    } else {
-      return {
-        statusCode: response.status,
-        body: JSON.stringify({ error: 'Failed to fetch robots.txt', statusCode: response.status }),
-      };
-    }
-  } catch (error) {
-    return {
-      statusCode: 500,
-      body: JSON.stringify({ error: `Error fetching robots.txt: ${error.message}` }),
-    };
-  }
-};

api/redirects.js

@@ -0,0 +1,29 @@
+const handler = async (url) => {
+  const redirects = [url];
+  const got = await import('got');
+
+  try {
+    await got.default(url, {
+      followRedirect: true,
+      maxRedirects: 12,
+      hooks: {
+        beforeRedirect: [
+          (options, response) => {
+            redirects.push(response.headers.location);
+          },
+        ],
+      },
+    });
+
+    return {
+      redirects: redirects,
+    };
+  } catch (error) {
+    throw new Error(`Error: ${error.message}`);
+  }
+};
+
+const middleware = require('./_common/middleware');
+
+module.exports = middleware(handler);
+module.exports.handler = middleware(handler);

api/robots-txt.js

@@ -0,0 +1,71 @@
+const axios = require('axios');
+const middleware = require('./_common/middleware');
+
+const parseRobotsTxt = (content) => {
+  const lines = content.split('\n');
+  const rules = [];
+
+  lines.forEach(line => {
+    line = line.trim();  // This removes trailing and leading whitespaces
+
+    let match = line.match(/^(Allow|Disallow):\s*(\S*)$/i);
+    if (match) {
+      const rule = {
+        lbl: match[1],
+        val: match[2],
+      };
+      
+      rules.push(rule);
+    } else {
+      match = line.match(/^(User-agent):\s*(\S*)$/i);
+      if (match) {
+        const rule = {
+          lbl: match[1],
+          val: match[2],
+        };
+        
+        rules.push(rule);
+      }
+    }
+  });
+  return { robots: rules };
+}
+
+const handler = async function(url) {
+  let parsedURL;
+  try {
+    parsedURL = new URL(url);
+  } catch (error) {
+    return {
+      statusCode: 400,
+      body: JSON.stringify({ error: 'Invalid url query parameter' }),
+    };
+  }
+
+  const robotsURL = `${parsedURL.protocol}//${parsedURL.hostname}/robots.txt`;
+
+  try {
+    const response = await axios.get(robotsURL);
+
+    if (response.status === 200) {
+      const parsedData = parseRobotsTxt(response.data);
+      if (!parsedData.robots || parsedData.robots.length === 0) {
+        return { skipped: 'No robots.txt file present, unable to continue' };
+      }
+      return parsedData;
+    } else {
+      return {
+        statusCode: response.status,
+        body: JSON.stringify({ error: 'Failed to fetch robots.txt', statusCode: response.status }),
+      };
+    }
+  } catch (error) {
+    return {
+      statusCode: 500,
+      body: JSON.stringify({ error: `Error fetching robots.txt: ${error.message}` }),
+    };
+  }
+};
+
+module.exports = middleware(handler);
+module.exports.handler = middleware(handler);

api/screenshot.js

@@ -1,16 +1,11 @@
 const puppeteer = require('puppeteer-core');
 const chromium = require('chrome-aws-lambda');
+const middleware = require('./_common/middleware');
 
-exports.handler = async (event, context, callback) => {
-  let browser = null;
-  let targetUrl = event.queryStringParameters.url;
+const handler = async (targetUrl) => {
 
   if (!targetUrl) {
-    callback(null, {
-      statusCode: 400,
-      body: JSON.stringify({ error: 'URL is missing from queryStringParameters' }),
-    });
-    return;
+    throw new Error('URL is missing from queryStringParameters');
   }
 
   if (!targetUrl.startsWith('http://') && !targetUrl.startsWith('https://')) {
@@ -20,16 +15,13 @@ exports.handler = async (event, context, callback) => {
   try {
     new URL(targetUrl);
   } catch (error) {
-    callback(null, {
-      statusCode: 400,
-      body: JSON.stringify({ error: 'URL provided is invalid' }),
-    });
-    return;
+    throw new Error('URL provided is invalid');
   }
 
+  let browser = null;
   try {
       browser = await puppeteer.launch({
-      args: chromium.args,
+      args: [...chromium.args, '--no-sandbox'], // Add --no-sandbox flag
       defaultViewport: { width: 800, height: 600 },
       executablePath: process.env.CHROME_PATH || await chromium.executablePath,
       headless: chromium.headless,
@@ -40,9 +32,7 @@ exports.handler = async (event, context, callback) => {
     let page = await browser.newPage();
 
     await page.emulateMediaFeatures([{ name: 'prefers-color-scheme', value: 'dark' }]);
-
     page.setDefaultNavigationTimeout(8000);
-
     await page.goto(targetUrl, { waitUntil: 'domcontentloaded' });
 
     await page.evaluate(() => {
@@ -57,24 +47,16 @@ exports.handler = async (event, context, callback) => {
     });
 
     const screenshotBuffer = await page.screenshot();
-
     const base64Screenshot = screenshotBuffer.toString('base64');
 
-    const response = {
-      statusCode: 200,
-      body: JSON.stringify({ image: base64Screenshot }),
-    };
+    return { image: base64Screenshot };
 
-    callback(null, response);
-  } catch (error) {
-    console.log(error);
-    callback(null, {
-      statusCode: 500,
-      body: JSON.stringify({ error: `An error occurred: ${error.message}` }),
-    });
   } finally {
     if (browser !== null) {
       await browser.close();
     }
   }
 };
+
+module.exports = middleware(handler);
+module.exports.handler = middleware(handler);

api/security-txt.js

@@ -1,5 +1,6 @@
 const { https } = require('follow-redirects');
 const { URL } = require('url');
+const middleware = require('./_common/middleware');
 
 const SECURITY_TXT_PATHS = [
   '/security.txt',
@@ -37,57 +38,35 @@ const isPgpSigned = (result) => {
   return false;
 };
 
-exports.handler = async (event, context) => {
-  const urlParam = event.queryStringParameters.url;
-  if (!urlParam) {
-    return { 
-      statusCode: 400, 
-      body: JSON.stringify({ error: 'Missing url parameter' }) 
-    };
-  }
+const handler = async (urlParam) => {
 
   let url;
   try {
     url = new URL(urlParam.includes('://') ? urlParam : 'https://' + urlParam);
   } catch (error) {
-    return {
-      statusCode: 500,
-      body: JSON.stringify({ error: 'Invalid URL format' }),
-    };
+    throw new Error('Invalid URL format');
   }
   url.pathname = '';
   
   for (let path of SECURITY_TXT_PATHS) {
     try {
       const result = await fetchSecurityTxt(url, path);
-      if (result && result.includes('<html')) return {
-        statusCode: 200,
-        body: JSON.stringify({ isPresent: false }),
-      };
+      if (result && result.includes('<html')) return { isPresent: false };
       if (result) {
         return {
-          statusCode: 200,
-          body: JSON.stringify({
           isPresent: true,
           foundIn: path,
           content: result,
           isPgpSigned: isPgpSigned(result),
           fields: parseResult(result),
-          }),
         };
       }
     } catch (error) {
-      return {
-        statusCode: 500,
-        body: JSON.stringify({ error: error.message }),
-      };
+      throw new Error(error.message);
     }
   }
   
-  return {
-    statusCode: 404,
-    body: JSON.stringify({ isPresent: false }),
-  };
+  return { isPresent: false };
 };
 
 async function fetchSecurityTxt(baseURL, path) {
@@ -110,3 +89,6 @@ async function fetchSecurityTxt(baseURL, path) {
     });
   });
 }
+
+module.exports = middleware(handler);
+module.exports.handler = middleware(handler);

api/sitemap.js

@@ -1,8 +1,9 @@
+const middleware = require('./_common/middleware');
+
 const axios = require('axios');
 const xml2js = require('xml2js');
 
-exports.handler = async (event) => {
-  const url = event.queryStringParameters.url;
+const handler = async (url) => {
   let sitemapUrl = `${url}/sitemap.xml`;
 
   try {
@@ -24,10 +25,7 @@ exports.handler = async (event) => {
         }
 
         if (!sitemapUrl) {
-          return {
-            statusCode: 404,
-            body: JSON.stringify({ skipped: 'No sitemap found' }),
-          };
+          return { skipped: 'No sitemap found' };
         }
 
         sitemapRes = await axios.get(sitemapUrl, { timeout: 5000 });
@@ -39,23 +37,18 @@ exports.handler = async (event) => {
     const parser = new xml2js.Parser();
     const sitemap = await parser.parseStringPromise(sitemapRes.data);
 
-    return {
-      statusCode: 200,
-      body: JSON.stringify(sitemap),
-    };
+    return sitemap;
   } catch (error) {
     // If error occurs
     console.log(error.message);
     if (error.code === 'ECONNABORTED') {
-      return {
-        statusCode: 500,
-        body: JSON.stringify({ error: 'Request timed out' }),
-      };
+      return { error: 'Request timed out' };
     } else {
-      return {
-        statusCode: 500,
-        body: JSON.stringify({ error: error.message }),
-      };
+      return { error: error.message };
     }
   }
 };
+
+module.exports = middleware(handler);
+module.exports.handler = middleware(handler);
+

api/social-tags.js

@@ -1,8 +1,9 @@
+const middleware = require('./_common/middleware');
+
 const axios = require('axios');
 const cheerio = require('cheerio');
 
-exports.handler = async (event, context) => {
-  let url = event.queryStringParameters.url;
+const handler = async (url) => {
   
   // Check if url includes protocol
   if (!url.startsWith('http://') && !url.startsWith('https://')) {
@@ -49,16 +50,9 @@ exports.handler = async (event, context) => {
     };
 
     if (Object.keys(metadata).length === 0) {
-      return {
-        statusCode: 200,
-        body: JSON.stringify({ skipped: 'No metadata found' }),
-      };
+      return { skipped: 'No metadata found' };
     }
-
-    return {
-      statusCode: 200,
-      body: JSON.stringify(metadata),
-    };
+    return metadata;
   } catch (error) {
     return {
       statusCode: 500,
@@ -66,3 +60,6 @@ exports.handler = async (event, context) => {
     };
   }
 };
+
+module.exports = middleware(handler);
+module.exports.handler = middleware(handler);

api/ssl-check.js

@@ -1,50 +0,0 @@
-const https = require('https');
-
-exports.handler = async function (event, context) {
-  const { url } = event.queryStringParameters;
-
-  const errorResponse = (message, statusCode = 500) => {
-    return {
-      statusCode: statusCode,
-      body: JSON.stringify({ error: message }),
-    };
-  };
-
-  if (!url) {
-    return errorResponse('URL query parameter is required', 400);
-  }
-
-  try {
-    const response = await new Promise((resolve, reject) => {
-      const req = https.request(url, res => {
-
-        // Check if the SSL handshake was authorized
-        if (!res.socket.authorized) {
-          resolve(errorResponse(`SSL handshake not authorized. Reason: ${res.socket.authorizationError}`));
-        } else {
-          let cert = res.socket.getPeerCertificate(true);
-          if (!cert || Object.keys(cert).length === 0) {
-            resolve(errorResponse("No certificate presented by the server."));
-          } else {
-            // omit the raw and issuerCertificate fields
-            const { raw, issuerCertificate, ...certWithoutRaw } = cert;
-            resolve({
-              statusCode: 200,
-              body: JSON.stringify(certWithoutRaw),
-            });
-          }
-        }
-      });
-
-      req.on('error', error => {
-        resolve(errorResponse(`Error fetching site certificate: ${error.message}`));
-      });
-
-      req.end();
-    });
-
-    return response;
-  } catch (error) {
-    return errorResponse(`Unexpected error occurred: ${error.message}`);
-  }
-};

api/ssl.js

@@ -0,0 +1,44 @@
+const tls = require('tls');
+const middleware = require('./_common/middleware');
+
+const handler = async (urlString) => {
+  try {
+    const parsedUrl = new URL(urlString);
+    const options = {
+      host: parsedUrl.hostname,
+      port: parsedUrl.port || 443,
+      servername: parsedUrl.hostname,
+      rejectUnauthorized: false,
+    };
+
+    return new Promise((resolve, reject) => {
+      const socket = tls.connect(options, () => {
+        if (!socket.authorized) {
+          return reject(new Error(`SSL handshake not authorized. Reason: ${socket.authorizationError}`));
+        }
+
+        const cert = socket.getPeerCertificate();
+        if (!cert || Object.keys(cert).length === 0) {
+          return reject(new Error(`
+          No certificate presented by the server.\n
+          The server is possibly not using SNI (Server Name Indication) to identify itself, and you are connecting to a hostname-aliased IP address.
+          Or it may be due to an invalid SSL certificate, or an incomplete SSL handshake at the time the cert is being read.`));
+        }
+
+        const { raw, issuerCertificate, ...certWithoutRaw } = cert;
+        resolve(certWithoutRaw);
+        socket.end();
+      });
+
+      socket.on('error', (error) => {
+        reject(new Error(`Error fetching site certificate: ${error.message}`));
+      });
+    });
+
+  } catch (error) {
+    throw new Error(error.message);
+  }
+};
+
+module.exports = middleware(handler);
+module.exports.handler = middleware(handler);

api/status.js

@@ -1,14 +1,10 @@
 const https = require('https');
 const { performance, PerformanceObserver } = require('perf_hooks');
+const middleware = require('./_common/middleware');
 
-exports.handler = async function(event, context) {
-  const { url } = event.queryStringParameters;
-
+const handler = async (url) => {
   if (!url) {
-    return {
-      statusCode: 400,
-      body: JSON.stringify({ error: 'You must provide a URL query parameter!' }),
-    };
+    throw new Error('You must provide a URL query parameter!');
   }
 
   let dnsLookupTime;
@@ -43,10 +39,7 @@ exports.handler = async function(event, context) {
     });
 
     if (responseCode < 200 || responseCode >= 400) {
-      return {
-        statusCode: 200,
-        body: JSON.stringify({ error: `Received non-success response code: ${responseCode}` }),
-      };
+      throw new Error(`Received non-success response code: ${responseCode}`);
     }
 
     performance.mark('B');
@@ -54,16 +47,13 @@ exports.handler = async function(event, context) {
     let responseTime = performance.now() - startTime;
     obs.disconnect();
 
-    return {
-      statusCode: 200,
-      body: JSON.stringify({ isUp: true, dnsLookupTime, responseTime, responseCode }),
-    };
+    return { isUp: true, dnsLookupTime, responseTime, responseCode };
 
   } catch (error) {
     obs.disconnect();
-    return {
-      statusCode: 200,
-      body: JSON.stringify({ error: `Error during operation: ${error.message}` }),
-    };
+    throw error;
   }
 };
+
+module.exports = middleware(handler);
+module.exports.handler = middleware(handler);

api/tech-stack.js

@@ -1,69 +1,31 @@
 const Wappalyzer = require('wappalyzer');
+const middleware = require('./_common/middleware');
 
-const analyze = async (url) => {
-
+const handler = async (url) => {
   const options = {};
 
   const wappalyzer = new Wappalyzer(options);
-  return (async function() {
+
   try {
-      await wappalyzer.init()
-      const headers = {}
+    await wappalyzer.init();
+    const headers = {};
     const storage = {
       local: {},
       session: {},
+    };
+    const site = await wappalyzer.open(url, headers, storage);
+    const results = await site.analyze();
+
+    if (!results.technologies || results.technologies.length === 0) {
+      throw new Error('Unable to find any technologies for site');
     }
-      const site = await wappalyzer.open(url, headers, storage)
-      const results = await site.analyze()
     return results;
   } catch (error) {
-      return error;
+    throw new Error(error.message);
   } finally {
-      await wappalyzer.destroy()
-    }
-  })();
-}
-
-exports.handler = async (event, context, callback) => {
-  // Validate URL parameter
-  if (!event.queryStringParameters || !event.queryStringParameters.url) {
-    return {
-      statusCode: 400,
-      body: JSON.stringify({ error: 'Missing url parameter' }),
-    };
-  }
-
-  // Get URL from param
-  let url = event.queryStringParameters.url;
-  if (!/^https?:\/\//i.test(url)) {
-    url = 'http://' + url;
-  }
-  
-  try {
-    return analyze(url).then(
-      (results) => {
-        if (!results.technologies || results.technologies.length === 0) {
-          return {
-            statusCode: 200,
-            body: JSON.stringify({ error: 'Unable to find any technologies for site' }),
-          };
-        }
-        return {
-          statusCode: 200,
-          body: JSON.stringify(results),
-        }
-      }
-    )
-    .catch((error) => {
-      return {
-      statusCode: 500,
-      body: JSON.stringify({ error: error.message }),
-      };
-    });
-  } catch (error) {
-    return {
-      statusCode: 500,
-      body: JSON.stringify({ error: error.message }),
-    };
+    await wappalyzer.destroy();
   }
 };
+
+module.exports = middleware(handler);
+module.exports.handler = middleware(handler);

api/threats.js

@@ -0,0 +1,103 @@
+const axios = require('axios');
+const xml2js = require('xml2js');
+const middleware = require('./_common/middleware');
+
+const getGoogleSafeBrowsingResult = async (url) => {
+  try {
+    const apiKey = process.env.GOOGLE_CLOUD_API_KEY;
+    if (!apiKey) {
+      return { error: 'GOOGLE_CLOUD_API_KEY is required for the Google Safe Browsing check' };
+    }
+    const apiEndpoint = `https://safebrowsing.googleapis.com/v4/threatMatches:find?key=${apiKey}`;
+
+    const requestBody = {
+      threatInfo: {
+        threatTypes: [
+          'MALWARE', 'SOCIAL_ENGINEERING', 'UNWANTED_SOFTWARE', 'POTENTIALLY_HARMFUL_APPLICATION', 'API_ABUSE'
+        ],
+        platformTypes: ["ANY_PLATFORM"],
+        threatEntryTypes: ["URL"],
+        threatEntries: [{ url }]
+      }
+    };
+
+    const response = await axios.post(apiEndpoint, requestBody);
+    if (response.data && response.data.matches) {
+      return {
+        unsafe: true,
+        details: response.data.matches
+      };
+    } else {
+      return { unsafe: false };
+    }
+  } catch (error) {
+    return { error: `Request failed: ${error.message}` };
+  }
+};
+
+const getUrlHausResult = async (url) => {
+  let domain = new URL(url).hostname;
+  return await axios({
+    method: 'post',
+    url: 'https://urlhaus-api.abuse.ch/v1/host/',
+    headers: {
+      'Content-Type': 'application/x-www-form-urlencoded'
+    },
+    data: `host=${domain}`
+  })
+  .then((x) => x.data)
+  .catch((e) => ({ error: `Request to URLHaus failed, ${e.message}`}));
+};
+
+
+const getPhishTankResult = async (url) => {
+  try {
+    const encodedUrl = Buffer.from(url).toString('base64');
+    const endpoint = `https://checkurl.phishtank.com/checkurl/?url=${encodedUrl}`;
+    const headers = {
+      'User-Agent': 'phishtank/web-check',
+    };
+    const response = await axios.post(endpoint, null, { headers, timeout: 3000 });
+    const parsed = await xml2js.parseStringPromise(response.data, { explicitArray: false });
+    return parsed.response.results;
+  } catch (error) {
+    return { error: `Request to PhishTank failed: ${error.message}` };
+  }
+}
+
+const getCloudmersiveResult = async (url) => {
+  const apiKey = process.env.CLOUDMERSIVE_API_KEY;
+  if (!apiKey) {
+    return { error: 'CLOUDMERSIVE_API_KEY is required for the Cloudmersive check' };
+  }
+  try {
+    const endpoint = 'https://api.cloudmersive.com/virus/scan/website';
+    const headers = {
+      'Content-Type': 'application/x-www-form-urlencoded',
+      'Apikey': apiKey,
+    };
+    const data = `Url=${encodeURIComponent(url)}`;
+    const response = await axios.post(endpoint, data, { headers });
+    return response.data;
+  } catch (error) {
+    return { error: `Request to Cloudmersive failed: ${error.message}` };
+  }
+};
+
+const handler = async (url) => {
+  try {
+    const urlHaus = await getUrlHausResult(url);
+    const phishTank = await getPhishTankResult(url);
+    const cloudmersive = await getCloudmersiveResult(url);
+    const safeBrowsing = await getGoogleSafeBrowsingResult(url);
+    if (urlHaus.error && phishTank.error && cloudmersive.error && safeBrowsing.error) {
+      throw new Error(`All requests failed - ${urlHaus.error} ${phishTank.error} ${cloudmersive.error} ${safeBrowsing.error}`);
+    }
+    return JSON.stringify({ urlHaus, phishTank, cloudmersive, safeBrowsing });
+  } catch (error) {
+    throw new Error(error.message);
+  }
+};
+
+module.exports = middleware(handler);
+module.exports.handler = middleware(handler);

api/tls.js

@@ -0,0 +1,29 @@
+const axios = require('axios');
+const middleware = require('./_common/middleware');
+
+const MOZILLA_TLS_OBSERVATORY_API = 'https://tls-observatory.services.mozilla.com/api/v1';
+
+const handler = async (url) => {
+  try {
+    const domain = new URL(url).hostname;
+    const scanResponse = await axios.post(`${MOZILLA_TLS_OBSERVATORY_API}/scan?target=${domain}`);
+    const scanId = scanResponse.data.scan_id;
+
+    if (typeof scanId !== 'number') {
+      return {
+        statusCode: 500,
+        body: JSON.stringify({ error: 'Failed to get scan_id from TLS Observatory' }),
+      };
+    }
+    const resultResponse = await axios.get(`${MOZILLA_TLS_OBSERVATORY_API}/results?id=${scanId}`);
+    return {
+      statusCode: 200,
+      body: JSON.stringify(resultResponse.data),
+    };
+  } catch (error) {
+    return { error: error.message };
+  }
+};
+
+module.exports = middleware(handler);
+module.exports.handler = middleware(handler);

api/trace-route.js

@@ -1,14 +1,8 @@
 const traceroute = require('traceroute');
 const url = require('url');
+const middleware = require('./_common/middleware');
 
-exports.handler = async function(event, context) {
-  const urlString = event.queryStringParameters.url;
-
-  try {
-    if (!urlString) {
-      throw new Error('URL parameter is missing!');
-    }
-
+const handler = async (urlString, context) => {
   // Parse the URL and get the hostname
   const urlObject = url.parse(urlString);
   const host = urlObject.hostname;
@@ -26,30 +20,13 @@ exports.handler = async function(event, context) {
         resolve(hops);
       }
     });
-
-      // Check if remaining time is less than 8.8 seconds, then reject promise
-      if (context.getRemainingTimeInMillis() < 8800) {
-        reject(new Error('Lambda is about to timeout'));
-      }
   });
 
   return {
-      statusCode: 200,
-      body: JSON.stringify({
     message: "Traceroute completed!",
     result,
-      }),
   };
-  } catch (err) {
-    const message = err.code === 'ENOENT'
-      ? 'Traceroute command is not installed on the host.'
-      : err.message;
-
-    return {
-      statusCode: 500,
-      body: JSON.stringify({
-        error: message,
-      }),
-    };
-  }
 };
+
+module.exports = middleware(handler);
+module.exports.handler = middleware(handler);

api/txt-records.js

@@ -1,9 +1,11 @@
 const dns = require('dns').promises;
+const middleware = require('./_common/middleware');
 
-exports.handler = async (event) => {
-  const url = new URL(event.queryStringParameters.url);
+const handler = async (url, event, context) => {
   try {
-    const txtRecords = await dns.resolveTxt(url.hostname);
+    const parsedUrl = new URL(url);
+    
+    const txtRecords = await dns.resolveTxt(parsedUrl.hostname);
 
     // Parsing and formatting TXT records into a single object
     const readableTxtRecords = txtRecords.reduce((acc, recordArray) => {
@@ -16,15 +18,16 @@ exports.handler = async (event) => {
       return { ...acc, ...recordObject };
     }, {});
 
-    return {
-      statusCode: 200,
-      body: JSON.stringify(readableTxtRecords),
-    };
+    return readableTxtRecords;
+
   } catch (error) {
-    console.error('Error:', error);
-    return {
-      statusCode: 500,
-      body: JSON.stringify({ error: error.message }),
-    };
+    if (error.code === 'ERR_INVALID_URL') {
+      throw new Error(`Invalid URL ${error}`);
+    } else {
+      throw error;
+    }
   }
 };
+
+module.exports = middleware(handler);
+module.exports.handler = middleware(handler);

api/whois.js

@@ -1,14 +1,7 @@
 const net = require('net');
 const psl = require('psl');
-// const { URL } = require('url');
-
-const errorResponse = (message, statusCode = 444) => {
-  return {
-    statusCode: statusCode,
-    body: JSON.stringify({ error: message }),
-  };
-};
-
+const axios = require('axios');
+const middleware = require('./_common/middleware');
 
 const getBaseDomain = (url) => {
   let protocol = '';
@@ -22,55 +15,7 @@ const getBaseDomain = (url) => {
   return protocol + parsed.domain;
 };
 
-
-exports.handler = async function(event, context) {
-  let url = event.queryStringParameters.url;
-
-  if (!url) {
-    return errorResponse('URL query parameter is required.', 400);
-  }
-
-  if (!url.startsWith('http://') && !url.startsWith('https://')) {
-    url = 'http://' + url;
-  }
-
-  let hostname;
-  try {
-    hostname = getBaseDomain(new URL(url).hostname);
-  } catch (error) {
-    return errorResponse(`Unable to parse URL: ${error}`, 400);
-  }
-
-  return new Promise((resolve, reject) => {
-    const client = net.createConnection({ port: 43, host: 'whois.internic.net' }, () => {
-      client.write(hostname + '\r\n');
-    });
-
-    let data = '';
-    client.on('data', (chunk) => {
-      data += chunk;
-    });
-
-    client.on('end', () => {
-      try {
-        const parsedData = parseWhoisData(data);
-        resolve({
-          statusCode: 200,
-          body: JSON.stringify(parsedData),
-        });
-      } catch (error) {
-        resolve(errorResponse(error.message));
-      }
-    });
-
-    client.on('error', (err) => {
-      resolve(errorResponse(err.message, 500));
-    });
-  });
-};
-
 const parseWhoisData = (data) => {
-
   if (data.includes('No match for')) {
     return { error: 'No matches found for domain in internic database'};
   }
@@ -100,3 +45,67 @@ const parseWhoisData = (data) => {
   return parsedData;
 };
 
+const fetchFromInternic = async (hostname) => {
+  return new Promise((resolve, reject) => {
+    const client = net.createConnection({ port: 43, host: 'whois.internic.net' }, () => {
+      client.write(hostname + '\r\n');
+    });
+
+    let data = '';
+    client.on('data', (chunk) => {
+      data += chunk;
+    });
+
+    client.on('end', () => {
+      try {
+        const parsedData = parseWhoisData(data);
+        resolve(parsedData);
+      } catch (error) {
+        reject(error);
+      }
+    });
+
+    client.on('error', (err) => {
+      reject(err);
+    });
+  });
+};
+
+const fetchFromMyAPI = async (hostname) => {
+  try {
+    const response = await axios.post('https://whois-api-zeta.vercel.app/', {
+      domain: hostname
+    });
+    return response.data;
+  } catch (error) {
+    console.error('Error fetching data from your API:', error.message);
+    return null;
+  }
+};
+
+const handler = async (url) => {
+  if (!url.startsWith('http://') && !url.startsWith('https://')) {
+    url = 'http://' + url;
+  }
+
+  let hostname;
+  try {
+    hostname = getBaseDomain(new URL(url).hostname);
+  } catch (error) {
+    throw new Error(`Unable to parse URL: ${error}`);
+  }
+
+  const [internicData, whoisData] = await Promise.all([
+    fetchFromInternic(hostname),
+    fetchFromMyAPI(hostname)
+  ]);
+
+  return {
+    internicData,
+    whoisData
+  };
+};
+
+module.exports = middleware(handler);
+module.exports.handler = middleware(handler);
+

docker-compose.yml

@@ -4,5 +4,5 @@ services:
     container_name: Web-Check
     image: lissy93/web-check
     ports:
-      - 8888:8888
+      - 3000:3000
     restart: unless-stopped

package.json

@@ -4,7 +4,7 @@
   "private": false,
   "description": "All-in-one OSINT tool for analyzing any website",
   "repository": "github:lissy93/web-check",
-  "homepage": "https://web-check.as93.net",
+  "homepage": "https://web-check.xyz",
   "license": "MIT",
   "author": {
     "name": "Alicia Sykes",
@@ -16,11 +16,13 @@
   },
   "scripts": {
     "dev": "netlify dev",
-    "serve": "netlify serve --offline",
+    "serve": "node server",
     "start": "react-scripts start",
     "build": "react-scripts build",
     "test": "react-scripts test",
-    "eject": "react-scripts eject"
+    "dev:vercel": "PLATFORM='vercel' npx vercel dev",
+    "dev:netlify": "PLATFORM='netlify' npx netlify dev",
+    "dev:node": "npx concurrently --names \"frontend,backend\" \"REACT_APP_API_ENDPOINT='http://localhost:3001/api' npx react-scripts start\" \"PLATFORM='node' DISABLE_GUI='true' PORT='3001' API_CORS_ORIGIN='*' npx nodemon server\""
   },
   "dependencies": {
     "@netlify/functions": "^1.6.0",
@@ -38,6 +40,11 @@
     "axios": "^1.4.0",
     "cheerio": "^1.0.0-rc.12",
     "chrome-aws-lambda": "^10.1.0",
+    "chromium": "^3.0.3",
+    "connect-history-api-fallback": "^2.0.0",
+    "cors": "^2.8.5",
+    "csv-parser": "^3.0.0",
+    "dotenv": "^16.3.1",
     "flatted": "^3.2.7",
     "follow-redirects": "^1.15.2",
     "got": "^13.0.0",
@@ -46,7 +53,7 @@
     "perf_hooks": "^0.0.1",
     "psl": "^1.9.0",
     "puppeteer": "^20.9.0",
-    "puppeteer-core": "^20.9.0",
+    "puppeteer-core": "^21.0.3",
     "react": "^18.2.0",
     "react-dom": "^18.2.0",
     "react-masonry-css": "^1.0.16",
@@ -54,12 +61,14 @@
     "react-scripts": "5.0.1",
     "react-simple-maps": "^3.0.0",
     "react-toastify": "^9.1.3",
+    "recharts": "^2.7.3",
     "styled-components": "^6.0.5",
     "traceroute": "^1.0.0",
     "typescript": "^5.1.6",
-    "wappalyzer": "^6.10.63",
+    "unzipper": "^0.10.14",
+    "wappalyzer": "^6.10.65",
     "web-vitals": "^3.4.0",
-    "xml2js": "^0.6.0"
+    "xml2js": "^0.6.2"
   },
   "eslintConfig": {
     "extends": [
@@ -82,5 +91,12 @@
   "compilerOptions": {
     "allowJs": true,
     "outDir": "./dist"
+  },
+  "devDependencies": {
+    "serverless-domain-manager": "^7.1.1",
+    "serverless-offline": "^12.0.4",
+    "serverless-webpack": "^5.13.0",
+    "webpack": "^5.88.2",
+    "webpack-node-externals": "^3.0.0"
   }
 }

public/error.html

@@ -0,0 +1,39 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Web-Check</title>
+  </head>
+  <body>
+    <section>
+      <a class="t" href="/"><h1><img src="https://i.ibb.co/q1gZN2p/web-check-logo.png" width="48" />Web-Check</h1></a>
+      <p class="moji">😪</p>
+      <p>There was an error finding this route.</p>
+      <span>Docs and Source: <a href="https://github.com/lissy93/web-check">github.com/lissy93/web-check</a></span>
+    </section>
+    <style>
+      body { background: #141d2b; color: #fff; }
+      h1 {
+        color: #9fef00; text-shadow: #0f1620 2px 2px 0px;
+        font-size: 3rem; margin: 1rem auto; flex-wrap: wrap;
+        display: flex; align-items: flex-start; gap: 1rem;
+      }
+      section {
+        display: flex; flex-direction: column; align-items: center; margin: 1rem; gap: 0.5rem;
+        background: #1a2332; box-shadow: #0f1620 4px 4px 0px; border-radius: 8px; padding: 1rem;
+        max-width: 800px; margin: 1rem auto;
+      }
+      p { font-size: 1.2rem; }
+      a { color: #9fef00; font-family: monospace; }
+      a.t { text-decoration: none; margin: 0;}
+      span { opacity: 0.8; font-size: 0.85rem; }
+      h1 {
+        font-family: PTMono, Ubuntu, Fira Sans, Helvetica, sans-serif;
+      }
+      p, span, a, section, div { font-family: source-code-pro, Menlo, Monaco, Consolas, 'Courier New', monospace; }
+      code { color: #9fef00cc;}
+      .moji { font-size: 8rem; margin: 0; }
+    </style>
+  </body>
+</html>

public/index.html

@@ -7,15 +7,29 @@
     <meta name="theme-color" content="#9fef00" />
     <meta
       name="description"
-      content="Web site created using create-react-app"
+      content="All-in-one OSINT tool, for quickly checking a websites data"
     />
     <link rel="apple-touch-icon" href="%PUBLIC_URL%/logo192.png" />
     <link rel="manifest" href="%PUBLIC_URL%/manifest.json" />
     <title>Web Check</title>
     <script defer data-domain="web-check.as93.net" src="https://no-track.as93.net/js/script.js"></script>
+
+    <!-- OpenGraph Social Tags -->
+    <meta property="og:title" content="Web Check">
+    <meta property="og:description" content="All-in-one Website OSINT Scanner">
+    <meta property="og:image" content="/banner.png">
+    <meta property="og:url" content="https://web-check.xyz">
+    <meta name="twitter:card" content="summary_large_image">
   </head>
   <body>
-    <noscript>All-in-one OSINT tool, for quickly checking a websites data</noscript>
+    <noscript>
+      <b>Welcome to Web-Check, the free and open source tool for viewing all available information about a website.</b><br />
+      Get started by entering a URL, and clicking the "Scan" button, or view the code and docs
+      on <a href="https://github.com/lissy93/web-check">GitHub</a>.<br />
+      <small>Licensed under MIT, ©️ <a href="https://aliciasykes.com">Alicia Sykes</a> 2023.</small>
+      <br /><br />
+      JavaScript is required to continue, please enable it in your browser.
+    </noscript>
     <div id="root"></div>
   </body>
 </html>