andrea.morten/web-check-2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Removal of node-fetch, updates all functions accordingly, use of custom chromium driver for netlify support, plus update of all older packages

Browse Source
This commit is contained in:
Alicia Sykes
2023-07-22 15:18:28 +01:00
parent 38fb1a1d4b
commit dd0be7ed05
10 changed files with 713 additions and 808 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
server/lambda/get-cookies.js
View File

@@ -1,4 +1,4 @@
const fetch = require('node-fetch');
const axios = require('axios');
exports.handler = async function(event, context) {
const { url } = event.queryStringParameters;
@@ -11,9 +11,8 @@ exports.handler = async function(event, context) {
}
try {
const response = await fetch(url);
const cookies = response.headers.get('set-cookie');
const response = await axios.get(url, {withCredentials: true});
const cookies = response.headers['set-cookie'];
return {
statusCode: 200,
body: JSON.stringify({ cookies }),

12
server/lambda/get-headers.js
View File

@@ -1,4 +1,4 @@
const fetch = require('node-fetch');
const axios = require('axios');
exports.handler = async function(event, context) {
const { url } = event.queryStringParameters;
@@ -11,14 +11,18 @@ exports.handler = async function(event, context) {
}
try {
const response = await fetch(url);
const headers = response.headers.raw();
const response = await axios.get(url, {
validateStatus: function (status) {
return status >= 200 && status < 600; // Resolve only if the status code is less than 600
},
});
return {
statusCode: 200,
body: JSON.stringify(headers),
body: JSON.stringify(response.headers),
};
} catch (error) {
console.log(error);
return {
statusCode: 500,
body: JSON.stringify({ error: error.message }),

11
server/lambda/lighthouse-report.js
View File

@@ -1,4 +1,4 @@
const fetch = require('node-fetch');
const axios = require('axios');
exports.handler = function(event, context, callback) {
const { url } = event.queryStringParameters;
@@ -12,13 +12,13 @@ exports.handler = function(event, context, callback) {
const apiKey = process.env.GOOGLE_CLOUD_API_KEY;
const endpoint = `https://www.googleapis.com/pagespeedonline/v5/runPagespeed?url=${encodeURIComponent(url)}&category=PERFORMANCE&category=ACCESSIBILITY&category=BEST_PRACTICES&category=SEO&category=PWA&strategy=mobile&key=${apiKey}`;
fetch(endpoint)
.then((res) => res.json() )
axios.get(endpoint)
.then(
(data) => {
(response) => {
callback(null, {
statusCode: 200,
body: JSON.stringify(data),
body: JSON.stringify(response.data),
});
}
).catch(
@@ -30,4 +30,3 @@ exports.handler = function(event, context, callback) {
}
);
};

11
server/lambda/read-robots-txt.js
View File

@@ -1,8 +1,8 @@
const fetch = require('node-fetch');
const axios = require('axios');
exports.handler = async function(event, context) {
const siteURL = event.queryStringParameters.url;
if (!siteURL) {
return {
statusCode: 400,
@@ -23,13 +23,12 @@ exports.handler = async function(event, context) {
const robotsURL = `${parsedURL.protocol}//${parsedURL.hostname}/robots.txt`;
try {
const response = await fetch(robotsURL);
const text = await response.text();
const response = await axios.get(robotsURL);
if (response.ok) {
if (response.status === 200) {
return {
statusCode: 200,
body: text,
body: response.data,
};
} else {
return {

3
server/lambda/screenshot.js
View File

@@ -31,9 +31,10 @@ exports.handler = async (event, context, callback) => {
browser = await puppeteer.launch({
args: chromium.args,
defaultViewport: { width: 800, height: 600 },
executablePath: process.env.CHROME_EXECUTABLE_PATH || await chromium.executablePath,
executablePath: process.env.CHROME_PATH || await chromium.executablePath,
headless: chromium.headless,
ignoreHTTPSErrors: true,
ignoreDefaultArgs: ['--disable-extensions'],
});
let page = await browser.newPage();

71
server/lambda/web-check.js
View File

@@ -1,71 +0,0 @@
const fetch = require('node-fetch');
const endpoints = {
dnsRecords: 'https://api.securitytrails.com/v1/domain/{address}',
subDomains: 'https://api.securitytrails.com/v1/domain/{address}/subdomains?include_inactive=true',
};
const apiKeys = {
shodan: process.env.SHODAN_API_KEY,
securityTrails: process.env.SECURITY_TRAILS_API_KEY,
};
const makeEndpoint = (endpoint, address) => endpoint.replace('{address}', address);
const errorObject = (msg, status) => {
return {
statusCode: status || 500,
body: JSON.stringify({ error: true, errorMsg: msg }),
}
};
/* Fetches list of DNS records for a given URL from SecurityTrails API */
const getDnsInfo = async (url) => {
const stOptions = {
method: 'GET',
headers: { Accept: 'application/json', APIKEY: apiKeys.securityTrails }
};
const dnsInfoRequest = await fetch(makeEndpoint(endpoints.dnsRecords, url), stOptions);
const dnsInfoResponse = await dnsInfoRequest.json();
if (dnsInfoResponse) return null;
return dnsInfoResponse?.current_dns;
};
/* Fetches a list of a domain's sub-domains from SecurityTrails API */
const getSubDomains = async (url) => {
const stOptions = {
method: 'GET',
headers: { Accept: 'application/json', APIKEY: apiKeys.securityTrails }
};
const subDomainRequest = await fetch(makeEndpoint(endpoints.subDomains, url), stOptions);
const subDomainResponse = await subDomainRequest.json();
if (!subDomainResponse.subdomains) return null;
return {
count: subDomainResponse?.subdomain_count,
list: subDomainResponse?.subdomains,
};
};
exports.handler = async (event, context) => {
const addressParam = event.queryStringParameters.address;
if (!addressParam) return errorObject('An address must be specified');
const address = decodeURIComponent(addressParam)
.replaceAll('https://', '')
.replaceAll('http://', '');
try {
const results = {};
if (apiKeys.securityTrails) {
results.dnsRecords = await getDnsInfo(address);
results.subDomains = await getSubDomains(address);
}
return {
statusCode: 200,
body: JSON.stringify(results),
};
} catch (error) {
return errorObject('Failed to fetch data');
}
};