diff --git a/api/content-links.js b/api/content-links.js new file mode 100644 index 0000000..34e4205 --- /dev/null +++ b/api/content-links.js @@ -0,0 +1,48 @@ +const axios = require('axios'); +const cheerio = require('cheerio'); +const urlLib = require('url'); + +exports.handler = async (event, context) => { + let url = event.queryStringParameters.url; + + // Check if url includes protocol + if (!url.startsWith('http://') && !url.startsWith('https://')) { + url = 'http://' + url; + } + + try { + const response = await axios.get(url); + const html = response.data; + const $ = cheerio.load(html); + const internalLinksMap = new Map(); + const externalLinksMap = new Map(); + + $('a[href]').each((i, link) => { + const href = $(link).attr('href'); + const absoluteUrl = urlLib.resolve(url, href); + + if (absoluteUrl.startsWith(url)) { + const count = internalLinksMap.get(absoluteUrl) || 0; + internalLinksMap.set(absoluteUrl, count + 1); + } else if (href.startsWith('http://') || href.startsWith('https://')) { + const count = externalLinksMap.get(absoluteUrl) || 0; + externalLinksMap.set(absoluteUrl, count + 1); + } + }); + + // Convert maps to sorted arrays + const internalLinks = [...internalLinksMap.entries()].sort((a, b) => b[1] - a[1]).map(entry => entry[0]); + const externalLinks = [...externalLinksMap.entries()].sort((a, b) => b[1] - a[1]).map(entry => entry[0]); + + return { + statusCode: 200, + body: JSON.stringify({ internal: internalLinks, external: externalLinks }), + }; + } catch (error) { + console.log(error); + return { + statusCode: 500, + body: JSON.stringify({ error: 'Failed fetching data' }), + }; + } +}; diff --git a/package.json b/package.json index 41bfb3d..1720e2f 100644 --- a/package.json +++ b/package.json @@ -37,6 +37,7 @@ "@types/styled-components": "^5.1.26", "aws-serverless-express": "^3.4.0", "axios": "^1.4.0", + "cheerio": "^1.0.0-rc.12", "chrome-aws-lambda": "^10.1.0", "flatted": "^3.2.7", "follow-redirects": "^1.15.2", diff --git a/src/components/Form/Input.tsx b/src/components/Form/Input.tsx index 11158ea..d65a7ec 100644 --- a/src/components/Form/Input.tsx +++ b/src/components/Form/Input.tsx @@ -44,6 +44,8 @@ const StyledInput = styled.input` const StyledLabel = styled.label` color: ${colors.textColor}; ${props => applySize(props.inputSize)}; + padding: 0; + font-size: 1.6rem; `; const Input = (inputProps: Props): JSX.Element => { diff --git a/src/components/Form/Nav.tsx b/src/components/Form/Nav.tsx index c518503..2c7825d 100644 --- a/src/components/Form/Nav.tsx +++ b/src/components/Form/Nav.tsx @@ -6,13 +6,14 @@ import colors from 'styles/colors'; import { ReactNode } from 'react'; const Header = styled(StyledCard)` - margin: 1rem; + margin: 1rem auto; display: flex; flex-wrap: wrap; align-items: baseline; justify-content: space-between; padding: 0.5rem 1rem; align-items: center; + width: 95vw; `; const Nav = (props: { children?: ReactNode}) => { diff --git a/src/components/Results/ContentLinks.tsx b/src/components/Results/ContentLinks.tsx new file mode 100644 index 0000000..d2ea0a5 --- /dev/null +++ b/src/components/Results/ContentLinks.tsx @@ -0,0 +1,89 @@ +import { Card } from 'components/Form/Card'; +import Row from 'components/Form/Row'; +import Heading from 'components/Form/Heading'; +import colors from 'styles/colors'; + +const cardStyles = ` + small { margin-top: 1rem; opacity: 0.5; } + a { + color: ${colors.textColor}; + } + details { + // display: inline; + display: flex; + transition: all 0.2s ease-in-out; + h3 { + display: inline; + } + summary { + padding: 0; + margin: 1rem 0 0 0; + cursor: pointer; + } + summary:before { + content: "►"; + position: absolute; + margin-left: -1rem; + color: ${colors.primary}; + cursor: pointer; + } + &[open] summary:before { + content: "▼"; + } + } +`; + +const getPathName = (link: string) => { + try { + const url = new URL(link); + return url.pathname; + } catch(e) { + return link; + } +}; + +const ContentLinksCard = (props: { data: any, title: string, actionButtons: any }): JSX.Element => { + const { internal, external} = props.data; + console.log('Internal Links', internal); + console.log('External Links', external); + return ( + + Summary + + + { internal && internal.length > 0 && ( +
+ Internal Links + {internal.map((link: string) => ( + + {getPathName(link)} + + ))} +
+ )} + { external && external.length > 0 && ( +
+ External Links + {external.map((link: string) => ( + + {link} + + ))} +
+ )} + {/* {portData.openPorts.map((port: any) => ( + + {port} + + ) + )} +
+ + Unable to establish connections to:
+ {portData.failedPorts.join(', ')} + */} + + ); +} + +export default ContentLinksCard; diff --git a/src/components/Results/DnsServer.tsx b/src/components/Results/DnsServer.tsx index 7900721..683de73 100644 --- a/src/components/Results/DnsServer.tsx +++ b/src/components/Results/DnsServer.tsx @@ -20,9 +20,9 @@ const DnsServerCard = (props: {data: any, title: string, actionButtons: any }): {dnsSecurity.dns.map((dns: any, index: number) => { return (<> { dnsSecurity.dns.length > 1 && DNS Server #{index+1} } - - { dns.hostname && } - + + { dns.hostname && } + ); })} {dnsSecurity.dns.length > 0 && ( diff --git a/src/components/misc/AdditionalResources.tsx b/src/components/misc/AdditionalResources.tsx new file mode 100644 index 0000000..663fc8f --- /dev/null +++ b/src/components/misc/AdditionalResources.tsx @@ -0,0 +1,226 @@ +import styled from 'styled-components'; +import colors from 'styles/colors'; +import { Card } from 'components/Form/Card'; + +const ResourceListOuter = styled.ul` +list-style: none; +margin: 0; +padding: 1rem; +display: grid; +gap: 0.5rem; +grid-template-columns: repeat(auto-fit, minmax(20rem, 1fr)); +li a.resource-wrap { + display: flex; + align-items: start; + gap: 0.25rem; + padding: 0.25rem; + background: ${colors.background}; + border-radius: 8px; + text-decoration: none; + color: ${colors.textColor}; + height: 100%; + + transition: all 0.2s ease-in-out; + cursor: pointer; + border: none; + border-radius: 0.25rem; + font-family: PTMono; + box-sizing: border-box; + width: -moz-available; + box-shadow: 3px 3px 0px ${colors.backgroundDarker}; + &:hover { + box-shadow: 5px 5px 0px ${colors.backgroundDarker}; + a { opacity: 1; } + } + &:active { + box-shadow: -3px -3px 0px ${colors.fgShadowColor}; + } +} +img { + width: 4rem; + border-radius: 4px; + margin: 0.25rem 0.1rem 0.1rem 0.1rem; +} +.resource-details { + max-width: 20rem; + display: flex; + flex-direction: column; + gap: 0.1rem; + p, a { + margin: 0; + } + a.resource-link { + color: ${colors.primary}; + opacity: 0.75; + font-size: 0.9rem; + transition: all 0.2s ease-in-out; + } + .resource-title { + font-weight: bold; + } + .resource-description { + color: ${colors.textColorSecondary}; + font-size: 0.9rem; + } +} +`; + +const Note = styled.small` + margin-top: 1rem; + opacity: 0.5; + display: block; + a { color: ${colors.primary}; } +`; + +const CardStyles = ` + margin: 0 auto 1rem auto; + width: 95vw; + position: relative; + transition: all 0.2s ease-in-out; +`; + +const resources = [ + { + title: 'SSL Labs Test', + link: 'https://ssllabs.com/ssltest/analyze.html', + icon: 'https://i.ibb.co/6bVL8JK/Qualys-ssl-labs.png', + description: 'Analyzes the SSL configuration of a server and grades it.', + }, + { + title: 'Virus Total', + link: 'https://virustotal.com', + icon: 'https://i.ibb.co/dWFz0RC/Virustotal.png', + description: 'Checks a URL against multiple antivirus engines.', + searchLink: 'https://www.virustotal.com/gui/domain/{URL}', + }, + { + title: 'Shodan', + link: 'https://shodan.io/', + icon: 'https://i.ibb.co/SBZ8WG4/shodan.png', + description: 'Search engine for Internet-connected devices.', + searchLink: 'https://www.shodan.io/search/report?query={URL}', + }, + { + title: 'Archive', + link: 'https://archive.org/', + icon: 'https://i.ibb.co/nfKMvCm/Archive-org.png', + description: 'View previous versions of a site via the Internet Archive.', + searchLink: 'https://web.archive.org/web/*/{URL}', + }, + { + title: 'URLScan', + link: 'https://urlscan.io/', + icon: 'https://i.ibb.co/cYXt8SH/Url-scan.png', + description: 'Scans a URL and provides information about the page.', + }, + { + title: 'Sucuri SiteCheck', + link: 'https://sitecheck.sucuri.net/', + icon: 'https://i.ibb.co/K5pTP1K/Sucuri-site-check.png', + description: 'Checks a URL against blacklists and known threats.', + searchLink: 'https://www.ssllabs.com/ssltest/analyze.html?d={URL}', + }, + { + title: 'Domain Tools', + link: 'https://whois.domaintools.com/', + icon: 'https://i.ibb.co/zJfCKjM/Domain-tools.png', + description: 'Run a WhoIs lookup on a domain.', + searchLink: 'https://whois.domaintools.com/{URL}', + }, + { + title: 'NS Lookup', + link: 'https://nslookup.io/', + icon: 'https://i.ibb.co/BLSWvBv/Ns-lookup.png', + description: 'View DNS records for a domain.', + searchLink: 'https://www.nslookup.io/domains/{URL}/dns-records/', + }, + { + title: 'DNS Checker', + link: 'https://dnschecker.org/', + icon: 'https://i.ibb.co/gyKtgZ1/Dns-checker.webp', + description: 'Check global DNS propagation across multiple servers.', + searchLink: 'https://dnschecker.org/#A/{URL}', + }, + { + title: 'Censys', + link: 'https://search.censys.io/', + icon: 'https://i.ibb.co/j3ZtXzM/censys.png', + description: 'Lookup hosts associated with a domain.', + searchLink: 'https://search.censys.io/search?resource=hosts&q={URL}', + }, + { + title: 'Page Speed Insights', + link: 'https://developers.google.com/speed/pagespeed/insights/', + icon: 'https://i.ibb.co/k68t9bb/Page-speed-insights.png', + description: 'Checks the performance, accessibility and SEO of a page on mobile + desktop.', + searchLink: 'https://developers.google.com/speed/pagespeed/insights/?url={URL}', + }, + { + title: 'DNS Dumpster', + link: 'https://dnsdumpster.com/', + icon: 'https://i.ibb.co/DtQ2QXP/Trash-can-regular.png', + description: 'DNS recon tool, to map out a domain from it\'s DNS records', + searchLink: '', + }, + { + title: 'BGP Tools', + link: 'https://bgp.tools/', + icon: 'https://i.ibb.co/zhcSnmh/Bgp-tools.png', + description: 'View realtime BGP data for any ASN, Prefix or DNS', + }, + { + title: 'Similar Web', + link: 'https://similarweb.com/', + icon: 'https://i.ibb.co/9YX8x3c/Similar-web.png', + description: 'View approx traffic and engagement stats for a website', + searchLink: 'https://similarweb.com/website/{URL}', + }, + { + title: 'Blacklist Checker', + link: 'https://blacklistchecker.com/', + icon: 'https://i.ibb.co/7ygCyz3/black-list-checker.png', + description: 'Check if a domain, IP or email is present on the top blacklists', + searchLink: 'https://blacklistchecker.com/check?input={URL}', + }, + { + title: 'Cloudflare Radar', + link: 'https://radar.cloudflare.com/', + icon: 'https://i.ibb.co/DGZXRgh/Cloudflare.png', + description: 'View traffic source locations for a domain through Cloudflare', + searchLink: 'https://radar.cloudflare.com/domains/domain/{URL}', + }, +]; + +const makeLink = (resource: any, scanUrl: string | undefined): string => { + return (scanUrl && resource.searchLink) ? resource.searchLink.replaceAll('{URL}', scanUrl.replace('https://', '')) : '#'; +}; + +const AdditionalResources = (props: { url?: string }): JSX.Element => { + return ( + + { + resources.map((resource, index) => { + return ( +
  • + + + + +
    • + ); + }) + } +     + + These tools are not affiliated with Web-Check. Please use them at your own risk.
    + At the time of listing, all of the above were available and free to use + - if this changes, please report it via GitHub (lissy93/web-check). +     +     ); +} + +export default AdditionalResources; diff --git a/src/components/misc/DocContent.tsx b/src/components/misc/DocContent.tsx new file mode 100644 index 0000000..8cdbc3b --- /dev/null +++ b/src/components/misc/DocContent.tsx @@ -0,0 +1,56 @@ +import styled from 'styled-components'; +import docs, { type Doc } from 'utils/docs'; +import colors from 'styles/colors'; +import Heading from 'components/Form/Heading'; + +const JobDocsContainer = styled.div` +p.doc-desc, p.doc-uses, ul { + margin: 0.25rem auto 1.5rem auto; +} +ul { + padding: 0 0.5rem 0 1rem; +} +ul li a { + color: ${colors.primary}; +} +summary { color: ${colors.primary};} +h4 { + border-top: 1px solid ${colors.primary}; + color: ${colors.primary}; + opacity: 0.75; + padding: 0.5rem 0; +} +`; + +const DocContent = (id: string) => { + const doc = docs.filter((doc: Doc) => doc.id === id)[0] || null; + return ( + doc? ( + {doc.title} + About +

    {doc.description}

    + Use Cases +

    {doc.use}

    + Links +
      + {doc.resources.map((resource: string | { title: string, link: string } , index: number) => ( + typeof resource === 'string' ? ( + + ) : ( + + ) + ))} +
    +
    + Example + Screenshot +
    +     ) + : ( + +

    No Docs provided for this widget yet

    +     + )); +}; + +export default DocContent; diff --git a/src/components/misc/ProgressBar.tsx b/src/components/misc/ProgressBar.tsx index d5bf0c7..1d2be06 100644 --- a/src/components/misc/ProgressBar.tsx +++ b/src/components/misc/ProgressBar.tsx @@ -202,6 +202,7 @@ const jobNames = [ 'sitemap', 'hsts', 'security-txt', + 'linked-pages', // 'whois', 'features', 'carbon', diff --git a/src/components/misc/ViewRaw.tsx b/src/components/misc/ViewRaw.tsx new file mode 100644 index 0000000..206c73a --- /dev/null +++ b/src/components/misc/ViewRaw.tsx @@ -0,0 +1,107 @@ +import React, { useState } from 'react'; +import styled from 'styled-components'; +import colors from 'styles/colors'; +import { Card } from 'components/Form/Card'; +import Button from 'components/Form/Button'; + +const CardStyles = ` +margin: 0 auto 1rem auto; +width: 95vw; +position: relative; +transition: all 0.2s ease-in-out; +display: flex; +flex-direction: column; +a { + color: ${colors.primary}; +} +.controls { + display: flex; + flex-wrap: wrap; + button { + max-width: 300px; + } +} +small { + margin-top: 0.5rem; + font-size: 0.8rem; + opacity: 0.5; +} +`; + +const StyledIframe = styled.iframe` + width: calc(100% - 2rem); + outline: none; + border: none; + border-radius: 4px; + min-height: 50vh; + height: 100%; + margin: 1rem; + background: ${colors.background}; +`; + +const ViewRaw = (props: { everything: { id: string, result: any}[] }) => { + const [resultUrl, setResultUrl] = useState(null); + const [error, setError] = useState(null); + + const makeResults = () => { + const result: {[key: string]: any} = {}; + props.everything.forEach((item: {id: string, result: any}) => { + result[item.id] = item.result; + }); + return result; + }; + + const fetchResultsUrl = async () => { + const resultContent = makeResults(); + const response = await fetch('https://jsonhero.io/api/create.json', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + title: 'web-check results', + content: resultContent, + readOnly: true, + ttl: 3600, + }) + }); + if (!response.ok) { + setError(`HTTP error! status: ${response.status}`); + } else { + setError(null); + } + await response.json().then( + (data) => setResultUrl(data.location) + ) + }; + + const handleDownload = () => { + const blob = new Blob([JSON.stringify(makeResults(), null, 2)], { type: "application/json" }); + const url = URL.createObjectURL(blob); + const link = document.createElement('a'); + link.href = url; + link.download = 'web-check-results.json'; + link.click(); + URL.revokeObjectURL(url); + } + return ( + +
    + + + { resultUrl && } +
    + { resultUrl && !error && + <> + + Your results are available to view here. + + } + { error &&

    {error}

    } + + These are the raw results generated from your URL, and in JSON format. + You can import these into your own program, for further analysis. + +     + ); +}; + +export default ViewRaw; diff --git a/src/pages/Home.tsx b/src/pages/Home.tsx index de879b8..33be540 100644 --- a/src/pages/Home.tsx +++ b/src/pages/Home.tsx @@ -60,8 +60,12 @@ const SiteFeaturesWrapper = styled(StyledCard)` .links { display: flex; justify-content: center; + gap: 0.5rem; a { width: 100%; + button { + width: calc(100% - 2rem); + } } } ul { diff --git a/src/pages/Results.tsx b/src/pages/Results.tsx index ed2a1ee..0886a2c 100644 --- a/src/pages/Results.tsx +++ b/src/pages/Results.tsx @@ -1,5 +1,5 @@ import { useState, useEffect, useCallback, ReactNode } from 'react'; -import { useParams } from "react-router-dom"; +import { useParams } from 'react-router-dom'; import styled from 'styled-components'; import { ToastContainer } from 'react-toastify'; import Masonry from 'react-masonry-css' @@ -10,10 +10,15 @@ import Modal from 'components/Form/Modal'; import Footer from 'components/misc/Footer'; import Nav from 'components/Form/Nav'; import { RowProps } from 'components/Form/Row'; -import ErrorBoundary from 'components/misc/ErrorBoundary'; -import docs from 'utils/docs'; import Loader from 'components/misc/Loader'; +import ErrorBoundary from 'components/misc/ErrorBoundary'; +import SelfScanMsg from 'components/misc/SelfScanMsg'; +import DocContent from 'components/misc/DocContent'; +import ProgressBar, { LoadingJob, LoadingState, initialJobs } from 'components/misc/ProgressBar'; +import ActionButtons from 'components/misc/ActionButtons'; +import AdditionalResources from 'components/misc/AdditionalResources'; +import ViewRaw from 'components/misc/ViewRaw'; import ServerLocationCard from 'components/Results/ServerLocation'; import ServerInfoCard from 'components/Results/ServerInfo'; @@ -40,17 +45,11 @@ import DomainLookup from 'components/Results/DomainLookup'; import DnsServerCard from 'components/Results/DnsServer'; import TechStackCard from 'components/Results/TechStack'; import SecurityTxtCard from 'components/Results/SecurityTxt'; -import SelfScanMsg from 'components/misc/SelfScanMsg'; +import ContentLinksCard from 'components/Results/ContentLinks'; - -import ProgressBar, { LoadingJob, LoadingState, initialJobs } from 'components/misc/ProgressBar'; -import ActionButtons from 'components/misc/ActionButtons'; import keys from 'utils/get-keys'; import { determineAddressType, AddressType } from 'utils/address-type-checker'; - import useMotherHook from 'hooks/motherOfAllHooks'; - - import { getLocation, ServerLocation, parseCookies, Cookie, @@ -80,25 +79,6 @@ const ResultsContent = styled.section` padding-bottom: 1rem; `; -const JobDocsContainer = styled.div` -p.doc-desc, p.doc-uses, ul { - margin: 0.25rem auto 1.5rem auto; -} -ul { - padding: 0 0.5rem 0 1rem; -} -ul li a { - color: ${colors.primary}; -} -summary { color: ${colors.primary};} -h4 { - border-top: 1px solid ${colors.primary}; - color: ${colors.primary}; - opacity: 0.75; - padding: 0.5rem 0; -} -`; - const Results = (): JSX.Element => { const startTime = new Date().getTime(); @@ -400,6 +380,14 @@ const Results = (): JSX.Element => { fetchRequest: () => fetch(`${api}/dns-server?url=${address}`).then(res => parseJson(res)), }); + // Get list of links included in the page content + const [linkedPagesResults, updateLinkedPagesResults] = useMotherHook({ + jobId: 'linked-pages', + updateLoadingJobs, + addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly }, + fetchRequest: () => fetch(`${api}/content-links?url=${address}`).then(res => parseJson(res)), + }); + /* Cancel remaining jobs after 10 second timeout */ useEffect(() => { const checkJobs = () => { @@ -438,7 +426,6 @@ const Results = (): JSX.Element => { { id: 'server-info', title: 'Server Info', result: shoadnResults?.serverInfo, Component: ServerInfoCard, refresh: updateShodanResults }, { id: 'redirects', title: 'Redirects', result: redirectResults, Component: RedirectsCard, refresh: updateRedirectResults }, { id: 'robots-txt', title: 'Crawl Rules', result: robotsTxtResults, Component: RobotsTxtCard, refresh: updateRobotsTxtResults }, - { id: 'sitemap', title: 'Pages', result: sitemapResults, Component: SitemapCard, refresh: updateSitemapResults }, { id: 'dnssec', title: 'DNSSEC', result: dnsSecResults, Component: DnsSecCard, refresh: updateDnsSecResults }, { id: 'status', title: 'Server Status', result: serverStatusResults, Component: ServerStatusCard, refresh: updateServerStatusResults }, { id: 'ports', title: 'Open Ports', result: portsResults, Component: OpenPortsCard, refresh: updatePortsResults }, @@ -448,8 +435,11 @@ const Results = (): JSX.Element => { { id: 'hsts', title: 'HSTS Check', result: hstsResults, Component: HstsCard, refresh: updateHstsResults }, { id: 'whois', title: 'Domain Info', result: whoIsResults, Component: WhoIsCard, refresh: updateWhoIsResults }, { id: 'dns-server', title: 'DNS Server', result: dnsServerResults, Component: DnsServerCard, refresh: updateDnsServerResults }, + { id: 'linked-pages', title: 'Linked Pages', result: linkedPagesResults, Component: ContentLinksCard, refresh: updateLinkedPagesResults }, { id: 'features', title: 'Site Features', result: siteFeaturesResults, Component: SiteFeaturesCard, refresh: updateSiteFeaturesResults }, + { id: 'sitemap', title: 'Pages', result: sitemapResults, Component: SitemapCard, refresh: updateSitemapResults }, { id: 'carbon', title: 'Carbon Footprint', result: carbonResults, Component: CarbonFootprintCard, refresh: updateCarbonResults }, + ]; const MakeActionButtons = (title: string, refresh: () => void, showInfo: (id: string) => void): ReactNode => { @@ -463,34 +453,7 @@ const Results = (): JSX.Element => { }; const showInfo = (id: string) => { - const doc = docs.filter((doc: any) => doc.id === id)[0] || null; - setModalContent( - doc? ( - {doc.title} - About -

    {doc.description}

    - Use Cases -

    {doc.use}

    - Links -
      - {doc.resources.map((resource: string | { title: string, link: string } , index: number) => ( - typeof resource === 'string' ? ( - - ) : ( - - ) - ))} -
    -
    - Example - Screenshot -
    -     ) - : ( - -

    No Docs provided for this widget yet

    -     - )); + setModalContent(DocContent(id)); setModalOpen(true); }; @@ -534,6 +497,8 @@ const Results = (): JSX.Element => { } + +