diff --git a/api/legacy-rank.js b/api/legacy-rank.js new file mode 100644 index 0000000..9a9f795 --- /dev/null +++ b/api/legacy-rank.js @@ -0,0 +1,70 @@ +const axios = require('axios'); +const unzipper = require('unzipper'); +const csv = require('csv-parser'); +const fs = require('fs'); +const middleware = require('./_common/middleware'); + +// Should also work with the following sources: +// https://www.domcop.com/files/top/top10milliondomains.csv.zip +// https://tranco-list.eu/top-1m.csv.zip +// https://www.domcop.com/files/top/top10milliondomains.csv.zip +// https://radar.cloudflare.com/charts/LargerTopDomainsTable/attachment?id=525&top=1000000 +// https://statvoo.com/dl/top-1million-sites.csv.zip + +const FILE_URL = 'https://s3-us-west-1.amazonaws.com/umbrella-static/top-1m.csv.zip'; +const TEMP_FILE_PATH = '/tmp/top-1m.csv'; + +const handler = async (url) => { + let domain = null; + + try { + domain = new URL(url).hostname; + } catch (e) { + throw new Error('Invalid URL'); + } + +// Download and unzip the file if not in cache +if (!fs.existsSync(TEMP_FILE_PATH)) { + const response = await axios({ + method: 'GET', + url: FILE_URL, + responseType: 'stream' + }); + + await new Promise((resolve, reject) => { + response.data + .pipe(unzipper.Extract({ path: '/tmp' })) + .on('close', resolve) + .on('error', reject); + }); +} + +// Parse the CSV and find the rank +return new Promise((resolve, reject) => { + const csvStream = fs.createReadStream(TEMP_FILE_PATH) + .pipe(csv({ + headers: ['rank', 'domain'], + })) + .on('data', (row) => { + if (row.domain === domain) { + csvStream.destroy(); + resolve({ + domain: domain, + rank: row.rank, + isFound: true, + }); + } + }) + .on('end', () => { + resolve({ + skipped: `Skipping, as ${domain} is not present in the Umbrella top 1M list.`, + domain: domain, + isFound: false, + }); + }) + .on('error', reject); +}); +}; + +exports.handler = middleware(handler); + diff --git a/api/rank.js b/api/rank.js new file mode 100644 index 0000000..a156d38 --- /dev/null +++ b/api/rank.js @@ -0,0 +1,24 @@ +const axios = require('axios'); +const middleware = require('./_common/middleware'); + +const handler = async (url) => { + const domain = url ? new URL(url).hostname : null; + if (!domain) throw new Error('Invalid URL'); + + try { + const auth = process.env.TRANCO_API_KEY ? // Auth is optional. + { auth: { username: process.env.TRANCO_USERNAME, password: process.env.TRANCO_API_KEY } } + : {}; + const response = await axios.get( + `https://tranco-list.eu/api/ranks/domain/${domain}`, { timeout: 2000 }, auth, + ); + if (!response.data || !response.data.ranks || response.data.ranks.length === 0) { + return { skipped: `Skipping, as ${domain} isn't ranked in the top 100 million sites yet.`}; + } + return response.data; + } catch (error) { + return { error: `Unable to fetch rank, ${error.message}` }; + } +}; + +exports.handler = middleware(handler); diff --git a/src/components/Results/Rank.tsx b/src/components/Results/Rank.tsx new file mode 100644 index 0000000..ff21712 --- /dev/null +++ b/src/components/Results/Rank.tsx @@ -0,0 +1,90 @@ + +import { AreaChart, Area, Tooltip, CartesianGrid, ResponsiveContainer } from 'recharts'; +import colors from 'styles/colors'; +import { Card } from 'components/Form/Card'; +import Row from 'components/Form/Row'; + +const cardStyles = ` +span.val { + &.up { color: ${colors.success}; } + &.down { color: ${colors.danger}; } +} +.rank-average { + text-align: center; + font-size: 1.8rem; + font-weight: bold; +} +.chart-container { + margin-top: 1rem; +} +`; + +const makeRankStats = (data: {date: string, rank: number }[]) => { + const average = Math.round(data.reduce((acc, cur) => acc + cur.rank, 0) / data.length); + const today = data[0].rank; + const yesterday = data[1].rank; + const percentageChange = ((today - yesterday) / yesterday) * 100; + return { + average, + percentageChange + }; +}; + +const makeChartData = (data: {date: string, rank: number }[]) => { + return data.map((d) => { + return { + date: d.date, + uv: d.rank + }; + }); +}; + +const RankCard = (props: { data: any, title: string, actionButtons: any }): JSX.Element => { + const data = props.data.ranks || []; + const { average, percentageChange } = makeRankStats(data); + const chartData = makeChartData(data); + return ( + +
{data[0].rank.toLocaleString()}
+ 0 ? '+':''} ${percentageChange.toFixed(2)}%`} /> + +
+ + + + + + + + + + ['Date : ', data[value].date]}/> + + + +
+ + {/* { domain.Domain_Name && } + { domain.Creation_Date && } + { domain.Updated_Date && } + { domain.Registry_Expiry_Date && } + { domain.Registry_Domain_ID && } + { domain.Registrar_WHOIS_Server && } + { domain.Registrar && + Registrar + {domain.Registrar} + } + { domain.Registrar_IANA_ID && } */} + + {/* + Is Up? + { serverStatus.isUp ? ✅ Online : ❌ Offline} + + + { serverStatus.responseTime && } */} + + ); +} + +export default RankCard; diff --git a/src/pages/Results.tsx b/src/pages/Results.tsx index 00c3004..279a0c5 100644 --- a/src/pages/Results.tsx +++ b/src/pages/Results.tsx @@ -51,6 +51,7 @@ import MailConfigCard from 'components/Results/MailConfig'; import HttpSecurityCard from 'components/Results/HttpSecurity'; import FirewallCard from 'components/Results/Firewall'; import ArchivesCard from 'components/Results/Archives'; +import RankCard from 'components/Results/Rank'; import keys from 'utils/get-keys'; import { determineAddressType, AddressType } from 'utils/address-type-checker'; @@ -430,6 +431,14 @@ const Results = (): JSX.Element => { fetchRequest: () => fetch(`${api}/archives?url=${address}`).then(res => parseJson(res)), }); + // Get website's global ranking, from Tranco + const [rankResults, updateRankResults] = useMotherHook({ + jobId: 'rank', + updateLoadingJobs, + addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly }, + fetchRequest: () => fetch(`${api}/rank?url=${address}`).then(res => parseJson(res)), + }); + /* Cancel remaining jobs after 10 second timeout */ useEffect(() => { const checkJobs = () => { @@ -475,6 +484,7 @@ const Results = (): JSX.Element => { { id: 'status', title: 'Server Status', result: serverStatusResults, Component: ServerStatusCard, refresh: updateServerStatusResults }, { id: 'ports', title: 'Open Ports', result: portsResults, Component: OpenPortsCard, refresh: updatePortsResults }, { id: 'security-txt', title: 'Security.Txt', result: securityTxtResults, Component: SecurityTxtCard, refresh: updateSecurityTxtResults }, + { id: 'rank', title: 'Global Ranking', result: rankResults, Component: RankCard, refresh: updateRankResults }, { id: 'screenshot', title: 'Screenshot', result: screenshotResult || lighthouseResults?.fullPageScreenshot?.screenshot, Component: ScreenshotCard, refresh: updateScreenshotResult }, { id: 'mail-config', title: 'Email Configuration', result: mailConfigResults, Component: MailConfigCard, refresh: updateMailConfigResults }, { id: 'hsts', title: 'HSTS Check', result: hstsResults, Component: HstsCard, refresh: updateHstsResults },