diff --git a/netlify.toml b/netlify.toml index a9952a7..2d7bc48 100644 --- a/netlify.toml +++ b/netlify.toml @@ -59,6 +59,16 @@ to = "/.netlify/functions/get-cookies" status = 301 force = true +[[redirects]] + from = "/get-dns" + to = "/.netlify/functions/get-dns" + status = 301 + force = true +[[redirects]] + from = "/read-robots-txt" + to = "/.netlify/functions/read-robots-txt" + status = 301 + force = true # For router history mode, ensure pages land on index [[redirects]] diff --git a/src/components/Form/Heading.tsx b/src/components/Form/Heading.tsx index 34b577a..2091f85 100644 --- a/src/components/Form/Heading.tsx +++ b/src/components/Form/Heading.tsx @@ -17,10 +17,14 @@ const StyledHeading = styled.h1` display: flex; gap: 1rem; align-items: center; - img { + img { // Some titles have an icon width: 2rem; border-radius: 4px; } + a { // If a title is a link, keep title styles + color: inherit; + text-decoration: none; + } ${props => { switch (props.size) { case 'xSmall': return `font-size: ${TextSizes.small};`; diff --git a/src/components/Form/Row.tsx b/src/components/Form/Row.tsx index 043f173..ea96860 100644 --- a/src/components/Form/Row.tsx +++ b/src/components/Form/Row.tsx @@ -1,21 +1,23 @@ import styled from 'styled-components'; import colors from 'styles/colors'; -interface RowProps { +export interface RowProps { lbl: string, val: string, key?: string, + children?: JSX.Element[], rowList?: RowProps[], + title?: string, } -const StyledRow = styled.div` +export const StyledRow = styled.div` display: flex; justify-content: space-between; padding: 0.25rem; &:not(:last-child) { border-bottom: 1px solid ${colors.primary}; } span.lbl { font-weight: bold; } span.val { - max-width: 200px; + max-width: 16rem; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; @@ -64,7 +66,6 @@ const formatDate = (dateString: string): string => { const formatValue = (value: any): string => { const isValidDate = (date: any) => date instanceof Date && !isNaN(date as any as number); if (isValidDate(new Date(value))) return formatDate(value); - if (typeof value === 'object') return JSON.stringify(value); if (typeof value === 'boolean') return value ? '✅' : '❌'; return value; }; @@ -74,11 +75,11 @@ const copyToClipboard = (text: string) => { } export const ExpandableRow = (props: RowProps) => { - const { lbl, val, rowList } = props; + const { lbl, val, key, title, rowList } = props; return (
- - {lbl} + + {lbl} {val} { rowList && @@ -86,7 +87,7 @@ export const ExpandableRow = (props: RowProps) => { { rowList?.map((row: RowProps, index: number) => { return ( - {row.lbl} + {row.lbl} copyToClipboard(row.val)}> {formatValue(row.val)} @@ -100,10 +101,11 @@ export const ExpandableRow = (props: RowProps) => { }; const Row = (props: RowProps) => { - const { lbl, val, key } = props; + const { lbl, val, key, title, children } = props; + if (children) return {children}; return ( - {lbl} + {lbl} copyToClipboard(val)}> {formatValue(val)} diff --git a/src/components/Results/Cookies.tsx b/src/components/Results/Cookies.tsx index d910289..bcb4da4 100644 --- a/src/components/Results/Cookies.tsx +++ b/src/components/Results/Cookies.tsx @@ -3,17 +3,18 @@ import styled from 'styled-components'; import colors from 'styles/colors'; import Card from 'components/Form/Card'; import Heading from 'components/Form/Heading'; -import Row, { ExpandableRow } from 'components/Form/Row'; +import { ExpandableRow } from 'components/Form/Row'; const Outer = styled(Card)``; const CookiesCard = (props: { cookies: any }): JSX.Element => { const cookies = props.cookies; - console.log('COOKIES: ', cookies); return ( Cookies - {/* { subject && } */} + { + cookies.length === 0 &&

No cookies found.

+ } { cookies.map((cookie: any, index: number) => { const attributes = Object.keys(cookie.attributes).map((key: string) => { diff --git a/src/components/Results/Headers.tsx b/src/components/Results/Headers.tsx index 38c6511..741d08e 100644 --- a/src/components/Results/Headers.tsx +++ b/src/components/Results/Headers.tsx @@ -3,35 +3,12 @@ import styled from 'styled-components'; import colors from 'styles/colors'; import Card from 'components/Form/Card'; import Heading from 'components/Form/Heading'; +import Row from 'components/Form/Row'; const Outer = styled(Card)` grid-row: span 2; `; -const Row = styled.div` - display: flex; - justify-content: space-between; - padding: 0.25rem; - &:not(:last-child) { border-bottom: 1px solid ${colors.primary}; } - span.lbl { font-weight: bold; } - span.val { - max-width: 200px; - white-space: nowrap; - overflow: hidden; - text-overflow: ellipsis; - } -`; - -const DataRow = (props: { lbl: string, val: string }) => { - const { lbl, val } = props; - return ( - - {lbl} - {val} - - ); -}; - const HeadersCard = (props: { headers: any }): JSX.Element => { const headers = props.headers; return ( @@ -40,7 +17,7 @@ const HeadersCard = (props: { headers: any }): JSX.Element => { { Object.keys(headers).map((header: string, index: number) => { return ( - + ) }) } diff --git a/src/components/Results/Lighthouse.tsx b/src/components/Results/Lighthouse.tsx index 733cef4..0f990c9 100644 --- a/src/components/Results/Lighthouse.tsx +++ b/src/components/Results/Lighthouse.tsx @@ -4,6 +4,7 @@ import styled from 'styled-components'; import colors from 'styles/colors'; import Card from 'components/Form/Card'; import Heading from 'components/Form/Heading'; +import { ExpandableRow } from 'components/Form/Row'; const processScore = (percentile: number) => { return `${Math.round(percentile * 100)}%`; @@ -78,23 +79,14 @@ const ScoreItem = (props: { scoreId: any, audits: Audit[] }) => { ); }; -const ExpandableRow = (props: { lbl: string, val: string, list: string[], audits: Audit[] }) => { - const { lbl, val, list, audits } = props; - return ( - -
- - {lbl} - {val} - - - { list.map((li: string) => { - return - }) } - -
- - ); +const makeValue = (audit: Audit) => { + let score = audit.score; + if (audit.displayValue) { + score = audit.displayValue; + } else if (audit.scoreDisplayMode) { + score = audit.score === 1 ? '✅ Pass' : '❌ Fail'; + } + return score; }; const LighthouseCard = (props: { lighthouse: any }): JSX.Element => { @@ -106,13 +98,15 @@ const LighthouseCard = (props: { lighthouse: any }): JSX.Element => { Performance { Object.keys(categories).map((title: string, index: number) => { const scoreIds = categories[title].auditRefs.map((ref: { id: string }) => ref.id); + const scoreList = scoreIds.map((id: string) => { + return { lbl: audits[id].title, val: makeValue(audits[id]), title: audits[id].description, key: id } + }) return ( ); }) } diff --git a/src/components/Results/RobotsTxt.tsx b/src/components/Results/RobotsTxt.tsx new file mode 100644 index 0000000..105b364 --- /dev/null +++ b/src/components/Results/RobotsTxt.tsx @@ -0,0 +1,35 @@ + +import styled from 'styled-components'; +import colors from 'styles/colors'; +import Card from 'components/Form/Card'; +import Heading from 'components/Form/Heading'; +import Row, { RowProps } from 'components/Form/Row'; + +const Outer = styled(Card)` + .content { + max-height: 20rem; + overflow-y: auto; + } +`; + +const RobotsTxtCard = (props: { robotTxt: RowProps[] }): JSX.Element => { + return ( + + Crawl Rules +
+ { + props.robotTxt.length === 0 &&

No crawl rules found.

+ } + { + props.robotTxt.map((row: RowProps, index: number) => { + return ( + + ) + }) + } +
+ + ); +} + +export default RobotsTxtCard; diff --git a/src/components/Results/Screenshot.tsx b/src/components/Results/Screenshot.tsx index a84b88e..925a1cb 100644 --- a/src/components/Results/Screenshot.tsx +++ b/src/components/Results/Screenshot.tsx @@ -6,7 +6,11 @@ import Heading from 'components/Form/Heading'; const Outer = styled(Card)` overflow: auto; -max-height: 20rem; +max-height: 28rem; +img { + border-radius: 6px; + width: 100%; +} `; const ScreenshotCard = (props: { screenshot: string }): JSX.Element => { diff --git a/src/components/Results/ServerLocation.tsx b/src/components/Results/ServerLocation.tsx index 50bebf5..4c4bc34 100644 --- a/src/components/Results/ServerLocation.tsx +++ b/src/components/Results/ServerLocation.tsx @@ -7,27 +7,12 @@ import Heading from 'components/Form/Heading'; import LocationMap from 'components/misc/LocationMap'; import Flag from 'components/misc/Flag'; import { TextSizes } from 'styles/typography'; +import Row, { StyledRow } from 'components/Form/Row'; const Outer = styled(Card)` grid-row: span 2 `; -const Row = styled.div` - display: flex; - justify-content: space-between; - padding: 0.25rem; - &:not(:last-child) { border-bottom: 1px solid ${colors.primary}; } -`; - -const RowLabel = styled.span` - font-weight: bold; -`; - -const RowValue = styled.span` - display: flex; - img { margin-left: 0.5rem; } -`; - const SmallText = styled.span` opacity: 0.5; font-size: ${TextSizes.xSmall}; @@ -35,10 +20,16 @@ const SmallText = styled.span` display: block; `; -const MapRow = styled(Row)` +const MapRow = styled(StyledRow)` + padding-top: 1rem; flex-direction: column; `; +const CountryValue = styled.span` + display: flex; + gap: 0.5rem; +`; + const ServerLocationCard = (location: ServerLocation): JSX.Element => { const { @@ -50,28 +41,17 @@ const ServerLocationCard = (location: ServerLocation): JSX.Element => { return ( Location - - City - - {postCode}, { city }, { region } - - - - Country - + + + Country + {country} { countryCode && } - + - { timezone && - Timezone{timezone} - } - { languages && - Languages{languages} - } - { currency && - Currency{currency} ({currencyCode}) - } + + + Latitude: {coords.latitude}, Longitude: {coords.longitude} diff --git a/src/pages/Home.tsx b/src/pages/Home.tsx index 2b43253..e9a07fc 100644 --- a/src/pages/Home.tsx +++ b/src/pages/Home.tsx @@ -81,7 +81,6 @@ const Home = (): JSX.Element => { fetch('https://ipapi.co/json/') .then(function(response) { response.json().then(jsonData => { - console.log(jsonData); setUserInput(jsonData.ip); setPlaceholder(defaultPlaceholder); setInputDisabled(true); diff --git a/src/pages/Results.tsx b/src/pages/Results.tsx index ee35ccf..19f0cbc 100644 --- a/src/pages/Results.tsx +++ b/src/pages/Results.tsx @@ -13,6 +13,9 @@ import BuiltWithCard from 'components/Results/BuiltWith'; import LighthouseCard from 'components/Results/Lighthouse'; import ScreenshotCard from 'components/Results/Screenshot'; import SslCertCard from 'components/Results/SslCert'; +import HeadersCard from 'components/Results/Headers'; +import CookiesCard from 'components/Results/Cookies'; +import RobotsTxtCard from 'components/Results/RobotsTxt'; import keys from 'utils/get-keys'; import { determineAddressType, AddressType } from 'utils/address-type-checker'; @@ -21,6 +24,8 @@ import { getServerInfo, ServerInfo, getHostNames, HostNames, makeTechnologies, TechnologyGroup, + parseCookies, Cookie, + parseRobotsTxt, Whois, } from 'utils/result-processor'; @@ -49,23 +54,68 @@ const Header = styled(Card)` padding: 0.5rem 1rem; `; -interface ResultsType { - serverLocation?: ServerLocation, - serverInfo?: ServerInfo, - hostNames?: HostNames | null, -}; - const Results = (): JSX.Element => { - const [ results, setResults ] = useState({}); + const [ serverInfo, setServerInfo ] = useState(); + const [ hostNames, setHostNames ] = useState(); const [ locationResults, setLocationResults ] = useState(); const [ whoIsResults, setWhoIsResults ] = useState(); const [ technologyResults, setTechnologyResults ] = useState(); const [ lighthouseResults, setLighthouseResults ] = useState(); const [ sslResults, setSslResults ] = useState(); + const [ headersResults, setHeadersResults ] = useState(); + const [ robotsTxtResults, setRobotsTxtResults ] = useState(); + const [ cookieResults, setCookieResults ] = useState(null); const [ screenshotResult, setScreenshotResult ] = useState(); const [ ipAddress, setIpAddress ] = useState(undefined); const [ addressType, setAddressType ] = useState('empt'); const { address } = useParams(); + + type LoadingState = 'loading' | 'skipped' | 'success' | 'error'; + + interface LoadingJob { + name: string, + state: LoadingState, + error?: string, + } + + const jobNames = [ + 'get-ip', + 'ssl', + 'cookies', + 'robots-txt', + 'headers', + 'lighthouse', + 'location', + 'server-info', + 'whois', + ] as const; + + const initialJobs = jobNames.map((job: string) => { + return { + name: job, + state: 'loading' as LoadingState, + } + }); + + const [ loadingJobs, setLoadingJobs ] = useState(initialJobs); + + const updateLoadingJobs = (job: string, newState: LoadingState, error?: string) => { + setLoadingJobs((prevJobs) => { + const newJobs = prevJobs.map((loadingJob: LoadingJob) => { + if (loadingJob.name === job) { + return { ...loadingJob, error, state: newState }; + } + return loadingJob; + }); + + if (newState === 'error') { + console.warn(`Error in ${job}: ${error}`); + } + + return newJobs; + }); + }; + useEffect(() => { @@ -77,43 +127,102 @@ const Results = (): JSX.Element => { /* Get IP address from URL */ useEffect(() => { + if (addressType !== 'url') return; const fetchIpAddress = () => { fetch(`/find-url-ip?address=${address}`) .then(function(response) { response.json().then(jsonData => { setIpAddress(jsonData.ip); + updateLoadingJobs('get-ip', 'success'); }); }) .catch(function(error) { - console.log(error) + updateLoadingJobs('get-ip', 'error', error); }); }; if (!ipAddress) { fetchIpAddress(); } - }, [address]); + }, [address, addressType]); /* Get SSL info */ useEffect(() => { + if (addressType !== 'url') return; fetch(`/ssl-check?url=${address}`) .then(response => response.json()) .then(response => { - console.log(response); - setSslResults(response); + if (Object.keys(response).length > 0) { + setSslResults(response); + updateLoadingJobs('ssl', 'success'); + } else { + updateLoadingJobs('ssl', 'error', 'No SSL Cert found'); + } }) - .catch(err => console.error(err)); - }, [address]) + .catch(err => updateLoadingJobs('ssl', 'error', err)); + }, [address, addressType]) + + /* Get Cookies */ + useEffect(() => { + if (addressType !== 'url') return; + fetch(`/get-cookies?url=${address}`) + .then(response => response.json()) + .then(response => { + setCookieResults(parseCookies(response.cookies)); + updateLoadingJobs('cookies', 'success'); + }) + .catch(err => updateLoadingJobs('cookies', 'error', err)); + }, [address, addressType]) + + /* Get Robots.txt */ + useEffect(() => { + if (addressType !== 'url') return; + fetch(`/read-robots-txt?url=${address}`) + .then(response => response.text()) + .then(response => { + setRobotsTxtResults(parseRobotsTxt(response)); + updateLoadingJobs('robots-txt', 'success'); + }) + .catch(err => updateLoadingJobs('robots-txt', 'error', err)); + }, [address, addressType]) + + /* Get Headers */ + useEffect(() => { + if (addressType !== 'url') return; + fetch(`/get-headers?url=${address}`) + .then(response => response.json()) + .then(response => { + setHeadersResults(response); + updateLoadingJobs('headers', 'success'); + }) + .catch(err => updateLoadingJobs('headers', 'error', err)); + }, [address, addressType]) /* Get Lighthouse report */ useEffect(() => { + if (addressType !== 'url') return; fetch(`/lighthouse-report?url=${address}`) .then(response => response.json()) .then(response => { setLighthouseResults(response.lighthouseResult); setScreenshotResult(response.lighthouseResult?.fullPageScreenshot?.screenshot?.data); + updateLoadingJobs('lighthouse', 'success'); }) - .catch(err => console.error(err)); - }, [address]) + .catch(err => { + // if (err.errorType === 'TimeoutError') { + // Netlify limits to 10 seconds, we can try again client-side... + const params = 'category=PERFORMANCE&category=ACCESSIBILITY&category=BEST_PRACTICES&category=SEO&category=PWA&strategy=mobile'; + const endpoint = `https://www.googleapis.com/pagespeedonline/v5/runPagespeed?url=${address}&${params}&key=${keys.googleCloud}`; + fetch(endpoint) + .then(response => response.json()) + .then(response => { + setLighthouseResults(response.lightHouseResult); + setScreenshotResult(response?.lighthouseResult?.fullPageScreenshot?.screenshot?.data); + updateLoadingJobs('lighthouse', 'success'); + }) + .catch(err => updateLoadingJobs('lighthouse', 'error', err)); + }); + }, [address, addressType]) + /* Get IP address location info */ useEffect(() => { @@ -122,10 +231,11 @@ const Results = (): JSX.Element => { .then(function(response) { response.json().then(jsonData => { setLocationResults(getLocation(jsonData)); + updateLoadingJobs('location', 'success'); }); }) .catch(function(error) { - console.log(error) + updateLoadingJobs('location', 'error', error); }); }; if (ipAddress) { @@ -136,18 +246,20 @@ const Results = (): JSX.Element => { /* Get hostnames and server info from Shodan */ useEffect(() => { const applyShodanResults = (response: any) => { - const serverInfo = getServerInfo(response); - const hostNames = getHostNames(response); - setResults({...results, serverInfo, hostNames }); + setServerInfo(getServerInfo(response)); + setHostNames(getHostNames(response)); } const fetchShodanData = () => { const apiKey = keys.shodan; fetch(`https://api.shodan.io/shodan/host/${ipAddress}?key=${apiKey}`) .then(response => response.json()) .then(response => { - if (!response.error) applyShodanResults(response) + if (!response.error) { + applyShodanResults(response) + updateLoadingJobs('server-info', 'success'); + } }) - .catch(err => console.error(err)); + .catch(err => updateLoadingJobs('server-info', 'error', err)); }; @@ -158,17 +270,21 @@ const Results = (): JSX.Element => { /* Get BuiltWith tech stack */ useEffect(() => { + if (addressType !== 'url') return; const apiKey = keys.builtWith; const endpoint = `https://api.builtwith.com/v21/api.json?KEY=${apiKey}&LOOKUP=${address}`; fetch(endpoint) .then(response => response.json()) .then(response => { setTechnologyResults(makeTechnologies(response)); - }); - }, [address]); + updateLoadingJobs('built-with', 'success'); + }) + .catch(err => updateLoadingJobs('built-with', 'error', err)); + }, [address, addressType]); /* Get WhoIs info for a given domain name */ useEffect(() => { + if (addressType !== 'url') return; const applyWhoIsResults = (response: any) => { const whoIsResults: Whois = { created: response.date_created, @@ -184,8 +300,9 @@ const Results = (): JSX.Element => { .then(response => response.json()) .then(response => { if (!response.error) applyWhoIsResults(response) + updateLoadingJobs('whois', 'success'); }) - .catch(err => console.error(err)); + .catch(err => updateLoadingJobs('whois', 'error', err)); }; if (addressType === 'url') { @@ -193,24 +310,40 @@ const Results = (): JSX.Element => { } }, [addressType, address]); + const makeSiteName = (address: string): string => { + try { + return new URL(address).hostname.replace('www.', ''); + } catch (error) { + return address; + } + } + return (
- Results - - { address && } - {address} + + Web Check + { address && + + { addressType === 'url' && } + {makeSiteName(address)} + + }
{ locationResults && } - { results.serverInfo && } - { results?.hostNames && } + { sslResults && } + { headersResults && } + { hostNames && } { whoIsResults && } { lighthouseResults && } + { cookieResults && } { screenshotResult && } { technologyResults && } - { sslResults && } + { robotsTxtResults && } + { serverInfo && } + ); diff --git a/src/utils/address-type-checker.ts b/src/utils/address-type-checker.ts index 636e477..51003cb 100644 --- a/src/utils/address-type-checker.ts +++ b/src/utils/address-type-checker.ts @@ -7,17 +7,29 @@ export type AddressType = 'ipV4' | 'ipV6' | 'url' | 'err' | 'empt'; /* Checks if a given string looks like a URL */ const isUrl = (value: string):boolean => { - const urlRegex= new RegExp('' - // + /(?:(?:(https?|ftp):)?\/\/)/.source - + /(?:([^:\n\r]+):([^@\n\r]+)@)?/.source - + /(?:(?:www\.)?([^/\n\r]+))/.source - + /(\/[^?\n\r]+)?/.source - + /(\?[^#\n\r]*)?/.source - + /(#?[^\n\r]*)?/.source + const urlPattern = new RegExp( + '^(https?:\\/\\/)?' + + '(?!([0-9]{1,3}\\.){3}[0-9]{1,3})' + // Exclude IP addresses + '(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\\-]*[a-zA-Z0-9])\\.)*' + // Domain name or a subdomain + '([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\\-]*[A-Za-z0-9])$', // Second level domain + 'i' // Case-insensitive ); - return urlRegex.test(value); + return urlPattern.test(value); }; +// /* Checks if a given string looks like a URL */ +// const isUrl = (value: string):boolean => { +// const urlRegex= new RegExp('' +// // + /(?:(?:(https?|ftp):)?\/\/)/.source +// + /(?:([^:\n\r]+):([^@\n\r]+)@)?/.source +// + /(?:(?:www\.)?([^/\n\r]+))/.source +// + /(\/[^?\n\r]+)?/.source +// + /(\?[^#\n\r]*)?/.source +// + /(#?[^\n\r]*)?/.source +// ); +// return urlRegex.test(value); +// }; + /* Checks if a given string looks like an IP Version 4 Address */ const isIpV4 = (value: string): boolean => { const ipPart = '(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)'; @@ -51,7 +63,8 @@ const isShort = (value: string): boolean => { }; /* Returns the address type for a given address */ -export const determineAddressType = (address: string): AddressType => { +export const determineAddressType = (address: string | undefined): AddressType => { + if (!address) return 'err'; if (isShort(address)) return 'empt'; if (isUrl(address)) return 'url'; if (isIpV4(address)) return 'ipV4'; diff --git a/src/utils/get-keys.ts b/src/utils/get-keys.ts index 4ef07ec..b8aebd4 100644 --- a/src/utils/get-keys.ts +++ b/src/utils/get-keys.ts @@ -3,6 +3,7 @@ const keys = { shodan: process.env.REACT_APP_SHODAN_API_KEY, whoApi: process.env.REACT_APP_WHO_API_KEY, builtWith: process.env.REACT_APP_BUILT_WITH_API_KEY, + googleCloud: process.env.REACT_APP_GOOGLE_CLOUD_API_KEY, }; export default keys; diff --git a/src/utils/result-processor.ts b/src/utils/result-processor.ts index e50309d..2a5e48b 100644 --- a/src/utils/result-processor.ts +++ b/src/utils/result-processor.ts @@ -57,6 +57,8 @@ export interface ServerInfo { os?: string, ip?: string, ports?: string, + loc?: string, + type?: string, }; export const getServerInfo = (response: any): ServerInfo => { @@ -67,6 +69,8 @@ export const getServerInfo = (response: any): ServerInfo => { os: response.os, ip: response.ip_str, ports: response.ports.toString(), + loc: response.city ? `${response.city}, ${response.country_name}` : '', + type: response.tags ? response.tags.toString() : '', }; }; @@ -115,3 +119,45 @@ export const makeTechnologies = (response: any): TechnologyGroup[] => { }, {}); return technologies; }; + +export type Cookie = { + name: string; + value: string; + attributes: Record; +}; + +export const parseCookies = (cookiesHeader: string): Cookie[] => { + if (!cookiesHeader) return []; + return cookiesHeader.split(/,(?=\s[A-Za-z0-9]+=)/).map(cookieString => { + const [nameValuePair, ...attributePairs] = cookieString.split('; ').map(part => part.trim()); + const [name, value] = nameValuePair.split('='); + const attributes: Record = {}; + attributePairs.forEach(pair => { + const [attributeName, attributeValue = ''] = pair.split('='); + attributes[attributeName] = attributeValue; + }); + return { name, value, attributes }; + }); +} + +type RobotsRule = { + lbl: string; + val: string; +}; + +export const parseRobotsTxt = (content: string): RobotsRule[] => { + const lines = content.split('\n'); + const rules: RobotsRule[] = []; + lines.forEach(line => { + const match = line.match(/^(Allow|Disallow):\s*(\S*)$/); + if (match) { + const rule: RobotsRule = { + lbl: match[1], + val: match[2], + }; + + rules.push(rule); + } + }); + return rules; +}