andrea.morten/web-check-2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Adds component for displaying DNS results

Browse Source
This commit is contained in:
Alicia Sykes
2023-06-24 16:43:07 +01:00
parent e0ffc8e418
commit 8d8307e60e
2 changed files with 34 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
src/utils/result-processor.ts
View File

@@ -148,8 +148,11 @@ type RobotsRule = {
export const parseRobotsTxt = (content: string): RobotsRule[] => {
const lines = content.split('\n');
const rules: RobotsRule[] = [];
lines.forEach(line => {
const match = line.match(/^(Allow|Disallow):\s*(\S*)$/);
line = line.trim(); // This removes trailing and leading whitespaces
let match = line.match(/^(Allow|Disallow):\s*(\S*)$/i);
if (match) {
const rule: RobotsRule = {
lbl: match[1],
@@ -157,7 +160,20 @@ export const parseRobotsTxt = (content: string): RobotsRule[] => {
};
rules.push(rule);
} else {
match = line.match(/^(User-agent):\s*(\S*)$/i);
if (match) {
const rule: RobotsRule = {
lbl: match[1],
val: match[2],
};
rules.push(rule);
}
}
});
return rules;
}