Writes get robots, refactors row to be reusable, adds loading progress logic

src/utils/address-type-checker.ts

@@ -7,17 +7,29 @@ export type AddressType = 'ipV4' | 'ipV6' | 'url' | 'err' | 'empt';
 
 /* Checks if a given string looks like a URL */
 const isUrl = (value: string):boolean => {
-  const urlRegex= new RegExp(''
-    // + /(?:(?:(https?|ftp):)?\/\/)/.source
-    + /(?:([^:\n\r]+):([^@\n\r]+)@)?/.source
-    + /(?:(?:www\.)?([^/\n\r]+))/.source
-    + /(\/[^?\n\r]+)?/.source
-    + /(\?[^#\n\r]*)?/.source
-    + /(#?[^\n\r]*)?/.source
+  const urlPattern = new RegExp(
+    '^(https?:\\/\\/)?' + 
+    '(?!([0-9]{1,3}\\.){3}[0-9]{1,3})' + // Exclude IP addresses
+    '(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\\-]*[a-zA-Z0-9])\\.)*' + // Domain name or a subdomain
+    '([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\\-]*[A-Za-z0-9])$', // Second level domain
+    'i' // Case-insensitive
   );
-  return urlRegex.test(value);
+  return urlPattern.test(value);
 };
 
+// /* Checks if a given string looks like a URL */
+// const isUrl = (value: string):boolean => {
+//   const urlRegex= new RegExp(''
+//     // + /(?:(?:(https?|ftp):)?\/\/)/.source
+//     + /(?:([^:\n\r]+):([^@\n\r]+)@)?/.source
+//     + /(?:(?:www\.)?([^/\n\r]+))/.source
+//     + /(\/[^?\n\r]+)?/.source
+//     + /(\?[^#\n\r]*)?/.source
+//     + /(#?[^\n\r]*)?/.source
+//   );
+//   return urlRegex.test(value);
+// };
+
 /* Checks if a given string looks like an IP Version 4 Address */
 const isIpV4 = (value: string): boolean => {
   const ipPart = '(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)';
@@ -51,7 +63,8 @@ const isShort = (value: string): boolean => {
 };
 
 /* Returns the address type for a given address */
-export const determineAddressType = (address: string): AddressType => {
+export const determineAddressType = (address: string | undefined): AddressType => {
+  if (!address) return 'err';
   if (isShort(address)) return 'empt';
   if (isUrl(address)) return 'url';
   if (isIpV4(address)) return 'ipV4';

src/utils/get-keys.ts

@@ -3,6 +3,7 @@ const keys = {
   shodan: process.env.REACT_APP_SHODAN_API_KEY,
   whoApi: process.env.REACT_APP_WHO_API_KEY,
   builtWith: process.env.REACT_APP_BUILT_WITH_API_KEY,
+  googleCloud: process.env.REACT_APP_GOOGLE_CLOUD_API_KEY,
 };
 
 export default keys;

src/utils/result-processor.ts

@@ -57,6 +57,8 @@ export interface ServerInfo {
   os?: string,
   ip?: string,
   ports?: string,
+  loc?: string,
+  type?: string,
 };
 
 export const getServerInfo = (response: any): ServerInfo => {
@@ -67,6 +69,8 @@ export const getServerInfo = (response: any): ServerInfo => {
     os: response.os,
     ip: response.ip_str,
     ports: response.ports.toString(),
+    loc: response.city ? `${response.city}, ${response.country_name}` : '',
+    type: response.tags ? response.tags.toString() : '',
   };
 };
 
@@ -115,3 +119,45 @@ export const makeTechnologies = (response: any): TechnologyGroup[] => {
   }, {});
   return technologies;
 };
+
+export type Cookie = {
+  name: string;
+  value: string;
+  attributes: Record<string, string>;
+};
+
+export const parseCookies = (cookiesHeader: string): Cookie[] => {
+  if (!cookiesHeader) return [];
+  return cookiesHeader.split(/,(?=\s[A-Za-z0-9]+=)/).map(cookieString => {
+    const [nameValuePair, ...attributePairs] = cookieString.split('; ').map(part => part.trim());
+    const [name, value] = nameValuePair.split('=');
+    const attributes: Record<string, string> = {};
+    attributePairs.forEach(pair => {
+      const [attributeName, attributeValue = ''] = pair.split('=');
+      attributes[attributeName] = attributeValue;
+    });
+    return { name, value, attributes };
+  });
+}
+
+type RobotsRule = {
+  lbl: string;
+  val: string;
+};
+
+export const parseRobotsTxt = (content: string): RobotsRule[] => {
+  const lines = content.split('\n');
+  const rules: RobotsRule[] = [];
+  lines.forEach(line => {
+    const match = line.match(/^(Allow|Disallow):\s*(\S*)$/);
+    if (match) {
+      const rule: RobotsRule = {
+        lbl: match[1],
+        val: match[2],
+      };
+      
+      rules.push(rule);
+    }
+  });
+  return rules;
+}