q25_new

gistfile11.js

function parseRobotFile(robotLines) {
    const rulesByAgent = { DoeBot: [], "*": [] };
    let currentRelevantAgentKey = null; 

    const UA_PREFIX = "User-agent:";
    const DIS_PREFIX = "Disallow:";

    for (const rawLine of robotLines) {
        const line = rawLine.trim();

        if (line.startsWith(UA_PREFIX)) {
            const agentName = line.substring(UA_PREFIX.length).trim();
            if (agentName === "DoeBot" || agentName === "*") {
                currentRelevantAgentKey = agentName;
            } else {
                currentRelevantAgentKey = null; 
            }
        } else if (line.startsWith(DIS_PREFIX) && currentRelevantAgentKey) {
            let path = line.substring(DIS_PREFIX.length).trim();
            
            const commentIndex = path.indexOf('#');
            if (commentIndex !== -1) {
                path = path.substring(0, commentIndex).trim();
            }

            if (path && !path.includes(' ')) {
                rulesByAgent[currentRelevantAgentKey].push(path);
            }
        }
    }

    return [...new Set([...rulesByAgent.DoeBot, ...rulesByAgent["*"]])].sort();
}