local http = require "http"
local nmap = require "nmap"
local shortport = require "shortport"
local strbuf = require "strbuf"
local table = require "table"
local stdnse = require "stdnse"
description = [[
Checks for disallowed entries in /robots.txt on a web server.
The higher the verbosity or debug level, the more disallowed entries are shown.
]]
---
--@output
-- 80/tcp open http syn-ack
-- | http-robots.txt:
-- | entry_count: 241
-- | display_count: 15
-- | entries:
-- | /search /sdch /groups /images /catalogs
-- | /news /nwshp /setnewsprefs? /index.html? /?
-- |_/addurl/image? /pagead/ /relpage/
--
--@xmloutput
--
author = "Eddie Bell"
license = "Same as Nmap--See http://nmap.org/book/man-legal.html"
categories = {"default", "discovery", "safe"}
portrule = shortport.http
local last_len = 0
-- split the output in 50 character length lines
local function wordwrap(output)
local line = strbuf.new()
local out = strbuf.new("")
local line_len = 0
for i,v in ipairs(output) do
if line_len + v:len() + 1 > 50 then
stdnse.print_debug(1, "i=%d, eol", i)
out = out .. strbuf.dump(line, ' ')
line_len = 0
line = strbuf.new(v)
else
stdnse.print_debug(1, "i=%d, line_len=%d", i, line_len)
line_len = line_len + v:len() + 1
line = line .. v
end
end
if line_len > 0 then
stdnse.print_debug(1, "line_len=%d", line_len)
out = out .. strbuf.dump(line, ' ')
end
return tostring(out)
end
local function buildOutput(output, w)
if w:len() == 0 then
return nil
end
-- check for duplicates
for i,v in ipairs(output) do
if w == v or w == v:sub(2) then
return nil
end
end
output[#output+1] = w
end
-- parse all disallowed entries in body and add them to a strbuf
local function parse_robots(body, output)
for line in body:gmatch("[^\r\n]+") do
for w in line:gmatch('[Dd]isallow:%s*(.*)') do
w = w:gsub("%s*#.*", "")
buildOutput(output, w)
end
end
return #output
end
action = function(host, port)
local dis_count
local answer = http.get(host, port, "/robots.txt" )
if answer.status ~= 200 then
return nil
end
local o = {}
local v_level = nmap.verbosity() + (nmap.debugging()*2)
local entries = {}
local detail = 15
dis_count = parse_robots(answer.body, entries)
if dis_count == 0 then
return
end
-- verbose/debug mode, print 50 entries
if v_level > 1 and v_level < 5 then
detail = 40
-- double debug mode, print everything
elseif v_level >= 5 then
detail = dis_count
end
-- check we have enough entries
if detail > dis_count then
detail = dis_count
end
o.entry_count = dis_count
local output_order = {"entry_count", "entries"}
if (detail ~= 0 and detail ~= dis_count) then
o.display_count = detail
table.insert(entries, detail+1, nil) -- Hide the rest
output_order = {"entry_count", "display_count", "entries"}
end
stdnse.set_tostring(entries, wordwrap)
o.entries = entries
stdnse.set_tostring(o,
stdnse.format_generator({
key_order = output_order
}))
return o
end