# /robots.txt for http://www.trojanypl.org/


# disallow a bunch of ill-behaved user agents (doubt this will deter them...)

User-agent: TurnitinBot
Disallow: /

User-agent: ExtractorPro
Disallow: /

User-agent: EmailSiphon
Disallow: /

User-agent: EmailWolf 
Disallow: /

User-agent: CherryPickerSE/1.0
Disallow: /

User-agent: CherryPickerElite/1.0
Disallow: /

User-agent: EmailCollector/1.0
Disallow: /

User-agent: EmailWolf 1.00
Disallow: /

User-agent: Crescent Internet ToolPak HTTP OLE Control v.1.0
Disallow: /

User-agent: EmailSiphon
Disallow: /

User-agent: Mozilla/2.0 (compatible; NEWT ActiveX; Win32)
Disallow: /


# let google's ad crawler do its thing
User-agent: Mediapartners-Google*
Disallow: 
Disallow: /rate		# dynamic
Disallow: /cgi-bin	# dynamic

# by default

User-agent: *
Disallow: /pliki		# not useful to spiders
Disallow: /trojany		# not useful to spiders
Disallow: /reklama		# not useful to spiders
Disallow: /cgi-bin		# dynamic