#---------------------------------------------------------------------- # robots.txt -- Robots Exclusion Configuration # # Do not allow robots/wanderers/spiders to index us # unless we specifically configure it. This file belongs in # the top-level of the web documents hierarchy (it must be # accessable as http://servername/robots.txt .) # # See here for more info about robots.txt: # # http://info.webcrawler.com/mak/projects/robots/norobots.html # #---------------------------------------------------------------------- # # allow Rick Kelly's online documentation # by gqwang Mon Aug 23 14:20:02 PDT 1999 # User-agent: htdig Disallow: /tmp # Allow JPL Knowledge Management search engine spiders. # This is the new JPL product independent agent name (Feb. 2002). # The product specific agent names will be phased out in 2002. # customer: (pjscott) # backup contact(s): (crhoades, bmcwatt, sleung) User-agent: jpl-km-spider Disallow: /tmp Disallow: / # # #---------------------------------------------------------------------- # Prevent all (other) robots from visiting the EIS webserver. User-agent: * Disallow: /