# This file controls access by robots that follow "standard robot # exclusion rules". For more information see: # http://info.webcrawler.com/mak/projects/robots/robots.html # # All robots should follow the rules herein: User-agent: * # Allow robots into /ABLE but not paths starting with: Disallow: /ABLE/NDVI Disallow: /ABLE/OSUprecip Disallow: /ABLE/archiveold Disallow: /ABLE/data_archive Disallow: /ABLE/gif Disallow: /ABLE/gif_archive Disallow: /ABLE/httpd_docs Disallow: /ABLE/qresponse Disallow: /ABLE/test? Disallow: /ABLE/vmestatus # # Allow robots into /ACP but not paths starting with: Disallow: /ACP/gif Disallow: /ACP/neopix # # Allow robots into /CES but not paths below: Disallow: /CES/* # # Allow robots into /ASP but not paths starting with: Disallow: /ASP/Library Disallow: /ASP/Templates Disallow: /ASP/development Disallow: /ASP/graphics Disallow: /ASP/holding Disallow: /ASP/original Disallow: /ASP/png # # Allow robots into /FERMI but not paths starting with: Disallow: /FERMI/Data Disallow: /FERMI/images # # Allow robots into /ANLMET but not paths starting with: Disallow: /ANLMET/gif Disallow: /ANLMET/gif_archive Disallow: /ANLMET/numeric Disallow: /ANLMET/text Disallow: /ANLMET/PDFandTIF # # Allow robots into /GCEP but not paths starting with: Disallow: /GCEP/legacy Disallow: /GCEP/images Disallow: /GCEP/Archives # #The following subdirectories are entirely disallowed: Disallow: /gif Disallow: /maps Disallow: /ARM Disallow: /ER #The following are disallowed because they are on CD jukebox (slow) Disallow: /TEXAQS Disallow: /VTMX Disallow: /PHX2001 Disallow: /ASCOT