robots.txt

• google'in kendi robots.txt'si ise söyledir:
  
  user-agent: *
  allow: /searchhistory/
  disallow: /search
  disallow: /groups
  disallow: /images
  disallow: /catalogs
  disallow: /catalog_list
  disallow: /news
  disallow: /nwshp
  disallow: /?
  disallow: /addurl/image?
  disallow: /pagead/
  disallow: /relpage/
  disallow: /sorry/
  disallow: /imgres
  disallow: /keyword/
  disallow: /u/
  disallow: /univ/
  disallow: /cobrand
  disallow: /custom
  disallow: /advanced_group_search
  disallow: /advanced_search
  disallow: /googlesite
  disallow: /preferences
  disallow: /setprefs
  disallow: /swr
  disallow: /url
  disallow: /wml
  disallow: /hws
  disallow: /bsd?
  disallow: /linux?
  disallow: /mac?
  disallow: /microsoft?
  disallow: /unclesam?
  disallow: /answers/search?q=
  disallow: /local?
  disallow: /local_url
  disallow: /froogle?
  disallow: /froogle_
  disallow: /print?
  disallow: /scholar?
  disallow: /palm
  disallow: /complete
  disallow: /sponsoredlinks
  disallow: /videosearch?
  disallow: /videopreview?
  disallow: /videoprograminfo?
  disallow: /maps?
  disallow: /translate?
  disallow: /ie?
  
  http://www.google.com/robots.txt