# Applies to all agents
User-agent: *

# Don't bother with dynamically-generated non-HTML pages
Disallow: /include/textrender.php
Disallow: /include/email_reveal.php
Disallow: /rss/

# It's crucial for us to block the voting pages, or else we would have millions
# of links pointing to it that would then redirect to the login page
Disallow: /question_vote.php
Disallow: /comment_vote.php

# We also block the AJAX versions, although links to those should never be
# found in the first place
Disallow: /question_vote_ajax.php
Disallow: /comment_vote_ajax.php

# Block asking questions; again this saves us from millions of links
# just redirecting to a login.
Disallow: /question_start.php

# Also, we do not want to crawl our user pages. Those lead to processor-
# intensive pages of listing questions per user vote, etc...
Disallow: /user.php

# Avoid crawling the question-list pages: we only want to access
# questions and answers on a per-celebrity basis, so we disallow all
# other combinations. These links must match the form given on each
# celebrity page, exactly. Otherwise, site processing would just go on
# endlessly... If this doesn't work, then we'll have to re-implement our
# sitemaps.
Disallow: /questions.php
Allow: /questions.php?what=questions&sort=best&when=all&type=celeb&id=
Allow: /questions.php?what=answers&sort=recent&when=all&type=celeb&id=