#
# robots.txt
#
# This file is to prevent the crawling and indexing of certain parts
# of your site by web crawlers and spiders run by sites like Yahoo!
# and Google. By telling these "robots" where not to go on your site,
# you save bandwidth and server resources.
#
# This file will be ignored unless it is at the root of your host:
# Used:    http://example.com/robots.txt
# Ignored: http://example.com/site/robots.txt
#
# For more information about the robots.txt standard, see:
# http://www.robotstxt.org/robotstxt.html

User-agent: *
# Crawl Delay added to slow down some well behaving bots.
Crawl-delay: 10
# CSS, JS, Images
Allow: /core/*.css$
Allow: /core/*.css?
Allow: /core/*.js$
Allow: /core/*.js?
Allow: /core/*.gif
Allow: /core/*.jpg
Allow: /core/*.jpeg
Allow: /core/*.png
Allow: /core/*.svg
Allow: /profiles/*.css$
Allow: /profiles/*.css?
Allow: /profiles/*.js$
Allow: /profiles/*.js?
Allow: /profiles/*.gif
Allow: /profiles/*.jpg
Allow: /profiles/*.jpeg
Allow: /profiles/*.png
Allow: /profiles/*.svg

# Directories
Disallow: /core/
Disallow: /profiles/

# Files
# Although deprecated, leaving Noindex for non-html files. This can/should be moved to nginx config.
Noindex: /README.txt
Noindex: /web.config

Disallow: /README.txt
Disallow: /web.config

# Paths (clean URLs)
Disallow: /admin/
Disallow: /comment/reply/
Disallow: /filter/tips/
Disallow: /node/add/
Disallow: /search/
Disallow: /user/register/
Disallow: /user/password/
Disallow: /user/login/
Disallow: /user/logout/

# Paths (no clean URLs)
Disallow: /index.php/admin/
Disallow: /index.php/comment/reply/
Disallow: /index.php/filter/tips/
Disallow: /index.php/node/add/
Disallow: /index.php/search/
Disallow: /index.php/user/password/
Disallow: /index.php/user/register/
Disallow: /index.php/user/login/
Disallow: /index.php/user/logout/

################################################################################

# Core
Disallow: /book/export/

# MSK: Content-types
Disallow: /block/
Disallow: /podcasts/
Disallow: /slideshows/
Disallow: /teaser/
Disallow: /term/
Disallow: /media-coverage/

# MSK: Modules and entities
Disallow: /dialog
Disallow: /msk_markup/
Disallow: /taxonomy/
Disallow: /templates
Disallow: /user/

# MSK tools
Disallow: /print
Disallow: /pdf
Disallow: /mail
Disallow: */print/
Disallow: */pdf/

# MSK: Redirects
Disallow: */redirect$
Disallow: */redirect-01$
Disallow: */redirect-02$
Disallow: */redirect-03$
Disallow: */redirect-04$
Disallow: */redirect-05$
Disallow: */redirect-06$
Disallow: */redirect-07$
Disallow: */redirect-08$
Disallow: */redirect-09$
Disallow: */redirect-10$

################################################################################

# MSK: Old website
Noindex: /mskcc/print/
Disallow: /mskcc/print/

# Drupal search
Noindex: /search/node

# MSK: Misc.
Noindex: /calgbtraining
Noindex: /cancer-care/patient-education/herbs/
Noindex: /drupal-guide
Noindex: /msk-prepare
Noindex: /experience/become-patient/first-appointment/begin-registration-msk-your-first-visit
Noindex: /josie-robertson-surgery-opening
Noindex: /josierobertson-openhouse
Noindex: /pressroom/available-download
Noindex: /register-josie-robertson-surgery-openhouse
Noindex: /register-josie-robertson-surgery-opening
Noindex: /celebration-leadership-and-festschrift
Noindex: /register-celebration-leadership-and-festschrift
Noindex: /celebration-leadership-and-festschrift-01
Noindex: /register-festschrift-celebration-leadership
Noindex: /appointments/request-appointment-01
Noindex: /ario2016/presentations
Noindex: /monmouthopening
Noindex: /register-msk-monmouth-opening
Noindex: /monmouthrsvp
Noindex: /hcp-education-training/medical-students/elective/apply/students-international-medical-schools
Noindex: /research-advantage/core-facilities/comparative-medicine-pathology-0/new-animal-user-onboarding
Noindex: /videos/msk-kimmel
Noindex: /cancer-care/diagnosis-treatment/symptom-management/integrative-medicine/meditation-101
Noindex: /msk-center-laboratory-medicine-opening-vip
Noindex: /msk-center-laboratory-medicine-opening
Noindex: /celebration-progress-msk-nassau-external
Noindex: /celebration-progress-msk-nassau-internal
Noindex: /departments/surgery/endometrial-cancer-research-day

Disallow: /calgbtraining
Disallow: /cancer-care/patient-education/herbs/
Disallow: /drupal-guide
Disallow: /msk-prepare
Disallow: /experience/become-patient/first-appointment/begin-registration-msk-your-first-visit
Disallow: /josie-robertson-surgery-opening
Disallow: /josierobertson-openhouse
Disallow: /pressroom/available-download
Disallow: /register-josie-robertson-surgery-openhouse
Disallow: /register-josie-robertson-surgery-opening
Disallow: /celebration-leadership-and-festschrift
Disallow: /register-celebration-leadership-and-festschrift
Disallow: /celebration-leadership-and-festschrift-01
Disallow: /register-festschrift-celebration-leadership
Disallow: /appointments/request-appointment-01
Disallow: /ario2016/presentations
Disallow: /monmouthopening
Disallow: /register-msk-monmouth-opening
Disallow: /monmouthrsvp
Disallow: /hcp-education-training/medical-students/elective/apply/students-international-medical-schools
Disallow: /research-advantage/core-facilities/comparative-medicine-pathology-0/new-animal-user-onboarding
Disallow: /videos/msk-kimmel
Disallow: /cancer-care/diagnosis-treatment/symptom-management/integrative-medicine/meditation-101
Disallow: /msk-center-laboratory-medicine-opening-vip
Disallow: /msk-center-laboratory-medicine-opening
Disallow: /celebration-progress-msk-nassau-external
Disallow: /celebration-progress-msk-nassau-internal
Disallow: /departments/surgery/endometrial-cancer-research-day

Sitemap: https://www.mskcc.org/sitemap.xml
# MSK: News Sitemap
Sitemap: https://www.mskcc.org/news.xml

User-agent: Twitterbot
Allow: /blog*
Allow: /press-releases*
Allow: /cancer-care/types*
Allow: /cancer-care/patient-education/*
Allow: /cancer-care/integrative-medicine/herbs*