# # robots.txt # # This file is to prevent the crawling and indexing of certain parts # of your site by web crawlers and spiders run by sites like Yahoo! # and Google. By telling these "robots" where not to go on your site, # you save bandwidth and server resources. # # This file will be ignored unless it is at the root of your host: # Used: http://example.com/robots.txt # Ignored: http://example.com/site/robots.txt # # For more information about the robots.txt standard, see: # http://www.robotstxt.org/robotstxt.html User-agent: * # Crawl Delay added to slow down some well behaving bots. Crawl-delay: 10 # CSS, JS, Images Allow: /core/*.css$ Allow: /core/*.css? Allow: /core/*.js$ Allow: /core/*.js? Allow: /core/*.gif Allow: /core/*.jpg Allow: /core/*.jpeg Allow: /core/*.png Allow: /core/*.svg Allow: /profiles/*.css$ Allow: /profiles/*.css? Allow: /profiles/*.js$ Allow: /profiles/*.js? Allow: /profiles/*.gif Allow: /profiles/*.jpg Allow: /profiles/*.jpeg Allow: /profiles/*.png Allow: /profiles/*.svg # Directories Disallow: /core/ Disallow: /profiles/ # Files # Although deprecated, leaving Noindex for non-html files. This can/should be moved to nginx config. Noindex: /README.txt Noindex: /web.config Disallow: /README.txt Disallow: /web.config # Paths (clean URLs) Disallow: /admin/ Disallow: /comment/reply/ Disallow: /filter/tips/ Disallow: /node/add/ Disallow: /search/ Disallow: /user/register/ Disallow: /user/password/ Disallow: /user/login/ Disallow: /user/logout/ # Paths (no clean URLs) Disallow: /index.php/admin/ Disallow: /index.php/comment/reply/ Disallow: /index.php/filter/tips/ Disallow: /index.php/node/add/ Disallow: /index.php/search/ Disallow: /index.php/user/password/ Disallow: /index.php/user/register/ Disallow: /index.php/user/login/ Disallow: /index.php/user/logout/ ################################################################################ # Core Disallow: /book/export/ # MSK: Content-types Disallow: /block/ # Disallow: /podcasts/ # Disallow: /slideshows/ # Disallow: /teaser/ Disallow: /term/ # Disallow: /media-coverage/ # MSK: Modules and entities # Disallow: /dialog Disallow: /msk_markup/ # Disallow: /taxonomy/ Disallow: /templates # Disallow: /user/ # MSK tools # Disallow: /print # Disallow: /pdf # Disallow: /mail # Disallow: */print/ # Disallow: */pdf/ # MSK: Redirects Disallow: */redirect$ Disallow: */redirect-01$ Disallow: */redirect-02$ Disallow: */redirect-03$ Disallow: */redirect-04$ Disallow: */redirect-05$ Disallow: */redirect-06$ Disallow: */redirect-07$ Disallow: */redirect-08$ Disallow: */redirect-09$ Disallow: */redirect-10$ ################################################################################ # MSK: Old website Disallow: /mskcc/print/ # MSK: Misc. Disallow: /calgbtraining # Disallow: /cancer-care/patient-education/herbs/ Disallow: /drupal-guide Disallow: /msk-prepare Disallow: /experience/become-patient/first-appointment/begin-registration-msk-your-first-visit Disallow: /archived Disallow: /news/available-download Disallow: /appointments/request-appointment-01 Disallow: /ario2016/presentations Disallow: /monmouthopening Disallow: /hcp-education-training/medical-students/elective/apply/canadian-medical-students/students-international-medical-schools Disallow: /research-advantage/core-facilities/comparative-medicine-pathology-0/new-animal-user-onboarding Disallow: /videos/msk-kimmel Disallow: /cancer-care/diagnosis-treatment/symptom-management/integrative-medicine/meditation-101 Disallow: /departments/surgery/endometrial-cancer-research-day Disallow: /community-affairs/programs-services/high-school-outreach/registration-major-trends Disallow: /research-programs/cell-engineering/eleventh-annual-center-cell-engineering-retreat Disallow: /experience/become-patient/remotesecondopinion Disallow: /sites/default/files/node/168388/documents/sample-remote-2nd-opinion-report-2.pdf Disallow: /sites/default/files/node/168388/documents/sample-remote-2nd-opinion-report-2_0.pdf Disallow: /videos/msk-impact-informational-video Disallow: /hcp-education-training/cne/msk-2-day-cne-discounted-fee Disallow: /hcp-education-training/cne/msk-1-day-cne-discounted-fee Disallow: /departments/division-subspecialty-medicine/integrative-medicine/fundamentals-oncology-acupuncture-online-course Disallow: /lw Disallow: /profile/kathleen-foley Disallow: /videos/anusha Disallow: /videos/how-can-msk-direct-care-advisor-help-me-and-my-family Disallow: /videos/msk-direct-getting-right-diagnosis Disallow: /elc Disallow: /lung-screening/not-eligible Disallow: /lung-screening/eligible Disallow: /videos/innovations-breast-screening Disallow: /direct/* Disallow: /profile/jerome-posner Disallow: /research-programs/stem-cell-biology/2020-gforce-meeting Disallow: /patient-screening-form Disallow: /remote-access-resources-msk-employees Disallow: /research-programs/cell-engineering/special-events/twelfth-annual-center-cell-engineering-retreat Disallow: /hics-medical-staff-redeployment-registration-form Disallow: /mayor-de-blasio-press-conference Disallow: /cancer-care/diagnosis-treatment/symptom-management/integrative-medicine/therapies/how-we-can-help-you-stay-calm-and-connected-through-covid-19 Disallow: /cancer-care/diagnosis-treatment/symptom-management/integrative-medicine/therapies/how-we-can-help-you-stay-calm-and-connected-through-covid-19/mind-body-and-spirit-classes Disallow: /msk-strategic-partnerships-resources-and-information-hub Disallow: /videos/msk-cancer-alliance-final Disallow: /videos/msk-cancer-alliance-quality-lecture-series-skin-toxicities-patients-receiving-targeted-therapy Disallow: /videos/msk-survivorship-lecture-series-food-choice-during-and-after Disallow: /videos/msk-survivorship-lecture-series-cancer-immunotherapy-101-overview-non-oncologist Disallow: /videos/how-treat-cns-lymphoma-mskcc-bmt-lecture-series Disallow: /videos/quality-and-outcomes-lecture-series-radiation-related-mucositis Disallow: /departments/radiology/radiology-image-request Disallow: /msk-why Disallow: /drupal-training-registration-form Disallow: /index.php/* Disallow: /msk-news/fall-2020/nursing-excellence-honored-robbins-family-awards Disallow: /msk-news/spring-2020/infectious-disease-specialist-mini-kamboj-discusses-covid-19 Disallow: /msk-news/spring-2020/msk-enters-new-era-opening-david-koch-center-cancer-care-msk-cancer Disallow: /resource-sheet Disallow: /videos/multidisciplinary-management-early-stage-triple-negative-and-her-2-breast-cancers-neoadjuvant-therapy Disallow: /videos/personalized-treatment-soft-tissue-sarcoma-primary-management Disallow: /event/optimal-living-program-exercise-and-breast Disallow: /optimal-living-program-exercise-and-breast-cancer-series Disallow: /cancer-care/diagnosis-treatment/symptom-management/integrative-medicine/membership/renewal-form Disallow: /experience/become-patient/becoming-patient Disallow: /videos/united-face-crisis-msk-s-response-covid-19 Disallow: /lung-cancer-awareness-month-day-1-events-1-00-pm-5-00-pm Disallow: /lung-cancer-awareness-month-day-2-events-10-00-am-2-00-pm Disallow: /videos/personalized-treatment-recurrent-and-metastatic-soft-tissue-sarcoma Disallow: /nursing-request-redeployment-staff Disallow: /nursing-offer-redeploy-staff Disallow: /ert-c* Disallow: /research-advantage/marie-josee-kravis-women-science-endeavor-wise/kravis-wise-symposium Disallow: /coronavirus/update-msk-community-covid-19-vaccines Disallow: /coronavirus/una-actualizacion-para-nuestra-comunidad-msk-sobre-las-vacunas-contra-la-covid-19 Disallow: /cancer-care/diagnosis-treatment/symptom-management/integrative-medicine/membership/integrative-medicine-home-member-resources Disallow: /videos/morning-meditation Disallow: /videos/chair-yoga Disallow: /videos/beginner-mat-yoga Disallow: /videos/30-minute-fitness-everyone Disallow: /videos/60-minute-fitness-everyone Disallow: /videos/barre-basics Disallow: /videos/barre-express Disallow: /videos/dance-cardio Disallow: /videos/core-strength Disallow: /videos/fitness-everyone Disallow: /videos/guided-imagery-meditation-mountain-walk Disallow: /videos/guided-imagery-meditation-floating-ocean Disallow: /videos/tai-chi Disallow: /videos/intermediate-mat-yoga Disallow: /cancer-care/diagnosis-treatment/symptom-management/integrative-medicine/membership/integrative-medicine-home-member-resources/videos Disallow: /employee-town-hall-question Disallow: /hcp-education-training/cne/aphon-chemotherapy-biotherapy-course Disallow: /msk-search-feedback Disallow: /join-us-gsk-cocktail-reception Disallow: /sites/default/files/node/2739/documents/2021-dos-research-booklet.pdf Disallow: /mskdirect/msk-kids-testimonial Disallow: /videos/september-2022-surgical-oncology-talk Disallow: /surgical-oncology-teaching-conference Disallow: /videos/causal-inference-methods-observational-research Sitemap: https://www.mskcc.org/sitemap.xml # MSK: News Sitemap Sitemap: https://www.mskcc.org/news.xml User-agent: Twitterbot Allow: /news* Allow: /news-releases* Allow: /cancer-care/types* Allow: /cancer-care/patient-education/* Allow: /cancer-care/integrative-medicine/herbs*