Sitemap: https://www.taylorssydney.edu.au/-/sitemap.xml


#User-agent: rogerbot 
#Disallow: /

#User-agent: AuditMyPC Webmaster Tool
#Disallow: /

User-agent: ShopWiki/1.0 ( +http://www.shopwiki.com/wiki/Help:Bot)
Disallow: /


User-agent: Mozilla/5.0 (compatible; ElectricMonk/3.1.1 +https://www.duedil.com/our-crawler/)
Disallow: /

User-agent: Math. Comp. PA Crawler
Disallow: /

User-agent: Mozilla/5.0 (compatible; IstellaBot/1.10.2 +http://www.tiscali.it/)
Disallow: /

# Strath GSA
User-agent: webteam-chat@lists.strath.ac.uk
Disallow: /

# Mechanize
User-agent: Mechanize/2.7.1 Ruby/2.0.0p247 (http://github.com/sparklemotion/mechanize/)
Disallow: /

# SISTRIX
User-agent: SISTRIX Crawler
Disallow: /

User-agent: *
Disallow: /~/
Allow: /~/icon/
Disallow: /layouts/System/
Disallow: /*.axd$
Disallow: /landingpages/
Disallow:  /lp/
Disallow: /sitecore/
Disallow: /index.php
Disallow: /http:/
Disallow: /https:/
Disallow: /www.youtube.com/
Disallow: /*.axd$
Disallow: regexpIgnoreCase:.*/accordians/electives
Disallow: regexpIgnoreCase:.*/global-locations/programs
Disallow: /Student-profile?id=*
Disallow: /blog?tag=*
Disallow: /blog?count=*
Disallow: /news-and-events/news?year=*
Disallow: alumni-party-registration/thanks-for-completing
Disallow: /success.bellerbys.com/
Disallow: /staging/