# cwjobs  - last edited 15 December 2017

User-agent: *

User-agent: Twitterbot
allow: /*?WT.mc_id=* # allow aggregated jobs

# 8/9/2017
Allow: /*?q*&page*

#  2/11/2016
Disallow: /*==$
Disallow: /*.ashx$

# 25/08/2015 RD added to block facets on unified results
Disallow: /*?page=1$  # don't index page=1
Disallow: /*&page=*  # this is where there is a page parameter on a facited link - its a catch all
Disallow: /*?WT.mc_id=* # stop aggregated jobs
Disallow: /*&WT.mc_id=* # stop aggregated jobs
Disallow: /*?similarjobsmodal=*
Disallow: /*&similarjobsmodal=*


#17/08/2016 Added by RD to block call backs
Disallow: /jsd/
Disallow: /insidejob/wp-admin/
Disallow: */apply$
Disallow: */apply?*

# 05/07/2016 Added by RD to block call backs - considered and decided to block!
Disallow: /Login/
Disallow: /savedjobs
Disallow: /personalisation

Disallow: /*?CompanyId=*
Disallow: /Accessible/
Disallow: /Authenticated/
Disallow: /cannedsearch/
Disallow: /CannedSearch/
Disallow: /CompanySearch/
Disallow: /JobSearch/JobBasket.aspx
Disallow: /JobSearch/JobContactDetails.aspx
Disallow: /JobSearch/JobPrinterFriendlyDetails.aspx
Disallow: /JobSearch/jobs-on-a-map.aspx
Disallow: /JobSearch/PromotedClick.aspx
Disallow: /JobSearch/RSS.aspx
Disallow: /JobSearch/SendToAFriend.aspx
Disallow: /JobSearch/UnAuthApplyOnline.aspx
Disallow: /JobSearch/PreExternalApplyOnline.aspx
Disallow: /JobSearch/JobsByEmailSetup.aspx
Disallow: /JobLink/

Disallow: /pgl/
Disallow: /pjb_ui/
Disallow: /WebServices/
Disallow: /Webservices/
Disallow: /Maintenance.aspx
Disallow: /service/nolayout.aspx


Disallow: /CompanyBrowse/Roger-Jones-Recruitment-Ltd_Vacancies_c336468.html

# 22/05/2015 added to block facets on results.aspx
Disallow: /*&Rate=*
Disallow: /*&RateType=*
Disallow: /*&ValidFromDay=*
Disallow: /*&CompanyType=*
#Disallow: /*&JobTitle=* 
Disallow: /*&Discipline=*
Disallow: /*&JobType1=*
Disallow: /*&JobType=*
Disallow: /*&radius=*  # this is where the radius is in a secondary faceted link
Disallow: /*&Radius=*  # this is where the radius is in a secondary faceted link
Disallow: /*&postedwithin=*  # this is where the postedwithin is in a secondary faceted link
Disallow: /*&Postedwithin=*  # this is where the postedwithin is in a secondary faceted link
Disallow: /*&sort=*  # this is where the sort is in a secondary faceted link
Disallow: /*&Sort=*  # this is where the sort is in a secondary faceted link
Disallow: /*&salary=*  # this is where the salary is in a secondary faceted link
Disallow: /*&Salary=*  # this is where the salary is in a secondary faceted link



# to be added 08/09/2015
Disallow: /*/jbe/*
Disallow: /JobsByEmail/*
Disallow: /JobSearch/AdvancedJobSearch.aspx
Disallow: /account/ # updated 09/03/17 RD
Disallow: /Account/ # updated 09/03/17 RD
Disallow: /AccountDetails/*
Disallow: /*/similarjobs/*
Disallow: /Jobsearch/ApplyViaDocuments.aspx
Disallow: /~/*
Disallow: /login
Disallow: /Help/*
Disallow: /SalaryChecker/SalaryCheckerResults.aspx
Disallow: /jobseekers/*_new.asp
Disallow: /tag/*
Disallow: /jobwidget/*

Disallow: /JobSearch/LocationSelect.aspx
Disallow: /JobSearch/EmailLink.aspx

# added 18/05/2016 to stop Google logging these
Disallow: /JobSearch/*/api/*
Disallow: /jobsearch/*/api/*

# added 18/05/2016 other old URLs to block RD
Disallow: /help/
Disallow: */articledetail.asp
Disallow: */archivedetails.asp
Disallow: /LocationHub/
Disallow: /jobs-at/jobs/in-
Disallow: /jobs-at/*/jobs/part-time/
Disallow: /jobs-at/*/jobs/permanent/
Disallow: /jobs-at/*/jobs/contract/
Disallow: /jobs-at/*/jobs/temporary/
Disallow: /mail_rd.asp
Disallow: /newwave_rd.asp
Disallow: /sitecore/*/www.
Disallow: /careers-advice/*/www.


User-Agent: OmniExplorer_Bot
Disallow: /

# removed 05 Sept 2014
# User-Agent: MJ12bot
# Disallow: /

User-Agent: Verticrawlbot
Disallow: /

User-agent: AddSearchBot
Disallow: /

User-agent: admantx
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: Baiduspider
User-agent: Baiduspider-video
User-agent: Baiduspider-image
Disallow: /

User-agent: ChangeDetection
Disallow: /

User-agent: dotbot
Disallow: /

User-agent: Exabot
Disallow: /

User-agent: GozaikBot
Disallow: /

User-agent: grapeshot
Disallow: /

User-Agent: Ichiro
Disallow: /

#User-Agent: LinkedInBot/1.0  removed 22/11/17 RD
#Disallow: /

User-agent: proximic
Disallow: /

User-agent: SeznamBot
Disallow: /

User-agent: sogou spider
Disallow: /

User-agent: trovitBot
Disallow: /

User-agent: WeSEE
Disallow: /

User-agent: WijuBot
Disallow: /

User-agent: XoviBot
Disallow: /

User-agent: YakazBot/1.0
Disallow: /


User-agent: Yandex
Disallow: /

# Pingdom added 24/09/2014 RD

User-agent: Pingdom.com
Disallow: /

User-agent: Pingdom.com_bot
Disallow: /

User-agent: pingdom.com_bot_version_1.4_(http://www.pingdom.com/)
Disallow: /

User-agent: Pingdom
Disallow: /

User-agent: Pingdom GIGRIB
Disallow: /

# end pingdom

# Mail.RU added 24/09/2014 RD
User-agent: Mail.RU_Bot/2.0
Disallow: /

User-agent: Mail.RU_Bot
Disallow: /

User-agent: mail.ru_bot/robots/2.0
Disallow: /

User-agent: Mail.RU
Disallow: /

User-agent: Mail.Ru
Disallow: /

User-agent: Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/2.0; +http://go.mail.ru/help/robots)
Disallow: /

# end Mail.RU

User-agent: Bing
Crawl-delay: 1
 
User-agent: Slurp
Crawl-delay: 1