User-agent: *
Allow: /
Disallow: /job/clk/*
Disallow: /pt/
Disallow: /redirect.php?
Disallow: /rss/
Disallow: /tracking.php
Disallow: /validation/
Disallow: /lookup.php
Disallow: /job-seekers/egress.php
Disallow: /challenge.php

User-agent: AdsBot-Google
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Googlebot
Allow: /

User-agent: Googlebot-Mobile
Allow: /

User-agent: Mediapartners-Google
Allow: /

User-agent: 008
Disallow: /

User-agent: adidxbot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: Baiduspider
Disallow: /

User-agent: Cliqzbot
Disallow: /

User-agent: dotbot
Disallow: /

User-agent: grapeshot
Disallow: /

User-agent: ia_archiver
Disallow: /

User-agent: JobdiggerSpider
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: mxbot/1.0
Disallow: /

User-agent: netseer/0.1
Disallow: /

User-agent: OmniExplorer_Bot
Disallow: /

User-agent: sistrix
Disallow: /

User-agent: spbot
Disallow: /

User-agent: Speedy
Disallow: /

User-agent: PiplBot
Disallow: /