# See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file # # To ban all spiders from the entire site uncomment the next two lines: User-agent: * # Directories # Paths (clean URLs) Disallow: /illustrations/ Disallow: /users/ Disallow: /v0/editor/ Disallow: /stories/*/versions #this will remove all tag results Disallow: /search?tags= Disallow: /?search/ Disallow: /?tags/ Disallow: /search?query=chalo Disallow: /stories/*/new_flag_story Disallow: /*?*utm_source= SITEMAP: https://storyweaver.org.in/sitemap.xml SITEMAP: https://storyweaver.org.in/sitemap_en.xml SITEMAP: https://storyweaver.org.in/sitemap_hi.xml SITEMAP: https://storyweaver.org.in/sitemap_mr.xml SITEMAP: https://storyweaver.org.in/sitemap_fa.xml SITEMAP: https://storyweaver.org.in/sitemap_fr.xml SITEMAP: https://storyweaver.org.in/sitemap_es.xml SITEMAP: https://storyweaver.org.in/sitemap_ar.xml SITEMAP: https://storyweaver.org.in/sitemap_ur.xml SITEMAP: https://storyweaver.org.in/sitemap_or.xml SITEMAP: https://storyweaver.org.in/sitemap_kok.xml