Stop crawlers from indexing package content

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>

diff --git a/src/static/robots.txt b/src/static/robots.txt
index 6ffbc308f73e8f30d660a4ef89c4099d05fb4fbf..1f53798bb4fe33c86020be7f10c44f29486fd190 100644 (file)
--- a/src/static/robots.txt
+++ b/src/static/robots.txt
@@ -1,3 +1,2 @@
 User-agent: *
 Disallow: /
-

diff --git a/src/web/packages.py b/src/web/packages.py
index cc038f71a8e352bc2ee874778222b542c2deabb5..9f820b38f20c0414fd19c7019f03ae95a816e4f0 100644 (file)
--- a/src/web/packages.py
+++ b/src/web/packages.py
@@ -200,6 +200,9 @@ class PackageFileDownloadHandler(base.BaseHandler):
                self.set_header("Content-Disposition", "attachment; filename=%s" % os.path.basename(filename))
                self.set_header("Content-Type", mimetype)
 
+               # These pages should not be indexed
+               self.add_header("X-Robots-Tag", "noindex")
+
                # Transfer the content chunk by chunk.
                while True:
                        buf = f.read(BUFFER_SIZE)
@@ -222,6 +225,9 @@ class PackageFileViewHandler(PackageFileDownloadHandler):
                content = f.read()
                f.close()
 
+               # These pages should not be indexed
+               self.add_header("X-Robots-Tag", "noindex")
+
                self.render("packages/view-file.html", pkg=pkg, filename=filename,
                        mimetype=mimetype, content=content, filesize=f.size)