Add robots.txt (#1382)

* Add robots.txt in views

Signed-off-by: Tushar Goel <[email protected]>

* Add changelog and bump version

Signed-off-by: Tushar Goel <[email protected]>

* Add migration for version update

Signed-off-by: Tushar Goel <[email protected]>

* Fix tests

Signed-off-by: Tushar Goel <[email protected]>

* Update robots.txt according to latest changes

Signed-off-by: Tushar Goel <[email protected]>

---------

Signed-off-by: Tushar Goel <[email protected]>

CHANGELOG.rst

@@ -7,7 +7,9 @@ Version v34.0.0rc1
 
 - We updated package-url models, WARNING: in next major version of 
  vulnerablecode i.e v35.0.0 qualifiers will be of type ``string`` and not ``dict``.
+- We added changelog and dates on packages and vulnerabilities.
 - We fixed table borders in Vulnerability details UI #1356 (#1358)
+- We added robots.txt in views.
 - We fixed import runner's process_inferences (#1360)
 - We fixed debian OVAL importer (#1361)
 - We added graph model diagrams #977(#1350)

vulnerabilities/migrations/0053_alter_packagechangelog_software_version_and_more.py

@@ -0,0 +1,31 @@
+# Generated by Django 4.1.13 on 2024-01-03 13:50
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("vulnerabilities", "0052_remove_duped_purls_with_namespace_name_mixed"),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name="packagechangelog",
+ name="software_version",
+ field=models.CharField(
+ default="34.0.0rc1",
+ help_text="Version of the software at the time of change",
+ max_length=100,
+ ),
+ ),
+ migrations.AlterField(
+ model_name="vulnerabilitychangelog",
+ name="software_version",
+ field=models.CharField(
+ default="34.0.0rc1",
+ help_text="Version of the software at the time of change",
+ max_length=100,
+ ),
+ ),
+ ]

vulnerabilities/templates/robots.txt

@@ -0,0 +1,2 @@
+User-agent: *
+Disallow: /

vulnerabilities/tests/test_view.py

@@ -168,3 +168,11 @@ def test_vulnerabilties_search_view_with_empty(self):
  def test_vulnerabilties_search_view_can_find_alias(self):
  response = self.client.get(f"/vulnerabilities/search?search=TEST-2022")
  self.assertEqual(response.status_code, 200)
+
+
+class CheckRobotsTxtTestCase(TestCase):
+ def test_robots_txt(self):
+ response = self.client.get("/robots.txt")
+ assert response.status_code == 200
+ response = self.client.post("/robots.txt")
+ assert response.status_code == 405

vulnerablecode/__init__.py

@@ -12,7 +12,7 @@
 import warnings
 from pathlib import Path
 
-__version__ = "33.6.5"
+__version__ = "34.0.0rc1"
 
 
 def command_line():

vulnerablecode/urls.py

@@ -44,6 +44,10 @@ def __init__(self, *args, **kwargs):
 api_router.register("aliases", AliasViewSet, basename="alias")
 
 urlpatterns = [
+ path(
+ "robots.txt",
+ TemplateView.as_view(template_name="robots.txt", content_type="text/plain"),
+ ),
  path(
  "",
  HomePage.as_view(),