feat(entity): import services and libraries into entity model

test/globalsearch/test_indexer.py

@@ -11,9 +11,7 @@
 
 def _initialize(client, models):
     for model, index in models:
-        client.create_index(
-            uid=index, options={"name": model.__name__, "primaryKey": "id"}
-        )
+        client.create_index(uid=index, options={"primaryKey": "id"})
 
 
 def test_index_model():

zoo/api/query.py

@@ -174,7 +174,7 @@ def resolve_all_dependency_usages(self, info, **kwargs):
             page_info=page_info, edges=edges, total_count=total
         )
 
-    def resolve_all_entites(self, info, **kwargs):
+    def resolve_all_entities(self, info, **kwargs):
         paginator = Paginator(**kwargs)
         total = Entity.objects.all().count()
         page_info = paginator.get_page_info(total)

zoo/api/types.py

@@ -151,7 +151,7 @@ class Meta:
 
 class Link(graphene.ObjectType):
     name = graphene.String()
-    url = graphene.List(graphene.String)
+    url = graphene.String()
     icon = graphene.String()
     entity = graphene.Field(lambda: Entity)
 
@@ -253,8 +253,8 @@ def from_db(cls, entity):
     @classmethod
     def get_node(cls, info, entity_id):
         try:
-            environment = services_models.Entity.objects.get(id=entity_id)
-            return cls.from_db(environment)
+            entity = entities_models.Entity.objects.get(id=entity_id)
+            return cls.from_db(entity)
         except ObjectDoesNotExist:
             return None
 

zoo/entities/management/commands/import_entities.py

@@ -0,0 +1,110 @@
+import structlog
+from django.core.management.base import BaseCommand
+
+from zoo.entities.enums import Kind
+from zoo.entities.models import Entity, Link
+from zoo.libraries.models import Library
+from zoo.services.models import Service
+
+log = structlog.get_logger()
+
+
+class Command(BaseCommand):
+    help = "Create entities from services and libraries"
+
+    def handle(self, *args, **options):
+        self.service_to_entity()
+        self.library_to_entity()
+
+    @staticmethod
+    def service_to_entity():
+        with open("service_exceptions.csv", "w+") as exceptions:
+            exceptions.write(
+                "Service name;Service owner;Service repository;Exception\n"
+            )
+            for service in Service.objects.all():
+                if service.repository:
+                    log.info("entity.import_services.processing", service=service)
+                    try:
+                        entity = Entity.objects.create(
+                            name=service.name,
+                            owner=service.owner,
+                            description=service.description,
+                            kind=Kind.COMPONENT,
+                            type="service",
+                            source=service.repository,
+                            tags=service.tags,
+                            service=service,
+                        )
+
+                        if service.docs_url:
+                            Link.objects.create(
+                                name="Documentation",
+                                url=service.docs_url,
+                                entity=entity,
+                            )
+
+                        if service.slack_channel:
+                            Link.objects.create(
+                                name="Discussion",
+                                url=f"https://app.slack.com/client/T024Z3H2Y/{service.slack_channel}",
+                                entity=entity,
+                            )
+                    except Exception as err:
+                        exceptions.write(
+                            f"{getattr(service, 'name', '')};{getattr(service, 'owner', '')};{getattr(service, 'repository', '')};{repr((err))}\n"
+                        )
+                else:
+                    exceptions.write(
+                        f"{getattr(service, 'name', '')};{getattr(service, 'owner', '')};;Missing repository for service\n"
+                    )
+
+    @staticmethod
+    def library_to_entity():
+        with open("library_exceptions.csv", "w+") as exceptions:
+            exceptions.write(
+                "Library name;Library owner;Library repository;Exception\n"
+            )
+            for library in Library.objects.all():
+                if library.repository:
+                    log.info("entity.import_libraries.processing", library=library)
+                    try:
+                        entity = Entity.objects.create(
+                            name=library.name,
+                            owner=library.owner,
+                            description=library.description,
+                            kind=Kind.COMPONENT,
+                            type="library",
+                            source=library.repository,
+                            tags=library.tags,
+                            library=library,
+                        )
+
+                        if library.docs_url:
+                            Link.objects.create(
+                                name="Documentation",
+                                url=library.docs_url,
+                                entity=entity,
+                            )
+
+                        if library.slack_channel:
+                            Link.objects.create(
+                                name="Discussion",
+                                url=f"https://app.slack.com/client/T024Z3H2Y/{library.slack_channel}",
+                                entity=entity,
+                            )
+
+                        if library.library_url:
+                            Link.objects.create(
+                                name="Library URL",
+                                url=library.library_url,
+                                entity=entity,
+                            )
+                    except Exception as err:
+                        exceptions.write(
+                            f"{getattr(library, 'name', '')};{getattr(library, 'owner', '')};{getattr(library, 'repository', '')};{repr((err))}\n"
+                        )
+                else:
+                    exceptions.write(
+                        f"{getattr(library, 'name', '')};{getattr(library, 'owner', '')};;Missing repository for library\n"
+                    )

zoo/entities/migrations/0003_allow_blank_on_group.py

@@ -0,0 +1,24 @@
+# Generated by Django 2.2.19 on 2021-10-13 09:52
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("entities", "0002_allow_null_product_owner_on_group"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="entity",
+            name="group",
+            field=models.OneToOneField(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.CASCADE,
+                to="entities.Group",
+            ),
+        ),
+    ]

zoo/entities/models.py

@@ -57,7 +57,8 @@ class Meta:
     group = models.OneToOneField(
         "entities.Group",
         on_delete=models.CASCADE,
-        default=None,
+        null=True,
+        blank=True,
     )
     source = models.ForeignKey(
         "repos.Repository",

zoo/repos/entities_yaml.py

@@ -56,7 +56,7 @@ def parse(_yaml: str) -> Union[List, None]:
 
 def generate_component_base(component):
     component_base_document = {
-        "apiVersion": "v1alpha1",
+        "apiVersion": "v1alpha1",  # TODO: make it configurable?
         "kind": component.kind,
         "metadata": {
             "name": component.name,

zoo/repos/tasks.py

@@ -7,6 +7,7 @@
 import structlog
 from celery import shared_task
 from django.conf import settings
+from django.db import transaction
 
 from ..analytics.tasks import repo_analyzers
 from ..auditing import runner
@@ -200,11 +201,12 @@ def update_or_create_components(data: List, proj: Dict) -> None:
         return
 
     def _do_cleanup():
-        affected_entities = Entity.objects.filter(source=repository)
-        Link.objects.filter(entity__in=affected_entities).delete()
-        Service.objects.filter(repository=repository).delete()
-        Library.objects.filter(repository=repository).delete()
-        Entity.objects.filter(source=repository).delete()
+        with transaction.atomic():
+            affected_entities = Entity.objects.filter(source=repository)
+            Link.objects.filter(entity__in=affected_entities).delete()
+            Service.objects.filter(repository=repository).delete()
+            Library.objects.filter(repository=repository).delete()
+            Entity.objects.filter(source=repository).delete()
 
     _do_cleanup()
     entity_builder = EntityBuilder()