deps load fix

- known error to make created_at and updated_at default to now

src/pipeline/models/__init__.py

@@ -149,15 +149,16 @@ class DependsOn(Base):
     # ideally, these are non-nullable but diff package managers are picky about this
     dependency_type = Column(String, nullable=True)
     semver_range = Column(String, nullable=True)
+    # TODO: make these default now now
     created_at = Column(DateTime, nullable=False)
     updated_at = Column(DateTime, nullable=False)
 
     def to_dict(self):
         return {
             "version_id": self.version_id,
             "dependency_id": self.dependency_id,
-            "dependency_type": self.dependency_type,
             "semver_range": self.semver_range,
+            # "dependency_type": self.dependency_type,
         }
 
 

src/pipeline/utils/crates/transformer.py

@@ -71,9 +71,9 @@ def dependencies(self):
             reader = csv.DictReader(f)
             for row in reader:
                 start_id = row["version_id"]
-                end_id = row["dependency_id"]
+                end_id = row["crate_id"]
                 req = row["req"]
-                kind = row["kind"]
+                kind = int(row["kind"])
 
                 # map string to enum
                 dependency_type = DependencyType(kind)

src/pipeline/utils/pg.py

@@ -1,7 +1,7 @@
 import os
 from typing import Iterable, List, Type
 
-from sqlalchemy import UUID, create_engine
+from sqlalchemy import UUID, create_engine, func
 from sqlalchemy.dialects import postgresql
 from sqlalchemy.dialects.postgresql import insert
 from sqlalchemy.orm import sessionmaker
@@ -156,14 +156,19 @@ def dependency_object_generator():
                     continue
                 dependency_id = dependency.id
 
+                now = func.now()
+
                 yield DependsOn(
                     version_id=version_id,
                     dependency_id=dependency_id,
                     semver_range=semver_range,
+                    # TODO: db should do this
+                    created_at=now,
+                    updated_at=now,
                     # dependency_type=dependency_type, TODO: add this
                 )
 
-        self._batch(dependency_object_generator(), DependsOn, 10000)
+        self._batch(dependency_object_generator(), DependsOn, 100000)
 
     def insert_load_history(self, package_manager_id: str):
         with self.session() as session: