Allow compiling with a library

backend/coreapp/compiler_wrapper.py

@@ -6,7 +6,16 @@
 from platform import uname
 import time
 
-from typing import Any, Callable, Dict, Optional, Tuple, TYPE_CHECKING, TypeVar
+from typing import (
+    Any,
+    Callable,
+    Dict,
+    Optional,
+    Tuple,
+    TYPE_CHECKING,
+    TypeVar,
+    Sequence,
+)
 
 from django.conf import settings
 
@@ -17,6 +26,7 @@
 import coreapp.util as util
 
 from .error import AssemblyError, CompilationError
+from .libraries import LIBRARY_BASE_PATH, Library
 from .models.scratch import Asm, Assembly
 from .sandbox import Sandbox
 
@@ -130,6 +140,7 @@ def compile_code(
         code: str,
         context: str,
         function: str = "",
+        libraries: Sequence[Library] = (),
     ) -> CompilationResult:
         if compiler == compilers.DUMMY:
             return CompilationResult(f"compiled({context}\n{code}".encode("UTF-8"), "")
@@ -182,6 +193,12 @@ def compile_code(
             # Run compiler
             try:
                 st = round(time.time() * 1000)
+                libraries_compiler_flags = " ".join(
+                    (
+                        compiler.library_include_flag + str(lib.include_path)
+                        for lib in libraries
+                    )
+                )
                 compile_proc = sandbox.run_subprocess(
                     cc_cmd,
                     mounts=(
@@ -195,7 +212,8 @@ def compile_code(
                         "INPUT": sandbox.rewrite_path(code_path),
                         "OUTPUT": sandbox.rewrite_path(object_path),
                         "COMPILER_DIR": sandbox.rewrite_path(compiler.path),
-                        "COMPILER_FLAGS": sandbox.quote_options(compiler_flags),
+                        "COMPILER_FLAGS": sandbox.quote_options(compiler_flags)
+                        + libraries_compiler_flags,
                         "FUNCTION": function,
                         "MWCIncludes": "/tmp",
                         "TMPDIR": "/tmp",

backend/coreapp/compilers.py

@@ -53,6 +53,7 @@ class Compiler:
     cc: str
     platform: Platform
     flags: ClassVar[Flags]
+    library_include_flag: str
     base_compiler: Optional["Compiler"] = None
     is_gcc: ClassVar[bool] = False
     is_ido: ClassVar[bool] = False
@@ -96,6 +97,7 @@ def to_json(self) -> Dict[str, object]:
 @dataclass(frozen=True)
 class DummyCompiler(Compiler):
     flags: ClassVar[Flags] = []
+    library_include_flag: str = ""
 
     def available(self) -> bool:
         return settings.DUMMY_COMPILER
@@ -110,17 +112,20 @@ def available(self) -> bool:
 @dataclass(frozen=True)
 class ClangCompiler(Compiler):
     flags: ClassVar[Flags] = COMMON_CLANG_FLAGS
+    library_include_flag: str = "-isystem"
 
 
 @dataclass(frozen=True)
 class ArmccCompiler(Compiler):
     flags: ClassVar[Flags] = COMMON_ARMCC_FLAGS
+    library_include_flag: str = "-J"
 
 
 @dataclass(frozen=True)
 class GCCCompiler(Compiler):
     is_gcc: ClassVar[bool] = True
     flags: ClassVar[Flags] = COMMON_GCC_FLAGS
+    library_include_flag: str = "-isystem"
 
 
 @dataclass(frozen=True)
@@ -137,22 +142,26 @@ class GCCSaturnCompiler(GCCCompiler):
 class IDOCompiler(Compiler):
     is_ido: ClassVar[bool] = True
     flags: ClassVar[Flags] = COMMON_IDO_FLAGS
+    library_include_flag: str = "-I"
 
 
 @dataclass(frozen=True)
 class MWCCCompiler(Compiler):
     is_mwcc: ClassVar[bool] = True
     flags: ClassVar[Flags] = COMMON_MWCC_FLAGS
+    library_include_flag: str = "-IZ:"
 
 
 @dataclass(frozen=True)
 class MSVCCompiler(Compiler):
     flags: ClassVar[Flags] = COMMON_MSVC_FLAGS
+    library_include_flag: str = "/IZ:"
 
 
 @dataclass(frozen=True)
 class WatcomCompiler(Compiler):
     flags: ClassVar[Flags] = COMMON_WATCOM_FLAGS
+    library_include_flag: str = "/IZ:"
 
 
 def from_id(compiler_id: str) -> Compiler:

backend/coreapp/libraries.py

@@ -14,16 +14,29 @@ class Library:
 
     @property
     def path(self) -> Path:
-        return LIBRARY_BASE_PATH / name / version
+        return LIBRARY_BASE_PATH / self.name / self.version
+
+    @property
+    def include_path(self) -> Path:
+        return self.path / "include"
+
+
+@dataclass(frozen=True)
+class LibraryVersions:
+    name: str
+    supported_versions: list[str]
+
+    @property
+    def path(self) -> Path:
+        return LIBRARY_BASE_PATH / self.name
 
-    def available(self) -> bool:
-        return self.path.exists()
 
 @cache
-def available_libraries() -> list[Library]:
+def available_libraries() -> list[LibraryVersions]:
     results = []
 
     for lib_dir in LIBRARY_BASE_PATH.iterdir():
+        versions = []
         if not lib_dir.is_dir():
             continue
         for version_dir in lib_dir.iterdir():
@@ -32,8 +45,14 @@ def available_libraries() -> list[Library]:
             if not (version_dir / "include").exists():
                 continue
 
-            results.append(Library(
-                name=lib_dir.name,
-                version=version_dir.name,
-            ))
+            versions.append(version_dir.name)
+
+        if len(versions) > 0:
+            results.append(
+                LibraryVersions(
+                    name=lib_dir.name,
+                    supported_versions=versions,
+                )
+            )
+
     return results

backend/coreapp/migrations/0038_scratch_libraries.py

@@ -0,0 +1,17 @@
+# Generated by Django 4.2.3 on 2023-09-23 14:25
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("coreapp", "0037_rename_psyq43_to_psyq44"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="scratch",
+            name="libraries",
+            field=models.JSONField(default=list),
+        ),
+    ]

backend/coreapp/models/scratch.py

@@ -1,11 +1,13 @@
+import json
 import logging
 
 from django.db import models
 from django.utils.crypto import get_random_string
 
-from typing import List
+from typing import Any, List
 
 from .profile import Profile
+from ..libraries import Library
 
 logger = logging.getLogger(__name__)
 
@@ -43,6 +45,27 @@ class CompilerConfig(models.Model):
     diff_flags = models.JSONField(default=list)
 
 
+class LibrariesField(models.JSONField):
+    def __init__(self, **kwargs: Any):
+        class MyEncoder(json.JSONEncoder):
+            def default(self, obj: Any) -> str:
+                if isinstance(obj, Library):
+                    obj = {"name": obj.name, "version": obj.version}
+                return super().default(obj)
+
+        return super().__init__(encoder=MyEncoder, **kwargs)
+
+    def to_python(self, value: Any) -> list[Library]:
+        res = super().to_python(value)
+        return [Library(name=lib["name"], version=lib["version"]) for lib in res]
+
+    def from_db_value(self, *args: Any, **kwargs: Any) -> list[Library]:
+        # We ignore the type error here as this is a bug in the django stubs.
+        # CC: https://github.com/typeddjango/django-stubs/issues/934
+        res = super().from_db_value(*args, **kwargs)  # type: ignore
+        return [Library(name=lib["name"], version=lib["version"]) for lib in res]
+
+
 class Scratch(models.Model):
     slug = models.SlugField(primary_key=True, default=gen_scratch_id)
     name = models.CharField(max_length=1024, default="Untitled", blank=False)
@@ -67,6 +90,7 @@ class Scratch(models.Model):
     score = models.IntegerField(default=-1)
     max_score = models.IntegerField(default=-1)
     match_override = models.BooleanField(default=False)
+    libraries = LibrariesField(default=list)
     parent = models.ForeignKey("self", null=True, blank=True, on_delete=models.SET_NULL)
     owner = models.ForeignKey(Profile, null=True, blank=True, on_delete=models.SET_NULL)
     project_function = models.ForeignKey(

backend/coreapp/sandbox.py

@@ -68,6 +68,7 @@ def sandbox_command(self, mounts: List[Path], env: Dict[str, str]) -> List[str]:
             "--bindmount_ro", "/proc",
             "--bindmount", f"{self.path}:/var/tmp",
             "--bindmount_ro", str(settings.COMPILER_BASE_PATH),
+            "--bindmount_ro", str(settings.LIBRARY_BASE_PATH),
             "--env", "PATH=/usr/bin:/bin",
             "--cwd", "/tmp",
             "--rlimit_fsize", "soft",

backend/coreapp/serializers.py

@@ -18,6 +18,7 @@
 
 from .flags import LanguageFlagSet
 from . import compilers
+from .libraries import Library
 
 
 def serialize_profile(
@@ -135,6 +136,11 @@ class ScratchCreateSerializer(serializers.Serializer[None]):
     rom_address = serializers.IntegerField(required=False)
 
 
+class LibrarySerializer(serializers.Serializer[Library]):
+    name = serializers.CharField()
+    version = serializers.CharField()
+
+
 class ScratchSerializer(serializers.HyperlinkedModelSerializer):
     slug = serializers.SlugField(read_only=True)
     url = UrlField()
@@ -146,6 +152,7 @@ class ScratchSerializer(serializers.HyperlinkedModelSerializer):
     project = serializers.SerializerMethodField()
     project_function = serializers.SerializerMethodField()
     language = serializers.SerializerMethodField()
+    libraries = serializers.ListField(child=LibrarySerializer(), default=list)
 
     class Meta:
         model = Scratch
@@ -243,6 +250,7 @@ class Meta:
             "project_function",
             "parent",
             "preset",
+            "libraries",
         ]
 
 

backend/coreapp/views/libraries.py

@@ -16,7 +16,7 @@ class LibrariesDetail(APIView):
     @staticmethod
     def libraries_json() -> list[dict[str, object]]:
         return [
-            { 'name': l.name, 'version': l.version }
+            {"name": l.name, "supported_versions": l.supported_versions}
             for l in libraries.available_libraries()
         ]
 

backend/coreapp/views/scratch.py

@@ -26,6 +26,7 @@
 
 from ..diff_wrapper import DiffWrapper
 from ..error import CompilationError, DiffError
+from ..libraries import Library
 from ..middleware import Request
 from ..models.github import GitHubRepo, GitHubRepoBusyException
 from ..models.project import Project, ProjectFunction
@@ -65,6 +66,7 @@ def compile_scratch(scratch: Scratch) -> CompilationResult:
             scratch.source_code,
             scratch.context,
             scratch.diff_label,
+            tuple(scratch.libraries),
         )
     except CompilationError as e:
         return CompilationResult(b"", str(e))
@@ -371,6 +373,12 @@ def compile(self, request: Request, pk: str) -> Response:
                 scratch.source_code = request.data["source_code"]
             if "context" in request.data:
                 scratch.context = request.data["context"]
+            if "libraries" in request.data:
+                libs = [
+                    Library(name=data["name"], version=data["version"])
+                    for data in request.data["libraries"]
+                ]
+                scratch.libraries = libs
 
         compilation = compile_scratch(scratch)
         diff = diff_compilation(scratch, compilation)