Support libraries in decomp.me (#843)

* Add script to download libraries, and download directx

* Add libraries endpoint

* Allow compiling with a library

* Add new libraries tab in frontend

* Add libraries support in CI

* Better look for the libraries

* Give pretty names to libraries

* Move Libraries to live under CompilerOpts

* typechecking hackery

* Make libraries trigger autorecomp and set the unsaved flag

* Fix libraryVersions

* Add new libraries download script to CI, docker and docs

---------

Co-authored-by: Ethan Roseman <[email protected]>

.github/workflows/ci.yml

@@ -80,6 +80,10 @@ jobs:
         run: |-
           cd backend
           poetry run python3 compilers/download.py
+      - name: Download libraries
+        run: |-
+          cd backend
+          poetry run python3 libraries/download.py
       - name: Install dkp dependencies (ppc)
         run: |-
           mkdir -p bin
@@ -175,16 +179,19 @@ jobs:
           mkdir -p sandbox && chmod 777 sandbox
           mkdir -p local_files && chmod 777 local_files
           mkdir -p compilers && chmod 777 compilers
+          mkdir -p libraries && chmod 777 libraries
           container_id=$(docker run \
             --detach \
             -v $(pwd):/decomp.me \
             -v $(pwd)/local_files:/local_files \
             -v $(pwd)/compilers:/compilers \
+            -v $(pwd)/libraries:/libraries \
             --device /dev/fuse \
             --security-opt apparmor=unconfined \
             --security-opt seccomp=unconfined \
             --entrypoint /bin/bash \
             -e COMPILER_BASE_PATH=/compilers \
+            -e LIBRARY_BASE_PATH=/libraries \
             -e WINEPREFIX=/tmp/wine \
             -e LOCAL_FILE_DIR=/local_files \
             -e USE_SANDBOX_JAIL=on \
@@ -195,6 +202,7 @@ jobs:
           docker exec ${container_id} /bin/bash -c 'cd /decomp.me/backend && \
             poetry install && \
             poetry run compilers/download.py --compilers-dir ${COMPILER_BASE_PATH} --podman && \
+            poetry run libraries/download.py --libraries-dir ${LIBRARY_BASE_PATH} && \
             poetry run python manage.py test'
 
   frontend_lint:

backend/coreapp/compiler_wrapper.py

@@ -6,7 +6,16 @@
 from platform import uname
 import time
 
-from typing import Any, Callable, Dict, Optional, Tuple, TYPE_CHECKING, TypeVar
+from typing import (
+    Any,
+    Callable,
+    Dict,
+    Optional,
+    Tuple,
+    TYPE_CHECKING,
+    TypeVar,
+    Sequence,
+)
 
 from django.conf import settings
 
@@ -17,6 +26,7 @@
 import coreapp.util as util
 
 from .error import AssemblyError, CompilationError
+from .libraries import LIBRARY_BASE_PATH, Library
 from .models.scratch import Asm, Assembly
 from .sandbox import Sandbox
 
@@ -130,6 +140,7 @@ def compile_code(
         code: str,
         context: str,
         function: str = "",
+        libraries: Sequence[Library] = (),
     ) -> CompilationResult:
         if compiler == compilers.DUMMY:
             return CompilationResult(f"compiled({context}\n{code}".encode("UTF-8"), "")
@@ -182,6 +193,12 @@ def compile_code(
             # Run compiler
             try:
                 st = round(time.time() * 1000)
+                libraries_compiler_flags = " ".join(
+                    (
+                        compiler.library_include_flag + str(lib.include_path)
+                        for lib in libraries
+                    )
+                )
                 compile_proc = sandbox.run_subprocess(
                     cc_cmd,
                     mounts=(
@@ -195,7 +212,9 @@ def compile_code(
                         "INPUT": sandbox.rewrite_path(code_path),
                         "OUTPUT": sandbox.rewrite_path(object_path),
                         "COMPILER_DIR": sandbox.rewrite_path(compiler.path),
-                        "COMPILER_FLAGS": sandbox.quote_options(compiler_flags),
+                        "COMPILER_FLAGS": sandbox.quote_options(
+                            compiler_flags + " " + libraries_compiler_flags
+                        ),
                         "FUNCTION": function,
                         "MWCIncludes": "/tmp",
                         "TMPDIR": "/tmp",

backend/coreapp/compilers.py

@@ -53,6 +53,7 @@ class Compiler:
     cc: str
     platform: Platform
     flags: ClassVar[Flags]
+    library_include_flag: str
     base_compiler: Optional["Compiler"] = None
     is_gcc: ClassVar[bool] = False
     is_ido: ClassVar[bool] = False
@@ -96,6 +97,7 @@ def to_json(self) -> Dict[str, object]:
 @dataclass(frozen=True)
 class DummyCompiler(Compiler):
     flags: ClassVar[Flags] = []
+    library_include_flag: str = ""
 
     def available(self) -> bool:
         return settings.DUMMY_COMPILER
@@ -110,17 +112,20 @@ def available(self) -> bool:
 @dataclass(frozen=True)
 class ClangCompiler(Compiler):
     flags: ClassVar[Flags] = COMMON_CLANG_FLAGS
+    library_include_flag: str = "-isystem"
 
 
 @dataclass(frozen=True)
 class ArmccCompiler(Compiler):
     flags: ClassVar[Flags] = COMMON_ARMCC_FLAGS
+    library_include_flag: str = "-J"
 
 
 @dataclass(frozen=True)
 class GCCCompiler(Compiler):
     is_gcc: ClassVar[bool] = True
     flags: ClassVar[Flags] = COMMON_GCC_FLAGS
+    library_include_flag: str = "-isystem"
 
 
 @dataclass(frozen=True)
@@ -137,22 +142,26 @@ class GCCSaturnCompiler(GCCCompiler):
 class IDOCompiler(Compiler):
     is_ido: ClassVar[bool] = True
     flags: ClassVar[Flags] = COMMON_IDO_FLAGS
+    library_include_flag: str = "-I"
 
 
 @dataclass(frozen=True)
 class MWCCCompiler(Compiler):
     is_mwcc: ClassVar[bool] = True
     flags: ClassVar[Flags] = COMMON_MWCC_FLAGS
+    library_include_flag: str = "-IZ:"
 
 
 @dataclass(frozen=True)
 class MSVCCompiler(Compiler):
     flags: ClassVar[Flags] = COMMON_MSVC_FLAGS
+    library_include_flag: str = "/IZ:"
 
 
 @dataclass(frozen=True)
 class WatcomCompiler(Compiler):
     flags: ClassVar[Flags] = COMMON_WATCOM_FLAGS
+    library_include_flag: str = "/IZ:"
 
 
 def from_id(compiler_id: str) -> Compiler:

backend/coreapp/libraries.py

@@ -0,0 +1,62 @@
+from dataclasses import dataclass
+from functools import cache
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from django.conf import settings
+
+if TYPE_CHECKING:
+    LIBRARY_BASE_PATH: Path
+else:
+    LIBRARY_BASE_PATH: Path = settings.LIBRARY_BASE_PATH
+
+
+@dataclass(frozen=True)
+class Library:
+    name: str
+    version: str
+
+    @property
+    def path(self) -> Path:
+        return LIBRARY_BASE_PATH / self.name / self.version
+
+    @property
+    def include_path(self) -> Path:
+        return self.path / "include"
+
+
+@dataclass(frozen=True)
+class LibraryVersions:
+    name: str
+    supported_versions: list[str]
+
+    @property
+    def path(self) -> Path:
+        return LIBRARY_BASE_PATH / self.name
+
+
+@cache
+def available_libraries() -> list[LibraryVersions]:
+    results = []
+
+    for lib_dir in LIBRARY_BASE_PATH.iterdir():
+        versions = []
+        if not lib_dir.is_dir():
+            continue
+        for version_dir in lib_dir.iterdir():
+            if not version_dir.is_dir():
+                continue
+            if not (version_dir / "include").exists():
+                continue
+
+            versions.append(version_dir.name)
+
+        if len(versions) > 0:
+            results.append(
+                LibraryVersions(
+                    name=lib_dir.name,
+                    supported_versions=versions,
+                )
+            )
+
+    return results

backend/coreapp/migrations/0038_scratch_libraries.py

@@ -0,0 +1,17 @@
+# Generated by Django 4.2.3 on 2023-09-23 14:25
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("coreapp", "0037_rename_psyq43_to_psyq44"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="scratch",
+            name="libraries",
+            field=models.JSONField(default=list),
+        ),
+    ]

backend/coreapp/models/scratch.py

@@ -1,11 +1,13 @@
+import json
 import logging
 
 from django.db import models
 from django.utils.crypto import get_random_string
 
-from typing import List
+from typing import Any, List
 
 from .profile import Profile
+from ..libraries import Library
 
 logger = logging.getLogger(__name__)
 
@@ -43,6 +45,27 @@ class CompilerConfig(models.Model):
     diff_flags = models.JSONField(default=list)
 
 
+class LibrariesField(models.JSONField):
+    def __init__(self, **kwargs: Any):
+        class MyEncoder(json.JSONEncoder):
+            def default(self, obj: Any) -> str:
+                if isinstance(obj, Library):
+                    obj = {"name": obj.name, "version": obj.version}
+                return super().default(obj)
+
+        return super().__init__(encoder=MyEncoder, **kwargs)
+
+    def to_python(self, value: Any) -> list[Library]:
+        res = super().to_python(value)
+        return [Library(name=lib["name"], version=lib["version"]) for lib in res]
+
+    def from_db_value(self, *args: Any, **kwargs: Any) -> list[Library]:
+        # We ignore the type error here as this is a bug in the django stubs.
+        # CC: https://github.com/typeddjango/django-stubs/issues/934
+        res = super().from_db_value(*args, **kwargs)  # type: ignore
+        return [Library(name=lib["name"], version=lib["version"]) for lib in res]
+
+
 class Scratch(models.Model):
     slug = models.SlugField(primary_key=True, default=gen_scratch_id)
     name = models.CharField(max_length=1024, default="Untitled", blank=False)
@@ -67,6 +90,7 @@ class Scratch(models.Model):
     score = models.IntegerField(default=-1)
     max_score = models.IntegerField(default=-1)
     match_override = models.BooleanField(default=False)
+    libraries = LibrariesField(default=list)
     parent = models.ForeignKey("self", null=True, blank=True, on_delete=models.SET_NULL)
     owner = models.ForeignKey(Profile, null=True, blank=True, on_delete=models.SET_NULL)
     project_function = models.ForeignKey(

backend/coreapp/sandbox.py

@@ -68,6 +68,7 @@ def sandbox_command(self, mounts: List[Path], env: Dict[str, str]) -> List[str]:
             "--bindmount_ro", "/proc",
             "--bindmount", f"{self.path}:/var/tmp",
             "--bindmount_ro", str(settings.COMPILER_BASE_PATH),
+            "--bindmount_ro", str(settings.LIBRARY_BASE_PATH),
             "--env", "PATH=/usr/bin:/bin",
             "--cwd", "/tmp",
             "--rlimit_fsize", "soft",

backend/coreapp/serializers.py

@@ -18,6 +18,7 @@
 
 from .flags import LanguageFlagSet
 from . import compilers
+from .libraries import Library
 
 
 def serialize_profile(
@@ -135,6 +136,11 @@ class ScratchCreateSerializer(serializers.Serializer[None]):
     rom_address = serializers.IntegerField(required=False)
 
 
+class LibrarySerializer(serializers.Serializer[Library]):
+    name = serializers.CharField()
+    version = serializers.CharField()
+
+
 class ScratchSerializer(serializers.HyperlinkedModelSerializer):
     slug = serializers.SlugField(read_only=True)
     url = UrlField()
@@ -146,6 +152,7 @@ class ScratchSerializer(serializers.HyperlinkedModelSerializer):
     project = serializers.SerializerMethodField()
     project_function = serializers.SerializerMethodField()
     language = serializers.SerializerMethodField()
+    libraries = serializers.ListField(child=LibrarySerializer(), default=list)
 
     class Meta:
         model = Scratch
@@ -243,6 +250,7 @@ class Meta:
             "project_function",
             "parent",
             "preset",
+            "libraries",
         ]
 
 

backend/coreapp/urls.py

@@ -1,9 +1,10 @@
 from django.urls import path
 
-from coreapp.views import compilers, stats, project, scratch, user
+from coreapp.views import compilers, libraries, stats, project, scratch, user
 
 urlpatterns = [
     path("compilers", compilers.CompilersDetail.as_view(), name="compilers"),
+    path("libraries", libraries.LibrariesDetail.as_view(), name="libraries"),
     path("stats", stats.StatsDetail.as_view(), name="stats"),
     *scratch.router.urls,
     *project.router.urls,

backend/coreapp/views/libraries.py

@@ -0,0 +1,33 @@
+from typing import Dict
+
+from django.utils.timezone import now
+from rest_framework.request import Request
+from rest_framework.response import Response
+from rest_framework.views import APIView
+
+from coreapp import libraries
+
+from ..decorators.django import condition
+
+boot_time = now()
+
+
+class LibrariesDetail(APIView):
+    @staticmethod
+    def libraries_json() -> list[dict[str, object]]:
+        return [
+            {"name": l.name, "supported_versions": l.supported_versions}
+            for l in libraries.available_libraries()
+        ]
+
+    @condition(last_modified_func=lambda request: boot_time)
+    def head(self, request: Request) -> Response:
+        return Response()
+
+    @condition(last_modified_func=lambda request: boot_time)
+    def get(self, request: Request) -> Response:
+        return Response(
+            {
+                "libraries": LibrariesDetail.libraries_json(),
+            }
+        )