diff --git a/backend/coreapp/compiler_wrapper.py b/backend/coreapp/compiler_wrapper.py index 3e2e6e5f..0b2ed097 100644 --- a/backend/coreapp/compiler_wrapper.py +++ b/backend/coreapp/compiler_wrapper.py @@ -6,7 +6,16 @@ from platform import uname import time -from typing import Any, Callable, Dict, Optional, Tuple, TYPE_CHECKING, TypeVar +from typing import ( + Any, + Callable, + Dict, + Optional, + Tuple, + TYPE_CHECKING, + TypeVar, + Sequence, +) from django.conf import settings @@ -17,6 +26,7 @@ import coreapp.util as util from .error import AssemblyError, CompilationError +from .libraries import LIBRARY_BASE_PATH, Library from .models.scratch import Asm, Assembly from .sandbox import Sandbox @@ -130,6 +140,7 @@ def compile_code( code: str, context: str, function: str = "", + libraries: Sequence[Library] = (), ) -> CompilationResult: if compiler == compilers.DUMMY: return CompilationResult(f"compiled({context}\n{code}".encode("UTF-8"), "") @@ -182,6 +193,12 @@ def compile_code( # Run compiler try: st = round(time.time() * 1000) + libraries_compiler_flags = " ".join( + ( + compiler.library_include_flag + str(lib.include_path) + for lib in libraries + ) + ) compile_proc = sandbox.run_subprocess( cc_cmd, mounts=( @@ -195,7 +212,8 @@ def compile_code( "INPUT": sandbox.rewrite_path(code_path), "OUTPUT": sandbox.rewrite_path(object_path), "COMPILER_DIR": sandbox.rewrite_path(compiler.path), - "COMPILER_FLAGS": sandbox.quote_options(compiler_flags), + "COMPILER_FLAGS": sandbox.quote_options(compiler_flags) + + libraries_compiler_flags, "FUNCTION": function, "MWCIncludes": "/tmp", "TMPDIR": "/tmp", diff --git a/backend/coreapp/compilers.py b/backend/coreapp/compilers.py index fd547a62..0e7d7d81 100644 --- a/backend/coreapp/compilers.py +++ b/backend/coreapp/compilers.py @@ -53,6 +53,7 @@ class Compiler: cc: str platform: Platform flags: ClassVar[Flags] + library_include_flag: str base_compiler: Optional["Compiler"] = None is_gcc: ClassVar[bool] = False is_ido: ClassVar[bool] = False @@ -96,6 +97,7 @@ def to_json(self) -> Dict[str, object]: @dataclass(frozen=True) class DummyCompiler(Compiler): flags: ClassVar[Flags] = [] + library_include_flag: str = "" def available(self) -> bool: return settings.DUMMY_COMPILER @@ -110,17 +112,20 @@ def available(self) -> bool: @dataclass(frozen=True) class ClangCompiler(Compiler): flags: ClassVar[Flags] = COMMON_CLANG_FLAGS + library_include_flag: str = "-isystem" @dataclass(frozen=True) class ArmccCompiler(Compiler): flags: ClassVar[Flags] = COMMON_ARMCC_FLAGS + library_include_flag: str = "-J" @dataclass(frozen=True) class GCCCompiler(Compiler): is_gcc: ClassVar[bool] = True flags: ClassVar[Flags] = COMMON_GCC_FLAGS + library_include_flag: str = "-isystem" @dataclass(frozen=True) @@ -137,22 +142,26 @@ class GCCSaturnCompiler(GCCCompiler): class IDOCompiler(Compiler): is_ido: ClassVar[bool] = True flags: ClassVar[Flags] = COMMON_IDO_FLAGS + library_include_flag: str = "-I" @dataclass(frozen=True) class MWCCCompiler(Compiler): is_mwcc: ClassVar[bool] = True flags: ClassVar[Flags] = COMMON_MWCC_FLAGS + library_include_flag: str = "-IZ:" @dataclass(frozen=True) class MSVCCompiler(Compiler): flags: ClassVar[Flags] = COMMON_MSVC_FLAGS + library_include_flag: str = "/IZ:" @dataclass(frozen=True) class WatcomCompiler(Compiler): flags: ClassVar[Flags] = COMMON_WATCOM_FLAGS + library_include_flag: str = "/IZ:" def from_id(compiler_id: str) -> Compiler: diff --git a/backend/coreapp/libraries.py b/backend/coreapp/libraries.py index 273c7448..991b269c 100644 --- a/backend/coreapp/libraries.py +++ b/backend/coreapp/libraries.py @@ -14,16 +14,29 @@ class Library: @property def path(self) -> Path: - return LIBRARY_BASE_PATH / name / version + return LIBRARY_BASE_PATH / self.name / self.version + + @property + def include_path(self) -> Path: + return self.path / "include" + + +@dataclass(frozen=True) +class LibraryVersions: + name: str + supported_versions: list[str] + + @property + def path(self) -> Path: + return LIBRARY_BASE_PATH / self.name - def available(self) -> bool: - return self.path.exists() @cache -def available_libraries() -> list[Library]: +def available_libraries() -> list[LibraryVersions]: results = [] for lib_dir in LIBRARY_BASE_PATH.iterdir(): + versions = [] if not lib_dir.is_dir(): continue for version_dir in lib_dir.iterdir(): @@ -32,8 +45,14 @@ def available_libraries() -> list[Library]: if not (version_dir / "include").exists(): continue - results.append(Library( - name=lib_dir.name, - version=version_dir.name, - )) + versions.append(version_dir.name) + + if len(versions) > 0: + results.append( + LibraryVersions( + name=lib_dir.name, + supported_versions=versions, + ) + ) + return results diff --git a/backend/coreapp/migrations/0038_scratch_libraries.py b/backend/coreapp/migrations/0038_scratch_libraries.py new file mode 100644 index 00000000..b3a5b2ae --- /dev/null +++ b/backend/coreapp/migrations/0038_scratch_libraries.py @@ -0,0 +1,17 @@ +# Generated by Django 4.2.3 on 2023-09-23 14:25 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("coreapp", "0037_rename_psyq43_to_psyq44"), + ] + + operations = [ + migrations.AddField( + model_name="scratch", + name="libraries", + field=models.JSONField(default=list), + ), + ] diff --git a/backend/coreapp/models/scratch.py b/backend/coreapp/models/scratch.py index 2c55b6d1..ede4a91b 100644 --- a/backend/coreapp/models/scratch.py +++ b/backend/coreapp/models/scratch.py @@ -1,11 +1,13 @@ +import json import logging from django.db import models from django.utils.crypto import get_random_string -from typing import List +from typing import Any, List from .profile import Profile +from ..libraries import Library logger = logging.getLogger(__name__) @@ -43,6 +45,27 @@ class CompilerConfig(models.Model): diff_flags = models.JSONField(default=list) +class LibrariesField(models.JSONField): + def __init__(self, **kwargs: Any): + class MyEncoder(json.JSONEncoder): + def default(self, obj: Any) -> str: + if isinstance(obj, Library): + obj = {"name": obj.name, "version": obj.version} + return super().default(obj) + + return super().__init__(encoder=MyEncoder, **kwargs) + + def to_python(self, value: Any) -> list[Library]: + res = super().to_python(value) + return [Library(name=lib["name"], version=lib["version"]) for lib in res] + + def from_db_value(self, *args: Any, **kwargs: Any) -> list[Library]: + # We ignore the type error here as this is a bug in the django stubs. + # CC: https://github.com/typeddjango/django-stubs/issues/934 + res = super().from_db_value(*args, **kwargs) # type: ignore + return [Library(name=lib["name"], version=lib["version"]) for lib in res] + + class Scratch(models.Model): slug = models.SlugField(primary_key=True, default=gen_scratch_id) name = models.CharField(max_length=1024, default="Untitled", blank=False) @@ -67,6 +90,7 @@ class Scratch(models.Model): score = models.IntegerField(default=-1) max_score = models.IntegerField(default=-1) match_override = models.BooleanField(default=False) + libraries = LibrariesField(default=list) parent = models.ForeignKey("self", null=True, blank=True, on_delete=models.SET_NULL) owner = models.ForeignKey(Profile, null=True, blank=True, on_delete=models.SET_NULL) project_function = models.ForeignKey( diff --git a/backend/coreapp/sandbox.py b/backend/coreapp/sandbox.py index 2a4c310b..13b61ce1 100644 --- a/backend/coreapp/sandbox.py +++ b/backend/coreapp/sandbox.py @@ -68,6 +68,7 @@ def sandbox_command(self, mounts: List[Path], env: Dict[str, str]) -> List[str]: "--bindmount_ro", "/proc", "--bindmount", f"{self.path}:/var/tmp", "--bindmount_ro", str(settings.COMPILER_BASE_PATH), + "--bindmount_ro", str(settings.LIBRARY_BASE_PATH), "--env", "PATH=/usr/bin:/bin", "--cwd", "/tmp", "--rlimit_fsize", "soft", diff --git a/backend/coreapp/serializers.py b/backend/coreapp/serializers.py index 64d03695..6cbfbb71 100644 --- a/backend/coreapp/serializers.py +++ b/backend/coreapp/serializers.py @@ -18,6 +18,7 @@ from .flags import LanguageFlagSet from . import compilers +from .libraries import Library def serialize_profile( @@ -135,6 +136,11 @@ class ScratchCreateSerializer(serializers.Serializer[None]): rom_address = serializers.IntegerField(required=False) +class LibrarySerializer(serializers.Serializer[Library]): + name = serializers.CharField() + version = serializers.CharField() + + class ScratchSerializer(serializers.HyperlinkedModelSerializer): slug = serializers.SlugField(read_only=True) url = UrlField() @@ -146,6 +152,7 @@ class ScratchSerializer(serializers.HyperlinkedModelSerializer): project = serializers.SerializerMethodField() project_function = serializers.SerializerMethodField() language = serializers.SerializerMethodField() + libraries = serializers.ListField(child=LibrarySerializer(), default=list) class Meta: model = Scratch @@ -243,6 +250,7 @@ class Meta: "project_function", "parent", "preset", + "libraries", ] diff --git a/backend/coreapp/views/libraries.py b/backend/coreapp/views/libraries.py index 5623cd47..c905d2b0 100644 --- a/backend/coreapp/views/libraries.py +++ b/backend/coreapp/views/libraries.py @@ -16,7 +16,7 @@ class LibrariesDetail(APIView): @staticmethod def libraries_json() -> list[dict[str, object]]: return [ - { 'name': l.name, 'version': l.version } + {"name": l.name, "supported_versions": l.supported_versions} for l in libraries.available_libraries() ] diff --git a/backend/coreapp/views/scratch.py b/backend/coreapp/views/scratch.py index 4794d9a6..ad9d6f7c 100644 --- a/backend/coreapp/views/scratch.py +++ b/backend/coreapp/views/scratch.py @@ -26,6 +26,7 @@ from ..diff_wrapper import DiffWrapper from ..error import CompilationError, DiffError +from ..libraries import Library from ..middleware import Request from ..models.github import GitHubRepo, GitHubRepoBusyException from ..models.project import Project, ProjectFunction @@ -65,6 +66,7 @@ def compile_scratch(scratch: Scratch) -> CompilationResult: scratch.source_code, scratch.context, scratch.diff_label, + tuple(scratch.libraries), ) except CompilationError as e: return CompilationResult(b"", str(e)) @@ -371,6 +373,12 @@ def compile(self, request: Request, pk: str) -> Response: scratch.source_code = request.data["source_code"] if "context" in request.data: scratch.context = request.data["context"] + if "libraries" in request.data: + libs = [ + Library(name=data["name"], version=data["version"]) + for data in request.data["libraries"] + ] + scratch.libraries = libs compilation = compile_scratch(scratch) diff = diff_compilation(scratch, compilation)