From cf49ed345369b499ddaa15464df74300731659a1 Mon Sep 17 00:00:00 2001 From: Fabian Bonk Date: Wed, 29 Apr 2020 15:48:19 +0200 Subject: [PATCH] add PCIe support --- .gitignore | 1 + AUTHORS | 1 + bindings/GNUmakefile | 14 +- bindings/genode/stubs.c | 3 + bindings/hvt/bindings.h | 1 + bindings/hvt/pci.c | 61 ++++++ bindings/hvt/start.c | 1 + bindings/muen/muen-pci.c | 11 ++ bindings/spt/pci.c | 11 ++ bindings/virtio/pci-stubs.c | 11 ++ docs/architecture.md | 40 +++- elftool/elftool.c | 106 ++++++++++- include/solo5/hvt_abi.h | 22 +++ include/solo5/mft_abi.h | 30 +++ include/solo5/solo5.h | 42 +++++ tenders/GNUmakefile | 4 +- tenders/common/mft.c | 2 + tenders/common/pci_attach.c | 177 ++++++++++++++++++ tenders/common/pci_attach.h | 40 ++++ tenders/hvt/hvt_cpu_x86_64.c | 14 +- tenders/hvt/hvt_cpu_x86_64.h | 8 + tenders/hvt/hvt_module_pci.c | 237 ++++++++++++++++++++++++ tests/test_blk/manifest.json | 1 + tests/test_dumpcore/manifest.json | 1 + tests/test_exception/manifest.json | 1 + tests/test_fpu/manifest.json | 1 + tests/test_globals/manifest.json | 1 + tests/test_hello/manifest.json | 1 + tests/test_mft_maxdevices/manifest.json | 1 + tests/test_net/manifest.json | 1 + tests/test_net_2if/manifest.json | 1 + tests/test_notls/manifest.json | 1 + tests/test_pci/GNUmakefile | 23 +++ tests/test_pci/manifest.json | 23 +++ tests/test_pci/test_pci.c | 110 +++++++++++ tests/test_quiet/manifest.json | 1 + tests/test_seccomp/manifest.json | 1 + tests/test_ssp/manifest.json | 1 + tests/test_time/manifest.json | 1 + tests/test_tls/manifest.json | 1 + tests/test_wnox/manifest.json | 1 + tests/test_xnow/manifest.json | 1 + tests/test_zeropage/manifest.json | 1 + 43 files changed, 989 insertions(+), 22 deletions(-) create mode 100644 bindings/hvt/pci.c create mode 100644 bindings/muen/muen-pci.c create mode 100644 bindings/spt/pci.c create mode 100644 bindings/virtio/pci-stubs.c create mode 100644 tenders/common/pci_attach.c create mode 100644 tenders/common/pci_attach.h create mode 100644 tenders/hvt/hvt_module_pci.c create mode 100644 tests/test_pci/GNUmakefile create mode 100644 tests/test_pci/manifest.json create mode 100644 tests/test_pci/test_pci.c diff --git a/.gitignore b/.gitignore index 660e6111..82686549 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,4 @@ tenders/spt/solo5-spt elftool/solo5-elftool opam/release solo5-*.tar.gz +.vscode diff --git a/AUTHORS b/AUTHORS index 33642adf..19ebef44 100644 --- a/AUTHORS +++ b/AUTHORS @@ -12,6 +12,7 @@ Adrian-Ken Rueegsegger Adam Steen Dan Williams (IBM) Emery Hemingway (Genode Labs) +Fabian Bonk Gabriel Jaldon Hannes Mehnert Ian Campbell (Docker, Inc.) diff --git a/bindings/GNUmakefile b/bindings/GNUmakefile index 147c245c..59914f37 100644 --- a/bindings/GNUmakefile +++ b/bindings/GNUmakefile @@ -39,22 +39,22 @@ common_hvt_SRCS := hvt/platform.c hvt/platform_intr.c hvt/time.c # hvt_SRCS := hvt/start.c $(common_SRCS) $(common_hvt_SRCS) \ hvt/platform_lifecycle.c hvt/yield.c hvt/tscclock.c hvt/console.c \ - hvt/net.c hvt/block.c + hvt/net.c hvt/block.c hvt/pci.c spt_SRCS := spt/start.c \ abort.c crt.c printf.c lib.c mem.c exit.c log.c cmdline.c tls.c mft.c \ - spt/bindings.c spt/block.c spt/net.c spt/platform.c \ + spt/bindings.c spt/block.c spt/net.c spt/pci.c spt/platform.c \ spt/sys_linux_$(CONFIG_ARCH).c virtio_SRCS := virtio/boot.S virtio/start.c $(common_SRCS) \ virtio/platform.c virtio/platform_intr.c \ virtio/pci.c virtio/serial.c virtio/time.c virtio/virtio_ring.c \ virtio/virtio_net.c virtio/virtio_blk.c virtio/tscclock.c \ - virtio/clock_subr.c virtio/pvclock.c + virtio/clock_subr.c virtio/pvclock.c virtio/pci-stubs.c muen_SRCS := muen/start.c $(common_SRCS) $(common_hvt_SRCS) \ muen/channel.c muen/reader.c muen/writer.c muen/muen-block.c \ - muen/muen-clock.c muen/muen-console.c muen/muen-net.c \ + muen/muen-clock.c muen/muen-console.c muen/muen-net.c muen/muen-pci.c \ muen/muen-platform_lifecycle.c muen/muen-yield.c muen/muen-sinfo.c genode_SRCS := genode/stubs.c @@ -108,6 +108,8 @@ endif ifdef CONFIG_SPT spt_OBJS := $(patsubst %.c,%.o,$(patsubst %.S,%.o,$(spt_SRCS))) +spt/pci.o: CFLAGS += -Wno-unused-parameter + spt/solo5_spt.o: $(spt_OBJS) $(LINK.bindings) @@ -119,6 +121,8 @@ endif ifdef CONFIG_VIRTIO virtio_OBJS := $(patsubst %.c,%.o,$(patsubst %.S,%.o,$(virtio_SRCS))) +virtio/pci-stubs.o: CFLAGS += -Wno-unused-parameter + virtio/solo5_virtio.o: $(virtio_OBJS) $(LINK.bindings) @@ -130,6 +134,8 @@ endif ifdef CONFIG_MUEN muen_OBJS := $(patsubst %.c,%.o,$(patsubst %.S,%.o,$(muen_SRCS))) +muen/muen-pci.o: CFLAGS += -Wno-unused-parameter + muen/solo5_muen.o: $(muen_OBJS) $(LINK.bindings) diff --git a/bindings/genode/stubs.c b/bindings/genode/stubs.c index 0f1ca5cf..2b703777 100644 --- a/bindings/genode/stubs.c +++ b/bindings/genode/stubs.c @@ -19,5 +19,8 @@ solo5_result_t solo5_block_read(solo5_handle_t handle, solo5_off_t offset, uint8 solo5_result_t solo5_set_tls_base(uintptr_t base) { return SOLO5_R_EUNSPEC; } +solo5_result_t solo5_pci_acquire(const char *name, struct solo5_pci_info *info) { return SOLO5_R_EUNSPEC; } +solo5_result_t solo5_dma_acquire(uint8_t **buffer, size_t *size) { return SOLO5_R_EUNSPEC; } + uintptr_t SSP_GUARD; void SSP_FAIL (void) { } diff --git a/bindings/hvt/bindings.h b/bindings/hvt/bindings.h index 2cdcafa0..29dea984 100644 --- a/bindings/hvt/bindings.h +++ b/bindings/hvt/bindings.h @@ -35,6 +35,7 @@ void time_init(const struct hvt_boot_info *bi); void console_init(void); void net_init(const struct hvt_boot_info *bi); void block_init(const struct hvt_boot_info *bi); +void pci_init(const struct hvt_boot_info *bi); /* tscclock.c: TSC-based clock */ uint64_t tscclock_monotonic(void); diff --git a/bindings/hvt/pci.c b/bindings/hvt/pci.c new file mode 100644 index 00000000..ba18403c --- /dev/null +++ b/bindings/hvt/pci.c @@ -0,0 +1,61 @@ +#include "bindings.h" + +static const struct mft *mft; + +#define setup_bar(index, n) \ + do { \ + if (e->u.pci_basic.map_bar ## n) { \ + info->bar ## n = (uint8_t *) HVT_REGION(index, n); \ + info->bar ## n ## _size = e->u.pci_basic.bar ## n ## _size; \ + } else { \ + info->bar ## n = NULL; \ + info->bar ## n ## _size = 0; \ + } \ + } while (0) + +solo5_result_t solo5_pci_acquire(const char *name, struct solo5_pci_info *info) +{ + unsigned mft_index; + const struct mft_entry *e = + mft_get_by_name(mft, name, MFT_DEV_PCI_BASIC, &mft_index); + if (e == NULL) + return SOLO5_R_EINVAL; + assert(e->attached); + + info->bus_master_enable = e->u.pci_basic.bus_master_enable; + info->class_code = e->u.pci_basic.class_code; + info->subclass_code = e->u.pci_basic.subclass_code; + info->progif = e->u.pci_basic.progif; + info->vendor_id = e->u.pci_basic.vendor; + info->device_id = e->u.pci_basic.device_id; + + uint8_t index = e->u.pci_basic.device_index; + setup_bar(index, 0); + setup_bar(index, 1); + setup_bar(index, 2); + setup_bar(index, 3); + setup_bar(index, 4); + setup_bar(index, 5); + + assert(info->bar0_size != 0); + + return SOLO5_R_OK; +} + +solo5_result_t solo5_dma_acquire(uint8_t **buffer, size_t *size) +{ + if (mft->dma_size > 0) { + *buffer = (uint8_t *) HVT_DMA_BASE; + *size = mft->dma_size; + return SOLO5_R_OK; + } else { + *buffer = NULL; + *size = 0; + return SOLO5_R_EINVAL; + } +} + +void pci_init(const struct hvt_boot_info *bi) +{ + mft = bi->mft; +} diff --git a/bindings/hvt/start.c b/bindings/hvt/start.c index 30163472..555f6757 100644 --- a/bindings/hvt/start.c +++ b/bindings/hvt/start.c @@ -44,6 +44,7 @@ void _start(const void *arg) time_init(arg); block_init(arg); net_init(arg); + pci_init(arg); mem_lock_heap(&si.heap_start, &si.heap_size); solo5_exit(solo5_app_main(&si)); diff --git a/bindings/muen/muen-pci.c b/bindings/muen/muen-pci.c new file mode 100644 index 00000000..64c31fa6 --- /dev/null +++ b/bindings/muen/muen-pci.c @@ -0,0 +1,11 @@ +#include "bindings.h" + +solo5_result_t solo5_pci_acquire(const char *name, struct solo5_pci_info *info) +{ + return SOLO5_R_EUNSPEC; +} + +solo5_result_t solo5_dma_acquire(uint8_t **buffer, size_t *size) +{ + return SOLO5_R_EUNSPEC; +} diff --git a/bindings/spt/pci.c b/bindings/spt/pci.c new file mode 100644 index 00000000..64c31fa6 --- /dev/null +++ b/bindings/spt/pci.c @@ -0,0 +1,11 @@ +#include "bindings.h" + +solo5_result_t solo5_pci_acquire(const char *name, struct solo5_pci_info *info) +{ + return SOLO5_R_EUNSPEC; +} + +solo5_result_t solo5_dma_acquire(uint8_t **buffer, size_t *size) +{ + return SOLO5_R_EUNSPEC; +} diff --git a/bindings/virtio/pci-stubs.c b/bindings/virtio/pci-stubs.c new file mode 100644 index 00000000..64c31fa6 --- /dev/null +++ b/bindings/virtio/pci-stubs.c @@ -0,0 +1,11 @@ +#include "bindings.h" + +solo5_result_t solo5_pci_acquire(const char *name, struct solo5_pci_info *info) +{ + return SOLO5_R_EUNSPEC; +} + +solo5_result_t solo5_dma_acquire(uint8_t **buffer, size_t *size) +{ + return SOLO5_R_EUNSPEC; +} diff --git a/docs/architecture.md b/docs/architecture.md index aa7c26c7..7dbef32a 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -97,12 +97,32 @@ Solo5 introduces the concept of an _application manifest_, which is defined by the developer at unikernel build time, using the following JSON format and customarily named `manifest.json`: -```jsonc +```json { "type": "solo5.manifest", "version": 1, + "dma_size": 16777216, "devices": [ - { "name": "NAME", "type": "TYPE" } + { + "name": "NAME", + "type": "TYPE" + }, + { + "name": "pci0", + "type": "PCI_BASIC", + "class": 2, + "subclass": 0, + "progif": 0, + "vendor": 32902, + "device_id": 4347, + "bus_master_enable": true, + "map_bar0": true, + "map_bar1": false, + "map_bar2": false, + "map_bar3": false, + "map_bar4": false, + "map_bar5": false + } // ... up to 63 user-specified devices ... ] } @@ -115,8 +135,20 @@ of the device, eg. `frontend` for a network or `storage` for a block device. _NAME_ must be composed of alphanumeric characters only, and within 1..67 characters in length. -_TYPE_ is the type of device being declared, currently `BLOCK_BASIC` or -`NET_BASIC`. +_TYPE_ is the type of device being declared, currently `BLOCK_BASIC`, +`NET_BASIC`, `PCI_BASIC` or `DMA_BASIC`. + +`dma_size` is the amount of required DMA-ready memory to be mapped into the +unikernel in bytes. + +`PCI_BASIC` contains additional fields besides `name` and `type`: +* `class` - PCI device's expected class code +* `subclass` - PCI device's expected subclass code +* `progif` - PCI device's expected programming interface +* `vendor` - PCI device's expected vendor +* `device_id` - PCI device's expected device ID +* `bus_master_enable` - `true` if the device should be a bus master +* `map_barN` - `true` if the `N`th `BAR` should be mapped into the unikernel Note that there is a maximum limit of 63 user-specified devices in the manifest. diff --git a/elftool/elftool.c b/elftool/elftool.c index 260e1a1e..6c69ca0a 100644 --- a/elftool/elftool.c +++ b/elftool/elftool.c @@ -33,6 +33,7 @@ #include #include #include +#include #include #include #include @@ -72,6 +73,17 @@ static void jexpect(enum jtypes t, jvalue *v, const char *filename, jtypestr(t), jtypestr(v->d)); } +static void jexpectbool(jvalue *v, const char *filename, const char *loc) +{ + if (v->d != jtrue && v->d != jfalse) + errx(1, "%s:%zu: %s: expected bool, got %s", filename, v->line, loc, + jtypestr(v->d)); +} + +static const char *jbool(jvalue *v) { + return v->d == jtrue ? "true" : "false"; +} + static const char out_header[] = \ "/* Generated by solo5-elftool version %s, do not edit */\n\n" "#define MFT_ENTRIES %d\n" @@ -79,13 +91,35 @@ static const char out_header[] = \ "\n" "MFT1_NOTE_DECLARE_BEGIN\n" "{\n" - " .version = MFT_VERSION, .entries = %d,\n" + " .version = MFT_VERSION, .entries = %d, .dma_size = %lld,\n" " .e = {\n" " { .name = \"\", .type = MFT_RESERVED_FIRST },\n"; static const char out_entry[] = \ " { .name = \"%s\", .type = MFT_DEV_%s },\n"; +static const char out_entry_pci[] = \ + " {\n" + " .name = \"%s\",\n" + " .type = MFT_DEV_PCI_BASIC,\n" + " .u = {\n" + " .pci_basic = {\n" + " .bus_master_enable = %s,\n" + " .map_bar0 = %s,\n" + " .map_bar1 = %s,\n" + " .map_bar2 = %s,\n" + " .map_bar3 = %s,\n" + " .map_bar4 = %s,\n" + " .map_bar5 = %s,\n" + " .class_code = %#llx,\n" + " .subclass_code = %#llx,\n" + " .progif = %#llx,\n" + " .vendor = %#llx,\n" + " .device_id = %#llx,\n" + " }\n" + " }\n" + " },\n"; + static const char out_footer[] = \ " }\n" "}\n" @@ -122,7 +156,7 @@ static int elftool_gen_mft(const char *source, const char *output) fclose(sfp); jexpect(jobject, root, source, "(root)"); - jvalue *jtype = NULL, *jversion = NULL, *jdevices = NULL; + jvalue *jtype = NULL, *jversion = NULL, *jdevices = NULL, *jdma_size = NULL; /* * The generated manifest always contains 1 entry of type * MFT_RESERVED_FIRST which is added implicitly by us. @@ -146,6 +180,10 @@ static int elftool_gen_mft(const char *source, const char *output) } jdevices = *i; } + else if (strcmp((*i)->n, "dma_size") == 0) { + jexpect(jint, *i, source, ".version"); + jdma_size = *i; + } else errx(1, "%s:%zu: (root): unknown key: %s", source, (*i)->line, (*i)->n); @@ -157,6 +195,8 @@ static int elftool_gen_mft(const char *source, const char *output) errx(1, "%s:%zu: missing .version", source, root->line); if (jdevices == NULL) errx(1, "%s:%zu: missing .devices[]", source, root->line); + if (jdma_size == NULL) + errx(1, "%s:%zu: missing .dma_size", source, root->line); if (strcmp("solo5.manifest", jtype->u.s) != 0) errx(1, "%s:%zu: .type: invalid value, expected \"solo5.manifest\"", @@ -168,10 +208,13 @@ static int elftool_gen_mft(const char *source, const char *output) errx(1, "%s:%zu: .devices[]: too many entries, maximum %d", source, jdevices->line, MFT_MAX_ENTRIES); - fprintf(ofp, out_header, SOLO5_VERSION, entries, entries); + fprintf(ofp, out_header, SOLO5_VERSION, entries, entries, jdma_size->u.i); for (jvalue **i = jdevices->u.v; *i; ++i) { jexpect(jobject, *i, source, ".devices[]"); - jvalue *jdevname = NULL, *jdevtype = NULL; + jvalue *jdevname = NULL, *jdevtype = NULL, *jdevclass = NULL, + *jdevsubclass = NULL, *jdevprogif = NULL, *jdevvendor = NULL, + *jdevdevice_id = NULL, *jdevmap_bar[6] = { NULL }, + *jdevbus_master_enable = NULL; for (jvalue **j = (*i)->u.v; *j; ++j) { if (strcmp((*j)->n, "name") == 0) { jexpect(jstring, *j, source, ".devices[...]"); @@ -181,6 +224,34 @@ static int elftool_gen_mft(const char *source, const char *output) jexpect(jstring, *j, source, ".devices[...]"); jdevtype = *j; } + else if (strcmp((*j)->n, "class") == 0) { + jexpect(jint, *j, source, ".devices[...]"); + jdevclass = *j; + } + else if (strcmp((*j)->n, "subclass") == 0) { + jexpect(jint, *j, source, ".devices[...]"); + jdevsubclass = *j; + } + else if (strcmp((*j)->n, "progif") == 0) { + jexpect(jint, *j, source, ".devices[...]"); + jdevprogif = *j; + } + else if (strcmp((*j)->n, "vendor") == 0) { + jexpect(jint, *j, source, ".devices[...]"); + jdevvendor = *j; + } + else if (strcmp((*j)->n, "device_id") == 0) { + jexpect(jint, *j, source, ".devices[...]"); + jdevdevice_id = *j; + } + else if (strncmp((*j)->n, "map_bar", 6) == 0 && (*j)->n[7] >= '0' && (*j)->n[7] <= '5') { + jexpectbool(*j, source, ".devices[...]"); + jdevmap_bar[(*j)->n[7] - '0'] = *j; + } + else if (strcmp((*j)->n, "bus_master_enable") == 0) { + jexpectbool(*j, source, ".devices[...]"); + jdevbus_master_enable = *j; + } else errx(1, "%s:%zu: .devices[...]: unknown key: %s", source, (*j)->line, (*j)->n); @@ -199,7 +270,32 @@ static int elftool_gen_mft(const char *source, const char *output) source, jdevname->line); if (jdevtype == NULL) errx(1, "%s:%zu: .devices[...]: missing .type", source, (*i)->line); - fprintf(ofp, out_entry, jdevname->u.s, jdevtype->u.s); + if (strcmp(jdevtype->u.s, "PCI_BASIC") == 0) { + if (!jdevclass || !jdevsubclass || !jdevprogif || !jdevvendor + || !jdevdevice_id || !jdevmap_bar[0] || !jdevmap_bar[1] + || !jdevmap_bar[2] || !jdevmap_bar[3] || !jdevmap_bar[4] + || !jdevmap_bar[5] || !jdevbus_master_enable) { + errx(1, "%s:%zu: .devices[...]: PCI_BASIC is missing class," + "subclass, progif, vendor, device_id, bus_master_enable, " + "or map_bar*", source, jdevtype->line); + } + fprintf(ofp, out_entry_pci, jdevname->u.s, jbool(jdevbus_master_enable), + jbool(jdevmap_bar[0]), jbool(jdevmap_bar[1]), jbool(jdevmap_bar[2]), + jbool(jdevmap_bar[3]), jbool(jdevmap_bar[4]), jbool(jdevmap_bar[5]), + jdevclass->u.i, jdevsubclass->u.i, jdevprogif->u.i, jdevvendor->u.i, + jdevdevice_id->u.i); + } + else { + if (jdevclass || jdevsubclass || jdevprogif || jdevvendor + || jdevdevice_id || jdevmap_bar[0] || jdevmap_bar[1] + || jdevmap_bar[2] || jdevmap_bar[3] || jdevmap_bar[4] + || jdevmap_bar[5] || jdevbus_master_enable) { + errx(1, "%s:%zu: .devices[...]: non-PCI_BASIC can't have class, " + "subclass, progif, vendor, device_id, bus_master_enable, " + "or map_bar*", source, jdevtype->line); + } + fprintf(ofp, out_entry, jdevname->u.s, jdevtype->u.s); + } } fprintf(ofp, out_footer); diff --git a/include/solo5/hvt_abi.h b/include/solo5/hvt_abi.h index 52111b82..1a2efb80 100644 --- a/include/solo5/hvt_abi.h +++ b/include/solo5/hvt_abi.h @@ -277,4 +277,26 @@ struct hvt_hc_halt { int exit_status; }; +/* + * Each requested PCIe region will be mapped starting at HVT_PCI_REGION_BASE + * with HVT_PCI_REGION_OFFSET between regions and HVT_PCI_DEVICE_OFFSET between + * devices. + */ +// 512 GiB, chosen so the PML4 lookup will be 0x1 +#define HVT_PCI_DEVICE_BASE (1ULL << 39) +// 16 GiB +#define HVT_PCI_DEVICE_SIZE (1ULL << 34) +// 1 GiB +#define HVT_PCI_REGION_SIZE (1ULL << 30) + +#define HVT_DEVICE(index) (HVT_PCI_DEVICE_BASE + (HVT_PCI_DEVICE_SIZE * index)) +#define HVT_REGION(index, n) (HVT_DEVICE(index) + (HVT_PCI_REGION_SIZE * n)) + +/* + * DMA-ready memory visible to all PCIe devices will be mapped startring at + * HVT_DMA_BASE. + */ +// 1 TiB, chosen so the PML4 lookup will be 0x2 +#define HVT_DMA_BASE (1ULL << 40) + #endif /* HVT_ABI_H */ diff --git a/include/solo5/mft_abi.h b/include/solo5/mft_abi.h index 537c7bc5..41852be1 100644 --- a/include/solo5/mft_abi.h +++ b/include/solo5/mft_abi.h @@ -45,6 +45,7 @@ typedef enum mft_type { MFT_DEV_BLOCK_BASIC = 1, MFT_DEV_NET_BASIC, + MFT_DEV_PCI_BASIC, MFT_RESERVED_FIRST = (1U << 30) } mft_type_t; @@ -64,6 +65,33 @@ struct mft_net_basic { uint16_t mtu; }; +#define SOLO5_MAX_PCI_REGIONS 6 + +/* + * MFT_DEV_PCI_BASIC (basic PCIe device) properties. + */ +struct mft_pci_basic { + bool bus_master_enable; + bool map_bar0; + size_t bar0_size; + bool map_bar1; + size_t bar1_size; + bool map_bar2; + size_t bar2_size; + bool map_bar3; + size_t bar3_size; + bool map_bar4; + size_t bar4_size; + bool map_bar5; + size_t bar5_size; + uint8_t class_code; + uint8_t subclass_code; + uint8_t progif; + uint16_t vendor; + uint16_t device_id; + uint8_t device_index; +}; + #define MFT_NAME_SIZE 68 /* Bytes, including string terminator */ #define MFT_NAME_MAX 67 /* Characters */ @@ -76,6 +104,7 @@ struct mft_entry { union { struct mft_block_basic block_basic; struct mft_net_basic net_basic; + struct mft_pci_basic pci_basic; } u; union { int hostfd; /* Backing host descriptor OR */ @@ -114,6 +143,7 @@ _Static_assert(MFT_ENTRIES <= MFT_MAX_ENTRIES, "MFT_ENTRIES out of range"); struct mft { uint32_t version; uint32_t entries; + size_t dma_size; struct mft_entry e[MFT_ENTRIES]; }; diff --git a/include/solo5/solo5.h b/include/solo5/solo5.h index 6389b5a7..80f4104c 100644 --- a/include/solo5/solo5.h +++ b/include/solo5/solo5.h @@ -299,4 +299,46 @@ solo5_result_t solo5_block_write(solo5_handle_t handle, solo5_off_t offset, solo5_result_t solo5_block_read(solo5_handle_t handle, solo5_off_t offset, uint8_t *buf, size_t size); +/* + * PCIe I/O. + */ + +struct solo5_pci_info { + uint16_t vendor_id; /* This device's PCI vendor. */ + uint16_t device_id; /* This device's device ID. */ + uint8_t class_code; /* This device's class code. */ + uint8_t subclass_code; /* This device's subclass code. */ + uint8_t progif; /* This device's programming interface. */ + bool bus_master_enable; /* This device is a bus master. */ + uint8_t *bar0; /* This device's BAR0 or NULL. */ + size_t bar0_size; /* This device's BAR0 size or 0. */ + uint8_t *bar1; /* This device's BAR1 or NULL. */ + size_t bar1_size; /* This device's BAR1 size or 0. */ + uint8_t *bar2; /* This device's BAR2 or NULL. */ + size_t bar2_size; /* This device's BAR2 size or 0. */ + uint8_t *bar3; /* This device's BAR3 or NULL. */ + size_t bar3_size; /* This device's BAR3 size or 0. */ + uint8_t *bar4; /* This device's BAR4 or NULL. */ + size_t bar4_size; /* This device's BAR4 size or 0. */ + uint8_t *bar5; /* This device's BAR5 or NULL. */ + size_t bar5_size; /* This device's BAR5 size or 0. */ +}; + +/* + * Acquires a handle to the PCIe device declared as (name) in the + * application manifest. The returned handle is stored in (*handle), and + * properties of the PCIe device are stored in (*info). Caller must supply + * space for struct solo5_pci_info in (info). + */ +solo5_result_t solo5_pci_acquire(const char *name, struct solo5_pci_info *info); + +/* + * Acquires a handle to the DMA-ready memory mapped into the unikernel's + * address space and visible to all PCIe devices. The address of the memory area + * will be stored in (*buffer), its size will be stored in (*size). If there + * is no DMA-ready memory avaible to the unikernel, (*buffer) is set to NULL, + * (*size) is set to 0 and the function returns SOLO5_R_EINVAL. + */ +solo5_result_t solo5_dma_acquire(uint8_t **buffer, size_t *size); + #endif diff --git a/tenders/GNUmakefile b/tenders/GNUmakefile index ff8ac6dd..53edcd56 100644 --- a/tenders/GNUmakefile +++ b/tenders/GNUmakefile @@ -32,7 +32,7 @@ ifneq ($(filter 1,$(CONFIG_HVT) $(CONFIG_SPT)),) common_LIB := common/libcommon.a common_SRCS := common/elf.c common/mft.c common/block_attach.c \ - common/tap_attach.c + common/tap_attach.c common/pci_attach.c common_OBJS := $(patsubst %.c,%.o,$(common_SRCS)) $(common_LIB): $(common_OBJS) @@ -56,7 +56,7 @@ ifdef CONFIG_HVT hvt_SRCS := hvt/hvt_boot_info.c hvt/hvt_core.c hvt/hvt_main.c \ hvt/hvt_cpu_$(CONFIG_ARCH).c -hvt_MODULES ?= blk net +hvt_MODULES ?= blk net pci ifeq ($(CONFIG_HOST), Linux) hvt_SRCS += hvt/hvt_kvm.c hvt/hvt_kvm_$(CONFIG_ARCH).c diff --git a/tenders/common/mft.c b/tenders/common/mft.c index 56024ab9..96117fa6 100644 --- a/tenders/common/mft.c +++ b/tenders/common/mft.c @@ -139,6 +139,8 @@ const char *mft_type_to_string(mft_type_t type) return "BLOCK_BASIC"; case MFT_DEV_NET_BASIC: return "NET_BASIC"; + case MFT_DEV_PCI_BASIC: + return "PCI_BASIC"; case MFT_RESERVED_FIRST: return "RESERVED_FIRST"; default: diff --git a/tenders/common/pci_attach.c b/tenders/common/pci_attach.c new file mode 100644 index 00000000..daf14acc --- /dev/null +++ b/tenders/common/pci_attach.c @@ -0,0 +1,177 @@ +/* + * Adapted from libixy-vfio. + * https://github.com/emmericp/ixy + */ + +#define _XOPEN_SOURCE 700 +#define _DEFAULT_SOURCE + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "pci_attach.h" + +volatile int VFIO_CONTAINER_FILE_DESCRIPTOR = -1; + +int get_vfio_container() +{ + return VFIO_CONTAINER_FILE_DESCRIPTOR; +} + +void set_vfio_container(int fd) +{ + VFIO_CONTAINER_FILE_DESCRIPTOR = fd; +} + +size_t MIN_DMA_MEMORY = 4096; // we can not allocate less than page_size memory + +void pci_enable_dma(int vfio_fd) { + // write to the command register (offset 4) in the PCIe config space + int command_register_offset = 4; + // bit 2 is "bus master enable", see PCIe 3.0 specification section 7.5.1.1 + int bus_master_enable_bit = 2; + // Get region info for config region + struct vfio_region_info conf_reg = {.argsz = sizeof(conf_reg)}; + conf_reg.index = VFIO_PCI_CONFIG_REGION_INDEX; + if (ioctl(vfio_fd, VFIO_DEVICE_GET_REGION_INFO, &conf_reg) == -1) { + err(1, "Failed to get vfio config region info."); + } + uint16_t dma = 0; + assert(pread(vfio_fd, &dma, 2, conf_reg.offset + command_register_offset) == 2); + dma |= 1 << bus_master_enable_bit; + assert(pwrite(vfio_fd, &dma, 2, conf_reg.offset + command_register_offset) == 2); +} + +// returns the devices file descriptor or -1 on error +int pci_attach(const char *pci_addr) { + // find iommu group for the device + // `readlink /sys/bus/pci/device//iommu_group` + char path[PATH_MAX], iommu_group_path[PATH_MAX]; + struct stat st; + snprintf(path, sizeof(path), "/sys/bus/pci/devices/%s/", pci_addr); + if (stat(path, &st) == -1) { + errx(1, "no such device: %s", pci_addr); + } + strncat(path, "iommu_group", sizeof(path) - strlen(path) - 1); + + int len = readlink(path, iommu_group_path, sizeof(iommu_group_path)); + if (len == -1) { + err(1, "Failed to find the iommu_group for the device %s", pci_addr); + } + + iommu_group_path[len] = '\0'; // append 0x00 to the string to end it + char* group_name = basename(iommu_group_path); + int groupid; + if (sscanf(group_name, "%d", &groupid) == -1) { + errx(1, "Failed to convert group id to int."); + } + + int firstsetup = 0; // Need to set up the container exactly once + int cfd = get_vfio_container(); + if (cfd == -1) { + firstsetup = 1; + // open vfio file to create new vfio container + cfd = open("/dev/vfio/vfio", O_RDWR); + if (cfd == -1) { + err(1, "Failed to open '/dev/vfio/vfio'"); + } + set_vfio_container(cfd); + + // check if the container's API version is the same as the VFIO API's + if (ioctl(cfd, VFIO_GET_API_VERSION) != VFIO_API_VERSION) { + err(1, "Failed to get a valid API version from the container."); + } + + // check if type1 is supported + if (ioctl(cfd, VFIO_CHECK_EXTENSION, VFIO_TYPE1_IOMMU) != 1) { + err(1, "Failed to get Type1 IOMMU support from the IOMMU container."); + } + } + + // open VFIO group containing the device + snprintf(path, sizeof(path), "/dev/vfio/%d", groupid); + int vfio_gfd = open(path, O_RDWR); + if (vfio_gfd == -1) { + err(1, "Failed to open vfio group."); + } + + // check if group is viable + struct vfio_group_status group_status = {.argsz = sizeof(group_status)}; + if (ioctl(vfio_gfd, VFIO_GROUP_GET_STATUS, &group_status) == -1) { + err(1, "Failed to get VFIO group status."); + } + if (!(group_status.flags & VFIO_GROUP_FLAGS_VIABLE)) { + errx(1, "Failed to get viable VFIO group - " + "are all devices in the group bound to the VFIO driver?"); + } + + // Add group to container + if (ioctl(vfio_gfd, VFIO_GROUP_SET_CONTAINER, &cfd) == -1) { + err(1, "Failed to set container."); + } + + if (firstsetup != 0) { + // Set vfio type (type1 is for IOMMU like VT-d or AMD-Vi) for the + // container. + // This can only be done after at least one group is in the container. + if (ioctl(cfd, VFIO_SET_IOMMU, VFIO_TYPE1_IOMMU) == -1) { + err(1, "Failed to set IOMMU type."); + } + } + + // get device file descriptor + int vfio_fd = ioctl(vfio_gfd, VFIO_GROUP_GET_DEVICE_FD, pci_addr); + if (vfio_fd == -1) { + err(1, "Failed to get device fd."); + } + + return vfio_fd; +} + +void *pci_map_region(int vfio_fd, int region_index, size_t *size) { + struct vfio_region_info region_info = {.argsz = sizeof(region_info)}; + region_info.index = region_index; + if (ioctl(vfio_fd, VFIO_DEVICE_GET_REGION_INFO, ®ion_info) == -1) { + err(1, "Failed to set IOMMU type."); + } + + void *buffer = mmap(NULL, region_info.size, PROT_READ | PROT_WRITE, + MAP_SHARED, vfio_fd, region_info.offset); + + if (buffer == MAP_FAILED) { + err(1, "Failed to mmap vfio resource."); + } + + *size = region_info.size; + + return buffer; +} + +void *pci_allocate_dma(size_t size) { + void *buffer = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_ANONYMOUS | MAP_HUGETLB | MAP_HUGE_2MB, -1, 0); + if (buffer == MAP_FAILED) { + err(1, "Failed to mmap hugepage."); + } + return buffer; +} diff --git a/tenders/common/pci_attach.h b/tenders/common/pci_attach.h new file mode 100644 index 00000000..833f584a --- /dev/null +++ b/tenders/common/pci_attach.h @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2015-2019 Contributors as noted in the AUTHORS file + * + * This file is part of Solo5, a sandboxed execution environment. + * + * Permission to use, copy, modify, and/or distribute this software + * for any purpose with or without fee is hereby granted, provided + * that the above copyright notice and this permission notice appear + * in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL + * WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE + * AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR + * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS + * OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, + * NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN + * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +/* + * pci_attach.h: Common functions for attaching PCIe devices. + */ + +#ifndef COMMON_PCI_ATTACH_H +#define COMMON_PCI_ATTACH_H + +#include + +int pci_attach(const char *pci_addr); + +void pci_enable_dma(int vfio_fd); + +void *pci_map_region(int vfio_fd, int region_index, size_t *size); + +void *pci_allocate_dma(size_t size); + +int get_vfio_container(); + +#endif /* COMMON_PCI_ATTACH_H */ diff --git a/tenders/hvt/hvt_cpu_x86_64.c b/tenders/hvt/hvt_cpu_x86_64.c index 78087829..245d4fed 100644 --- a/tenders/hvt/hvt_cpu_x86_64.c +++ b/tenders/hvt/hvt_cpu_x86_64.c @@ -55,7 +55,7 @@ void hvt_x86_setup_pagetables(uint8_t *mem, size_t mem_size) /* * For simplicity we currently use 2MB pages and only a single - * PML4/PDPTE/PDE. Sanity check that the guest size is a multiple of the + * PML4/PDPTE/PDE. Sanity check that the guest size is a multiple of the * page size and will fit in a single PDE (512 entries). Additionally, * check that the guest size is at least 2MB. */ @@ -86,14 +86,14 @@ void hvt_x86_setup_pagetables(uint8_t *mem, size_t mem_size) */ if (paddr < X86_PT0_MAP_START) continue; - /* - * Map the remainder of the pages below X86_GUEST_MIN_BASE as read-only; - * these are used for input from hvt to the guest only, with the rest - * reserved for future use. - */ + /* + * Map the remainder of the pages below X86_GUEST_MIN_BASE as read-only; + * these are used for input from hvt to the guest only, with the rest + * reserved for future use. + */ if (paddr < X86_GUEST_MIN_BASE) *pt0e = paddr | X86_PDPT_P; - else + else *pt0e = paddr | (X86_PDPT_P | X86_PDPT_RW); } assert(paddr == X86_GUEST_PAGE_SIZE); diff --git a/tenders/hvt/hvt_cpu_x86_64.h b/tenders/hvt/hvt_cpu_x86_64.h index 73d20a03..63cc6ac1 100644 --- a/tenders/hvt/hvt_cpu_x86_64.h +++ b/tenders/hvt/hvt_cpu_x86_64.h @@ -202,6 +202,14 @@ static const struct x86_sreg hvt_x86_sreg_unusable = { #define X86_PDE_SIZE 0x1000 #define X86_PT0E_BASE 0x5000 #define X86_PTE_SIZE 0x1000 +#define X86_PDPTE_DMA_BASE 0x6000 +#define X86_PDPTE_DMA_SIZE 0x1000 +#define X86_PDE_DMA_BASE 0x7000 +#define X86_PDE_DMA_SIZE 0x1000 +#define X86_PDPTE_PCI_BASE 0x8000 +#define X86_PDPTE_PCI_SIZE 0x1000 +#define X86_PDE_PCI_BASE 0x9000 +#define X86_PDE_PCI_SIZE 0x1000 #define X86_BOOT_INFO_BASE 0x10000 #define X86_PT0_MAP_START X86_BOOT_INFO_BASE #define X86_GUEST_MIN_BASE HVT_GUEST_MIN_BASE diff --git a/tenders/hvt/hvt_module_pci.c b/tenders/hvt/hvt_module_pci.c new file mode 100644 index 00000000..b623aebc --- /dev/null +++ b/tenders/hvt/hvt_module_pci.c @@ -0,0 +1,237 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../common/pci_attach.h" +#include "hvt.h" +#include "hvt_kvm.h" +#include "hvt_cpu_x86_64.h" +#include "solo5.h" +#include "hvt_abi.h" + +static bool module_in_use; +static unsigned device_index = 0; +static int max_slots; +static int current_slot = 1; // slot 0 contains the unikernel's main memory +static struct mft *host_mft; + +static int handle_cmdarg(char *cmdarg, struct mft *mft) +{ + char name[MFT_NAME_SIZE]; + unsigned dom, bus, dev, func; + char addr[13]; /* PCIe addresses look like '00:00.0'. */ + int rc = sscanf(cmdarg, "--pci:%" XSTR(MFT_NAME_MAX) + "[A-Za-z0-9]=%4x:%2x:%2x.%1x", name, &dom, &bus, &dev, &func); + if (rc != 5) + return -1; + + sprintf(addr, "%04x:%02x:%02x.%01x", dom, bus, dev, func); + + struct mft_entry *e = mft_get_by_name(mft, name, MFT_DEV_PCI_BASIC, NULL); + if (e == NULL) { + warnx("Resource not declared in manifest: '%s'", name); + return -1; + } + + printf("attaching device %s\n", name); + + int fd = pci_attach(addr); + if (fd < 0) { + errx(1, "Could not attach PCIe device: %s", name); + } + + // TODO check vendor, device_id, ... + + e->u.pci_basic.device_index = device_index++; + + if (e->u.pci_basic.bus_master_enable) { + pci_enable_dma(fd); + } + + e->b.hostfd = fd; + e->attached = true; + + module_in_use = true; + + return 0; +} + +#define HUGE_PAGE_BITS 21 +#define HUGE_PAGE_SIZE (1 << HUGE_PAGE_BITS) + +static void hvt_x86_setup_dma_pagetable(struct hvt *hvt, struct mft *mft) +{ + size_t size = mft->dma_size; + assert((size & (HUGE_PAGE_SIZE - 1)) == 0); + assert(size >= HUGE_PAGE_SIZE); + assert(size < (512 * HUGE_PAGE_SIZE)); + + uint64_t *pml4 = (uint64_t *) (hvt->mem + X86_PML4_BASE); + uint64_t *pdpte = (uint64_t *) (hvt->mem + X86_PDPTE_DMA_BASE); + uint64_t *pde = (uint64_t *) (hvt->mem + X86_PDE_DMA_BASE); + + memset(pdpte, 0, X86_PDPTE_DMA_SIZE); + memset(pde, 0, X86_PDE_DMA_SIZE); + + pml4[2] = X86_PDPTE_DMA_BASE | (X86_PDPT_P | X86_PDPT_RW); + pdpte[0] = X86_PDE_DMA_BASE | (X86_PDPT_P | X86_PDPT_RW); + + uint64_t paddr = HVT_DMA_BASE; + for (; paddr < HVT_DMA_BASE + size; paddr += HUGE_PAGE_SIZE, pde++) + *pde = paddr | (X86_PDPT_P | X86_PDPT_RW | X86_PDPT_PS); +} + +/* + * TODO this works but is super-wonky + * We always map 2 MiB per region into the unikernel, even if the region is + * smaller. We only support 1 pci device. + */ +static void hvt_x86_setup_pci_device_pagetable(struct hvt *hvt, + struct mft_entry *e) +{ + assert(e->type == MFT_DEV_PCI_BASIC); + struct mft_pci_basic pci = e->u.pci_basic; + assert(pci.device_index == 0); + + uint64_t *pml4 = (uint64_t *) (hvt->mem + X86_PML4_BASE); + uint64_t *pdpte = (uint64_t *) (hvt->mem + X86_PDPTE_PCI_BASE); + uint64_t *pde = (uint64_t *) (hvt->mem + X86_PDE_PCI_BASE); + + pml4[1] = X86_PDPTE_PCI_BASE | (X86_PDPT_P | X86_PDPT_RW); + pdpte[0] = X86_PDE_PCI_BASE | (X86_PDPT_P | X86_PDPT_RW); + +#define setup_region_pagetable(n) \ + do { \ + if (pci.map_bar ## n) { \ + assert(pci.bar ## n ## _size <= HUGE_PAGE_SIZE); \ + pde[n] = HVT_REGION(pci.device_index, (n)) | \ + (X86_PDPT_P | X86_PDPT_RW | X86_PDPT_PS); \ + } \ + } while (0) + + setup_region_pagetable(0); + setup_region_pagetable(1); + setup_region_pagetable(2); + setup_region_pagetable(3); + setup_region_pagetable(4); + setup_region_pagetable(5); +} + +static int get_slot() +{ + if (current_slot >= max_slots) { + errx(1, "max_slots (%d) exceeded", max_slots); + } + return current_slot++; +} + +static int setup(struct hvt *hvt, struct mft *mft) +{ + if (!module_in_use && mft->dma_size != 0) + warn("DMA memory requested but no PCI devices mapped, " + "ignoring DMA request"); + + if (!module_in_use) + return 0; + + host_mft = mft; + + max_slots = ioctl(hvt->b->kvmfd, KVM_CHECK_EXTENSION, KVM_CAP_NR_MEMSLOTS); + if (max_slots <= 0) { + err(1, "could not get KVM_CAP_NR_MEMSLOTS"); + } + + if (mft->dma_size > 0) { + if (mft->dma_size & (HUGE_PAGE_SIZE - 1)) { + mft->dma_size = + ((mft->dma_size >> HUGE_PAGE_BITS) + 1) << HUGE_PAGE_BITS; + } + void *buffer = pci_allocate_dma(mft->dma_size); + if (buffer == MAP_FAILED) { + err(1, "could not allocate dma memory"); + } + + struct vfio_iommu_type1_dma_map dma_map = { + .argsz = sizeof(dma_map), + .flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE, + .vaddr = (uint64_t) buffer, + .iova = HVT_DMA_BASE, + .size = mft->dma_size + }; + if (ioctl(get_vfio_container(), VFIO_IOMMU_MAP_DMA, &dma_map) == -1) { + err(1, "could not map dma vfio"); + } + + // make sure the DMA-memory is page-aligned + // see KVM_SET_USER_MEMORY_REGION documentation + assert((((uint64_t) buffer) & (HUGE_PAGE_SIZE - 1)) == 0); + + struct kvm_userspace_memory_region region = { + .slot = get_slot(), + .guest_phys_addr = HVT_DMA_BASE, + .memory_size = mft->dma_size, + .userspace_addr = (uint64_t) buffer + }; + if (ioctl(hvt->b->vmfd, KVM_SET_USER_MEMORY_REGION, ®ion) == -1) + err(1, "KVM: ioctl (SET_USER_MEMORY_REGION) for DMA memory failed"); + + hvt_x86_setup_dma_pagetable(hvt, mft); + } + + for (int i = 0; i < mft->entries; i++) { + struct mft_entry *e = &mft->e[i]; + +#define map_region(n) \ + do { \ + if (e->u.pci_basic.map_bar ## n) { \ + size_t size; \ + void *buffer = pci_map_region(e->b.hostfd, \ + VFIO_PCI_BAR ## n ## _REGION_INDEX, &size); \ + if (buffer == MAP_FAILED) { \ + errx(1, "could not map BAR" #n); \ + } \ + e->u.pci_basic.bar ## n ## _size = size; \ + uint64_t phys_addr = HVT_REGION(e->u.pci_basic.device_index, n); \ + printf("mapping BAR" #n " to %#lx\n", phys_addr); \ + struct kvm_userspace_memory_region region = { \ + .slot = get_slot(), \ + .guest_phys_addr = phys_addr, \ + .memory_size = size, \ + .userspace_addr = (uint64_t) buffer \ + }; \ + if (ioctl(hvt->b->vmfd, KVM_SET_USER_MEMORY_REGION, \ + ®ion) == -1) \ + err(1, "KVM: ioctl (SET_USER_MEMORY_REGION) for " \ + "BAR" #n " failed"); \ + hvt_x86_setup_pci_device_pagetable(hvt, e); \ + } \ + } while (0) + + if (e->type == MFT_DEV_PCI_BASIC) { + map_region(0); + map_region(1); + map_region(2); + map_region(3); + map_region(4); + map_region(5); + } + } + + return 0; +} + +static char *usage(void) +{ + return "--pci:NAME=ADDR (attach PCIe device at address ADDR as NAME)"; +} + +DECLARE_MODULE(pci, + .setup = setup, + .handle_cmdarg = handle_cmdarg, + .usage = usage +) diff --git a/tests/test_blk/manifest.json b/tests/test_blk/manifest.json index 668406f1..527a9baa 100644 --- a/tests/test_blk/manifest.json +++ b/tests/test_blk/manifest.json @@ -1,5 +1,6 @@ { "type": "solo5.manifest", "version": 1, + "dma_size": 0, "devices": [ { "name": "storage", "type": "BLOCK_BASIC" } ] } diff --git a/tests/test_dumpcore/manifest.json b/tests/test_dumpcore/manifest.json index 1100fc5b..a6c72956 100644 --- a/tests/test_dumpcore/manifest.json +++ b/tests/test_dumpcore/manifest.json @@ -1,5 +1,6 @@ { "type": "solo5.manifest", "version": 1, + "dma_size": 0, "devices": [ ] } diff --git a/tests/test_exception/manifest.json b/tests/test_exception/manifest.json index 1100fc5b..a6c72956 100644 --- a/tests/test_exception/manifest.json +++ b/tests/test_exception/manifest.json @@ -1,5 +1,6 @@ { "type": "solo5.manifest", "version": 1, + "dma_size": 0, "devices": [ ] } diff --git a/tests/test_fpu/manifest.json b/tests/test_fpu/manifest.json index 1100fc5b..a6c72956 100644 --- a/tests/test_fpu/manifest.json +++ b/tests/test_fpu/manifest.json @@ -1,5 +1,6 @@ { "type": "solo5.manifest", "version": 1, + "dma_size": 0, "devices": [ ] } diff --git a/tests/test_globals/manifest.json b/tests/test_globals/manifest.json index 1100fc5b..a6c72956 100644 --- a/tests/test_globals/manifest.json +++ b/tests/test_globals/manifest.json @@ -1,5 +1,6 @@ { "type": "solo5.manifest", "version": 1, + "dma_size": 0, "devices": [ ] } diff --git a/tests/test_hello/manifest.json b/tests/test_hello/manifest.json index 1100fc5b..a6c72956 100644 --- a/tests/test_hello/manifest.json +++ b/tests/test_hello/manifest.json @@ -1,5 +1,6 @@ { "type": "solo5.manifest", "version": 1, + "dma_size": 0, "devices": [ ] } diff --git a/tests/test_mft_maxdevices/manifest.json b/tests/test_mft_maxdevices/manifest.json index fd935ac5..036785c3 100644 --- a/tests/test_mft_maxdevices/manifest.json +++ b/tests/test_mft_maxdevices/manifest.json @@ -1,6 +1,7 @@ { "type": "solo5.manifest", "version": 1, + "dma_size": 0, "devices": [ { "name": "storage0", "type": "BLOCK_BASIC" }, { "name": "storage1", "type": "BLOCK_BASIC" }, diff --git a/tests/test_net/manifest.json b/tests/test_net/manifest.json index c0b22846..1e0af18a 100644 --- a/tests/test_net/manifest.json +++ b/tests/test_net/manifest.json @@ -1,5 +1,6 @@ { "type": "solo5.manifest", "version": 1, + "dma_size": 0, "devices": [ { "name": "service0", "type": "NET_BASIC" } ] } diff --git a/tests/test_net_2if/manifest.json b/tests/test_net_2if/manifest.json index ddb05e2e..62eb0b3e 100644 --- a/tests/test_net_2if/manifest.json +++ b/tests/test_net_2if/manifest.json @@ -1,6 +1,7 @@ { "type": "solo5.manifest", "version": 1, + "dma_size": 0, "devices": [ { "name": "service0", "type": "NET_BASIC" }, { "name": "service1", "type": "NET_BASIC" } diff --git a/tests/test_notls/manifest.json b/tests/test_notls/manifest.json index 1100fc5b..a6c72956 100644 --- a/tests/test_notls/manifest.json +++ b/tests/test_notls/manifest.json @@ -1,5 +1,6 @@ { "type": "solo5.manifest", "version": 1, + "dma_size": 0, "devices": [ ] } diff --git a/tests/test_pci/GNUmakefile b/tests/test_pci/GNUmakefile new file mode 100644 index 00000000..710d5552 --- /dev/null +++ b/tests/test_pci/GNUmakefile @@ -0,0 +1,23 @@ +# Copyright (c) 2015-2019 Contributors as noted in the AUTHORS file +# +# This file is part of Solo5, a sandboxed execution environment. +# +# Permission to use, copy, modify, and/or distribute this software +# for any purpose with or without fee is hereby granted, provided +# that the above copyright notice and this permission notice appear +# in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL +# WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE +# AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR +# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS +# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, +# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +include $(TOPDIR)/Makefile.common + +test_NAME := test_pci + +include ../Makefile.tests diff --git a/tests/test_pci/manifest.json b/tests/test_pci/manifest.json new file mode 100644 index 00000000..3835e58d --- /dev/null +++ b/tests/test_pci/manifest.json @@ -0,0 +1,23 @@ +{ + "type": "solo5.manifest", + "version": 1, + "dma_size": 4194304, + "devices": [ + { + "name": "pci0", + "type": "PCI_BASIC", + "class": 2, + "subclass": 0, + "progif": 0, + "vendor": 32902, + "device_id": 4347, + "bus_master_enable": true, + "map_bar0": true, + "map_bar1": false, + "map_bar2": false, + "map_bar3": false, + "map_bar4": false, + "map_bar5": false + } + ] +} diff --git a/tests/test_pci/test_pci.c b/tests/test_pci/test_pci.c new file mode 100644 index 00000000..1d8c3a5f --- /dev/null +++ b/tests/test_pci/test_pci.c @@ -0,0 +1,110 @@ +/* + * Copyright (c) 2015-2019 Contributors as noted in the AUTHORS file + * + * This file is part of Solo5, a sandboxed execution environment. + * + * Permission to use, copy, modify, and/or distribute this software + * for any purpose with or without fee is hereby granted, provided + * that the above copyright notice and this permission notice appear + * in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL + * WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE + * AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR + * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS + * OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, + * NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN + * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#include "solo5.h" +#include "../../bindings/lib.c" + +static void puts(const char *s) +{ + solo5_console_write(s, strlen(s)); +} + +static inline uint32_t get_reg32(const uint8_t* addr, int reg) { + __asm__ volatile ("" : : : "memory"); + return *((volatile uint32_t*) (addr + reg)); +} + +static inline void set_reg32(uint8_t* addr, int reg, uint32_t value) { + __asm__ volatile ("" : : : "memory"); + *((volatile uint32_t*) (addr + reg)) = value; +} + +#define LEDCTL 0x00200 + +/* LEDCTL Bit Masks */ +#define IXGBE_LED_IVRT_BASE 0x00000040 +#define IXGBE_LED_BLINK_BASE 0x00000080 +#define IXGBE_LED_MODE_MASK_BASE 0x0000000F +#define IXGBE_LED_OFFSET(_base, _i) ((_base) << (8 * (_i))) +#define IXGBE_LED_MODE_SHIFT(_i) (8*(_i)) +#define IXGBE_LED_IVRT(_i) IXGBE_LED_OFFSET(IXGBE_LED_IVRT_BASE, _i) +#define IXGBE_LED_BLINK(_i) IXGBE_LED_OFFSET(IXGBE_LED_BLINK_BASE, _i) +#define IXGBE_LED_MODE_MASK(_i) IXGBE_LED_OFFSET(IXGBE_LED_MODE_MASK_BASE, _i) + +static bool pci_acquire(void) +{ + struct solo5_pci_info info; + + if (solo5_pci_acquire("pci0", &info) != SOLO5_R_OK) { + puts("Could not acquire 'pci0' device\n"); + return false; + } + + uint8_t *buffer = info.bar0; + + uint32_t masked = get_reg32(buffer, LEDCTL) & ~IXGBE_LED_MODE_MASK(1); + + for (int i = 0; i < 5; i++) { + puts("led on\n"); + set_reg32(buffer, LEDCTL, masked | (0xE << IXGBE_LED_MODE_SHIFT(1))); + solo5_yield(solo5_clock_monotonic() + (500 * 1000 * 1000), NULL); + puts("led off\n"); + set_reg32(buffer, LEDCTL, masked | (0xF << IXGBE_LED_MODE_SHIFT(1))); + solo5_yield(solo5_clock_monotonic() + (500 * 1000 * 1000), NULL); + } + + size_t size; + uint8_t *dma; + if (solo5_dma_acquire(&dma, &size) != SOLO5_R_OK) { + puts("Could not acquire dma memory\n"); + return false; + } + + if (size != 4194304) { + puts("WRONG DMA SIZE\n"); + return false; + } + + if (((uint64_t) dma) != (1ULL << 40)) { + puts("WRONG DMA ADDRESS\n"); + return false; + } + + for (size_t i = 0; i < size; i++) + dma[i] = 0; + + return true; +} + +int solo5_app_main(const struct solo5_start_info *si) +{ + puts("\n**** Solo5 standalone test_pci ****\n\n"); + + puts(si->cmdline); + + if (pci_acquire()) { + puts("SUCCESS\n"); + return SOLO5_EXIT_SUCCESS; + } + else { + puts("FAILURE\n"); + return SOLO5_EXIT_FAILURE; + } +} diff --git a/tests/test_quiet/manifest.json b/tests/test_quiet/manifest.json index 1100fc5b..a6c72956 100644 --- a/tests/test_quiet/manifest.json +++ b/tests/test_quiet/manifest.json @@ -1,5 +1,6 @@ { "type": "solo5.manifest", "version": 1, + "dma_size": 0, "devices": [ ] } diff --git a/tests/test_seccomp/manifest.json b/tests/test_seccomp/manifest.json index 1100fc5b..a6c72956 100644 --- a/tests/test_seccomp/manifest.json +++ b/tests/test_seccomp/manifest.json @@ -1,5 +1,6 @@ { "type": "solo5.manifest", "version": 1, + "dma_size": 0, "devices": [ ] } diff --git a/tests/test_ssp/manifest.json b/tests/test_ssp/manifest.json index 1100fc5b..a6c72956 100644 --- a/tests/test_ssp/manifest.json +++ b/tests/test_ssp/manifest.json @@ -1,5 +1,6 @@ { "type": "solo5.manifest", "version": 1, + "dma_size": 0, "devices": [ ] } diff --git a/tests/test_time/manifest.json b/tests/test_time/manifest.json index 1100fc5b..a6c72956 100644 --- a/tests/test_time/manifest.json +++ b/tests/test_time/manifest.json @@ -1,5 +1,6 @@ { "type": "solo5.manifest", "version": 1, + "dma_size": 0, "devices": [ ] } diff --git a/tests/test_tls/manifest.json b/tests/test_tls/manifest.json index 1100fc5b..a6c72956 100644 --- a/tests/test_tls/manifest.json +++ b/tests/test_tls/manifest.json @@ -1,5 +1,6 @@ { "type": "solo5.manifest", "version": 1, + "dma_size": 0, "devices": [ ] } diff --git a/tests/test_wnox/manifest.json b/tests/test_wnox/manifest.json index 1100fc5b..a6c72956 100644 --- a/tests/test_wnox/manifest.json +++ b/tests/test_wnox/manifest.json @@ -1,5 +1,6 @@ { "type": "solo5.manifest", "version": 1, + "dma_size": 0, "devices": [ ] } diff --git a/tests/test_xnow/manifest.json b/tests/test_xnow/manifest.json index 1100fc5b..a6c72956 100644 --- a/tests/test_xnow/manifest.json +++ b/tests/test_xnow/manifest.json @@ -1,5 +1,6 @@ { "type": "solo5.manifest", "version": 1, + "dma_size": 0, "devices": [ ] } diff --git a/tests/test_zeropage/manifest.json b/tests/test_zeropage/manifest.json index 1100fc5b..a6c72956 100644 --- a/tests/test_zeropage/manifest.json +++ b/tests/test_zeropage/manifest.json @@ -1,5 +1,6 @@ { "type": "solo5.manifest", "version": 1, + "dma_size": 0, "devices": [ ] }