Content descriptors integration

Makefile

@@ -26,27 +26,28 @@ include ports.mk
 PWD     := $(shell pwd)
 BUILD   ?= build
 ARCH    ?= $(HOSTARCH)
-OBJ     := $(BUILD)/$(ARCH)/obj
-BIN     := $(BUILD)/$(ARCH)/bin
-LIB		:= $(BUILD)/$(ARCH)/lib
-TESTS   := $(BUILD)/$(ARCH)/tests
-TMPBIN	:= $(BUILD)/$(ARCH)/tmp
-INC     := $(BUILD)/$(ARCH)/include
+OBJ     := $(BUILD)/$(ARCH)-$(OSNAME)/obj
+BIN     := $(BUILD)/$(ARCH)-$(OSNAME)/bin
+LIB		:= $(BUILD)/$(ARCH)-$(OSNAME)/lib
+TESTS   := $(BUILD)/$(ARCH)-$(OSNAME)/tests
+TMPBIN	:= $(BUILD)/$(ARCH)-$(OSNAME)/tmp
+INC     := $(BUILD)/$(ARCH)-$(OSNAME)/include
 SRC     := .
-TMP     ?= $(PWD)/$(BUILD)/$(ARCH)/tmp
+TMP     ?= $(PWD)/$(BUILD)/$(ARCH)-$(OSNAME)/tmp
 
 # These are for cross-compilation, where binaries used in the build need
 # be be built for the host.
 HOSTARCH ?= $(ARCH)
-HOSTBIN ?= $(BUILD)/$(HOSTARCH)/bin
-HOSTLIB ?= $(BUILD)/$(HOSTARCH)/lib
-HOSTINC ?= $(BUILD)/$(HOSTARCH)/include
+HOSTOSNAME ?= $(OSNAME)
+HOSTBIN ?= $(BUILD)/$(HOSTARCH)-$(HOSTOSNAME)/bin
+HOSTLIB ?= $(BUILD)/$(HOSTARCH)-$(HOSTOSNAME)/lib
+HOSTINC ?= $(BUILD)/$(HOSTARCH)-$(HOSTOSNAME)/include
 
 TEST_TMP := $(TESTS)
 # Vars for configuration files or files that live outside bin and lib
 ALTROOT := $(BUILD)/$(ARCH)/altroot
 ETC     := $(ALTROOT)/etc
-PLUGINS := $(BUILD)/$(ARCH)/mldb_plugins
+PLUGINS := $(BUILD)/$(ARCH)-$(OSNAME)/mldb_plugins
 
 JML_BUILD := mldb/jml-build
 INCLUDE := -Imldb

block/content_descriptor.cc

@@ -460,13 +460,22 @@ getStream(const std::map<Utf8String, Any> & options) const
     //cerr << "url = " << descriptor.getUrlStringUtf8() << " compression = "
     //     << compression << " mapped = " << isMapped << endl;
 
-    if (isMapped) {
+    while (isMapped) {  // actually an if, but now we can break out
         // Just get one single big block
         auto contentHandler = getContent(descriptor);
 
         struct Vals {
             FsObjectInfo info;
             FrozenMemoryRegion mem;
+
+#if 0
+            ~Vals()
+            {
+                cerr << endl << endl << endl;
+                cerr << "NO MORE MAPPING VALS" << endl;
+                cerr << endl << endl << endl;
+            }
+#endif
         };
 
         auto vals = std::make_shared<Vals>();
@@ -493,7 +502,10 @@ getStream(const std::map<Utf8String, Any> & options) const
                                    vals->mem.length());
 
             if (outputSize < 0) {
-                throw Exception("decompressed size unknown");
+                if (outputSize == Decompressor::LENGTH_UNKNOWN)
+                    break;  // do as an istream as we can't run the splitting algorithm
+
+                throw Exception("decompressed size unknown: %i", (int)outputSize);
             }
 
             static MemorySerializer serializer;
@@ -546,10 +558,8 @@ getStream(const std::map<Utf8String, Any> & options) const
         filter_istream stream(handler, descriptor.getUrlStringUtf8(), options2);
         return stream;
     }
-    else {
-        // Not mapped.  We go block by block.
-    }
-
+
+    // Not mapped.  We go block by block.
     filter_istream result(descriptor.getUrlStringUtf8(), options2);
     return result;
 }

block/testing/content_descriptor_test.cc

@@ -147,3 +147,18 @@ BOOST_AUTO_TEST_CASE( test_compressed_random_access )
 {
 
 }
+
+BOOST_AUTO_TEST_CASE( test_parallel_decompress_zstd )
+{
+    string input_file = "mldb_test_data/Books_5.json.zstd";
+    ContentDescriptor descriptor = jsonDecode<ContentDescriptor>("file://" + input_file);
+    std::shared_ptr<ContentHandler> handler = getDecompressedContent(descriptor);
+
+    auto onBlock = [&] (size_t blockNum, uint64_t blockOffset,
+                        FrozenMemoryRegion block)
+    {
+        return true;
+    };
+
+    handler->forEachBlockParallel(0, 1024 * 1024 /* requested block size */, 1 /* maxParallelism */, onBlock);
+}

jml-build/os/Darwin.mk

@@ -11,7 +11,7 @@ READLINK:=readlink
 linker_rpath=
 SO_EXTENSION:=.dylib
 
-VIRTUALENV ?= virtualenv
+VIRTUALENV ?= virtualenv-$(ARCH)-$(OSNAME)-$(PYTHON_VERSION)
 PYTHON ?= $(VIRTUALENV)/bin/python
 PIP ?= $(VIRTUALENV)/bin/pip
 PYTHON_DEPENDENCIES_PRE_CMD ?= $(PIP) install -U pip==21.2.3

jml-build/os/Linux.mk

@@ -15,7 +15,7 @@ DIST_CODENAME:=$(shell lsb_release -sc)
 MACHINE_NAME:=$(shell uname -n)
 READLINK:=readlink -f
 
-VIRTUALENV ?= virtualenv
+VIRTUALENV ?= virtualenv-$(ARCH)-$(OSNAME)-$(PYTHON_VERSION)
 PYTHON ?= $(VIRTUALENV)/bin/python
 PIP ?= $(VIRTUALENV)/bin/pip
 PYTHON_DEPENDENCIES_PRE_CMD ?= $(PIP) install -U pip==21.1.3

makefile-main-plugin.mk

@@ -2,6 +2,7 @@
 
 # Empty default suffixes to speed up initialization
 .SUFFIXES:
+OSNAME:=$(shell uname -s)
 
 toolchain ?= gcc
 PYTHON_ENABLED:=1
@@ -35,27 +36,28 @@ default: all
 
 BUILD   ?= build
 ARCH    ?= $(shell uname -m)
-OBJ     := $(BUILD)/$(ARCH)/obj
-BIN     := $(BUILD)/$(ARCH)/bin
+OBJ     := $(BUILD)/$(ARCH)-$(OSNAME)/obj
+BIN     := $(BUILD)/$(ARCH)-$(OSNAME)/bin
 LIB	:= $(BUILD)/$(ARCH)/lib
-TESTS   := $(BUILD)/$(ARCH)/tests
-TMPBIN	:= $(BUILD)/$(ARCH)/tmp
-INC     := $(BUILD)/$(ARCH)/include
+TESTS   := $(BUILD)/$(ARCH)-$(OSNAME)/tests
+TMPBIN	:= $(BUILD)/$(ARCH)-$(OSNAME)/tmp
+INC     := $(BUILD)/$(ARCH)-$(OSNAME)/include
 SRC     := .
-TMP     ?= $(BUILD)/$(ARCH)/tmp
+TMP     ?= $(BUILD)/$(ARCH)-$(OSNAME)/tmp
 
 # These are for cross-compilation, where binaries used in the build need
 # be be built for the host.
 HOSTARCH ?= $(ARCH)
-HOSTBIN ?= $(BUILD)/$(HOSTARCH)/bin
-HOSTLIB ?= $(BUILD)/$(HOSTARCH)/lib
-HOSTINC ?= $(BUILD)/$(HOSTARCH)/include
+HOSTOSNAME ?= $(OSNAME)
+HOSTBIN ?= $(BUILD)/$(HOSTARCH)-$(HOSTOSNAME)/bin
+HOSTLIB ?= $(BUILD)/$(HOSTARCH)-$(HOSTOSNAME)/lib
+HOSTINC ?= $(BUILD)/$(HOSTARCH)-$(HOSTOSNAME)/include
 
 TEST_TMP := $(TESTS)
 # Vars for configuration files or files that live outside bin and lib
 ALTROOT := $(BUILD)/$(ARCH)/altroot
 ETC     := $(ALTROOT)/etc
-PLUGINS := $(BUILD)/$(ARCH)/mldb_plugins
+PLUGINS := $(BUILD)/$(ARCH)-$(OSNAME)/mldb_plugins
 
 JML_BUILD := mldb/jml-build
 INCLUDE := -I. -Imldb

plugins/jml/randomforest.h

@@ -321,11 +321,16 @@ struct PartitionData {
 
         bool ordinal = features.at(featureToSplitOn).ordinal;
 
+        // Density of example numbers within our set of rows.  When this
+        // gets too low, we do essentially random accesses and it kills
+        // our cache performance.  In that case we can re-index to reduce
+        // the size.
         double useRatio = 1.0 * rows.size() / rows.back().exampleNum;
 
         //todo: Re-index when usable data fits inside cache
-        bool reIndex = useRatio < 0.1;
+        bool reIndex = useRatio < 0.25;
         //reIndex = false;
+        //using namespace std;
         //cerr << "useRatio = " << useRatio << endl;
 
         if (!reIndex) {