Skip to content

Commit

Permalink
blob handling is too slow abstratt.cloudfier#235
Browse files Browse the repository at this point in the history
  • Loading branch information
@abstratt
abstratt committed Sep 4, 2017
1 parent a5a53ee commit f9360a3
Show file tree
Hide file tree
Showing 4 changed files with 74 additions and 35 deletions.
8 changes: 1 addition & 7 deletions com.abstratt.kirra.api/src/main/java/com/abstratt/kirra/Blob.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
public class Blob {
private final String token;
private final long contentLength;
private final byte[] contents;
private final String contentType;
private final String originalName;
public String getOriginalName() {
Expand All @@ -18,17 +17,13 @@ public String getToken() {
public long getContentLength() {
return contentLength;
}
public byte[] getContents() {
return contents;
}
public String getContentType() {
return contentType;
}
public Blob(String token, long contentLength, byte[] contents, String contentType, String originalName) {
public Blob(String token, long contentLength, String contentType, String originalName) {
super();
this.token = token;
this.contentLength = contentLength;
this.contents = contents;
this.contentType = contentType;
this.originalName = originalName;
}
Expand All @@ -38,7 +33,6 @@ public Map<String, Object> toMap() {
asMap.put("contentType", contentType);
asMap.put("originalName", originalName);
asMap.put("contentLength", contentLength);
asMap.put("contents", contents);
return asMap;
}
}
18 changes: 12 additions & 6 deletions com.abstratt.kirra.api/src/main/java/com/abstratt/kirra/InstanceManagement.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.abstratt.kirra;

import java.io.InputStream;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
Expand Down Expand Up @@ -298,14 +299,19 @@ public default List<Instance> getRelatedInstances(Instance anchorInstance, Strin

public void zap();

default public void deleteBlob(TypeRef entityRef, String objectId, String blobPropertyName, String blobToken) {
default public void deleteBlob(TypeRef entityRef, String objectId, String blobPropertyName, String blob) {
throw new UnsupportedOperationException();
}
default public Blob addBlob(TypeRef entityRef, String objectId, String blobPropertyName, Blob newBlob) {
throw new UnsupportedOperationException();
}
default public Blob getBlob(TypeRef entityRef, String objectId, String blobPropertyName, String blobToken) {
default public Blob getBlob(TypeRef entityRef, String objectId, String blobPropertyName, String token) {
throw new UnsupportedOperationException();
}
default public InputStream readBlob(TypeRef entityRef, String objectId, String blobPropertyName, String token) {
throw new UnsupportedOperationException();
}

default public Blob writeBlob(TypeRef entityRef, String objectId, String blobPropertyName, String token, InputStream contents) {
throw new UnsupportedOperationException();
}
default public Blob createBlob(TypeRef entityRef, String objectId, String blobPropertyName, String contentType, String originalName) {
throw new UnsupportedOperationException();
}
}
57 changes: 46 additions & 11 deletions com.abstratt.kirra.populator/src/com/abstratt/kirra/populator/DataPopulator.java
View file
Original file line number Diff line number Diff line change
@@ -1,21 +1,20 @@
package com.abstratt.kirra.populator;

import java.io.BufferedInputStream;
import java.io.BufferedInputStream;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.MalformedURLException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.nio.charset.StandardCharsets;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Base64;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashMap;
Expand All @@ -29,6 +28,7 @@
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;

import com.abstratt.kirra.Blob;
import com.abstratt.kirra.DataElement;
import com.abstratt.kirra.Entity;
import com.abstratt.kirra.Instance;
Expand All @@ -45,6 +45,7 @@
public class DataPopulator {
class Loader {
private Map<String, Map<String, List<String>>> instances;
private List<BlobAction> blobActions;
private Map<Instance, List<LinkingAction>> simpleLinkingActions;
private Map<Instance, List<LinkingAction>> multipleLinkingActions;

Expand Down Expand Up @@ -94,6 +95,7 @@ private Instance processInstance(Entity entity, JsonNode instanceNode) {
validRelationships.put(relationship.getName(), relationship);
List<LinkingAction> simpleInstanceLinkingActions = new LinkedList<>();
List<LinkingAction> multipleInstanceLinkingActions = new LinkedList<>();
blobActions = new LinkedList<>();
for (Iterator<String> propertyNames = instanceNode.fieldNames(); propertyNames.hasNext();) {
String slotName = propertyNames.next();
if (validProperties.containsKey(slotName))
Expand All @@ -109,6 +111,12 @@ else if (validRelationships.containsKey(slotName)) {
Instance created = repository.createInstance(newInstance);
simpleLinkingActions.put(created, simpleInstanceLinkingActions);
multipleLinkingActions.put(created, multipleInstanceLinkingActions);
if (!blobActions.isEmpty()) {
blobActions.forEach(action -> action.accept(created));
repository.updateInstance(created);
blobActions.clear();
}

getEntityInstances(entity.getEntityNamespace(), entity.getName()).add(created.getObjectId());
return created;
}
Expand Down Expand Up @@ -222,13 +230,14 @@ private void setProperty(Instance newInstance, JsonNode propertyValue, DataEleme
case START_OBJECT:
if (property.getTypeRef().getKind() == TypeKind.Blob) {
Map<String, Object> asMap = new LinkedHashMap<>();
asMap.put("token", propertyValue.get("token").asText());
asMap.put("contentType", propertyValue.get("contentType").asText());
asMap.put("originalName", Optional.ofNullable(propertyValue.get("originalName")).map(it -> it.asText()).orElse(null));
asMap.put("contentLength", propertyValue.get("contentLength").asInt());
asMap.put("contents", propertyValue.get("contents").asText());
String contentType = propertyValue.get("contentType").asText();
String originalName = Optional.ofNullable(propertyValue.get("originalName")).map(it -> it.asText()).orElse(null);
asMap.put("contentType", contentType);
asMap.put("originalName", originalName);
value = asMap;
}
blobActions.add(new BlobAction(property.getName(), propertyValue.get("contents").asText(), contentType, propertyValue.get("originalName").asText()));
}

break;
}
newInstance.setValue(property.getName(), value);
Expand All @@ -247,7 +256,11 @@ private void setSingleRelationship(Instance newInstance, JsonNode jsonNode, Rela
newInstance.setSingleRelated(relationship.getName(), related);
}

class LinkingAction implements Consumer<Instance> {
abstract class DelayedAction implements Consumer<Instance> {

}

class LinkingAction extends DelayedAction {
private Relationship relationship;
private JsonNode instanceNode;
private String slotName;
Expand All @@ -261,6 +274,27 @@ public void accept(Instance instance) {
setRelationship(instance, instanceNode.get(slotName), relationship);
}
}

class BlobAction extends DelayedAction {
private String slotName;
private String contents;
private String originalName;
private String contentType;
BlobAction(String slotName, String contents, String contentType, String originalName) {
this.slotName = slotName;
this.contents = contents;
this.contentType = contentType;
this.originalName = originalName;
}
@Override
public void accept(Instance instance) {
Blob blob = repository.createBlob(instance.getTypeRef(), instance.getObjectId(), slotName, contentType, originalName);
instance.setValue(slotName, blob.toMap());
repository.updateInstance(instance);
byte[] asBytes = Base64.getDecoder().decode(contents);
repository.writeBlob(instance.getTypeRef(), instance.getObjectId(), slotName, blob.getToken(), new ByteArrayInputStream(asBytes));
}
}

}

Expand All @@ -285,6 +319,7 @@ public File getDataFile() {
}

public int populate() {
repository.zap();
File dataFile = getDataFile();
InputStream in = null;
try {
Expand Down
26 changes: 15 additions & 11 deletions ....rest.resources/src/main/java/com/abstratt/kirra/rest/resources/InstanceBlobResource.java
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
package com.abstratt.kirra.rest.resources;

import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.file.Files;
import java.util.List;
import java.util.UUID;

Expand All @@ -29,6 +31,7 @@
import com.abstratt.kirra.Property;
import com.abstratt.kirra.TypeRef;
import com.abstratt.kirra.TypeRef.TypeKind;
import com.abstratt.kirra.rest.common.CommonHelper;
import com.abstratt.kirra.rest.common.KirraContext;
import com.abstratt.kirra.rest.common.KirraContext.Upload;
import com.abstratt.kirra.rest.common.Paths;
Expand Down Expand Up @@ -66,11 +69,11 @@ public Response downloadBlob(@PathParam("entityName") String entityName, @PathPa
ResourceHelper.ensure(instance != null, "Instance not found", Status.NOT_FOUND);
Blob blob = instanceManagement.getBlob(entityRef, objectId, blobPropertyName, blobToken);
ResourceHelper.ensure(blob != null, "Blob not found", Status.NOT_FOUND);
ByteArrayInputStream blobContents = new ByteArrayInputStream(blob.getContents());
InputStream blobContents = instanceManagement.readBlob(entityRef, objectId, blobPropertyName, blobToken);
StreamingOutput stream = new StreamingOutput() {
@Override
public void write(OutputStream paramOutputStream) throws IOException, WebApplicationException {
IOUtils.copy(blobContents, paramOutputStream);
IOUtils.copyLarge(blobContents, paramOutputStream);
paramOutputStream.flush();
}
};
Expand All @@ -82,7 +85,7 @@ public void write(OutputStream paramOutputStream) throws IOException, WebApplica
@POST
@Produces("application/json")
@Consumes("*/*")
public void addBlob(@PathParam("entityName") String entityName, @PathParam("objectId") String objectId,
public String addBlob(@PathParam("entityName") String entityName, @PathParam("objectId") String objectId,
@PathParam("propertyName") String blobPropertyName) {
TypeRef entityRef = new TypeRef(entityName, TypeRef.TypeKind.Entity);
Entity entity = KirraContext.getSchemaManagement().getEntity(entityRef);
Expand All @@ -99,16 +102,17 @@ public void addBlob(@PathParam("entityName") String entityName, @PathParam("obje

uploads.forEach(it -> ResourceHelper.ensure(it.getContents().length() <= ATTACHMENT_LIMIT,
"Attachments limited to " + (ATTACHMENT_LIMIT/1024) + " kb, actual size: " + (it.getContents().length()/1024) + " kb", Status.BAD_REQUEST));
ResourceHelper.ensure(uploads.size() == 1 || blobProperty.isMultiple(), "Too many attachments received",
Status.BAD_REQUEST);

byte[] blobContents;
try {
blobContents = FileUtils.readFileToByteArray(uploads.get(0).getContents());
Blob newBlob = new Blob(UUID.randomUUID().toString(), blobContents.length, blobContents, uploads.get(0).getContentType(), uploads.get(0).getOriginalName());
KirraContext.getInstanceManagement().addBlob(entityRef, objectId, blobPropertyName, newBlob);
InstanceManagement instanceManagement = KirraContext.getInstanceManagement();
try (
InputStream contents = Files.newInputStream(uploads.get(0).getContents().toPath());
) {
Blob newBlob = instanceManagement.createBlob(entityRef, objectId, blobPropertyName, uploads.get(0).getContentType(), uploads.get(0).getOriginalName());
Blob updatedBlob = instanceManagement.writeBlob(entityRef, objectId, blobPropertyName, newBlob.getToken(), contents);
return CommonHelper.buildGson(ResourceHelper.resolve(true, Paths.ENTITIES, entityName, Paths.INSTANCES)).create().toJson(updatedBlob);
} catch (IOException e) {
ResourceHelper.fail(e, "Error processing attachment", Status.INTERNAL_SERVER_ERROR);
return null;
}
}
}

1 comment on commit f9360a3

@abstratt
Copy link
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Meant to mention abstratt/cloudfier#235...

Please sign in to comment.