Skip to content

Commit

Permalink
Add support for Array-Type in JSON
Browse files Browse the repository at this point in the history 
Fixes: #35
  • Loading branch information
@muditsin
muditsin committed Feb 2, 2019
1 parent 260efe6 commit e87df86
Show file tree
Hide file tree
Showing 3 changed files with 99 additions and 25 deletions.
6 changes: 5 additions & 1 deletion serde/src/main/java/edu/berkeley/cs/succinct/DataType.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,11 @@ public enum DataType {
LONG(4),
FLOAT(5),
DOUBLE(6),
STRING(7);
STRING(7),
STRINGARRAY(8),
LONGARRAY(9),
BYTEARRAY(10),
BOOLARRAY(11);

private final int order;

Expand Down
63 changes: 46 additions & 17 deletions serde/src/main/java/edu/berkeley/cs/succinct/block/json/JsonBlockSerializer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.fasterxml.jackson.databind.node.ValueNode;
import edu.berkeley.cs.succinct.DataType;
Expand Down Expand Up @@ -67,28 +68,48 @@ private void flattenJsonTree(String currentPath, JsonNode jsonNode, ByteArrayOut
flattenJsonTree(pathPrefix + entry.getKey(), entry.getValue(), out);
}
} else if (jsonNode.isArray()) {
throw new SerializationException("Arrays in JSON are not supported yet.");
ArrayNode arrayNode = (ArrayNode) jsonNode;
DataType primitiveArrayType = getNodeType(arrayNode.get(0));
DataType jsonArrayType = getArrayNodeType(primitiveArrayType);
for (int i = 0; i < arrayNode.size(); i++) {
if (getNodeType(arrayNode.get(i)) != primitiveArrayType)
throw new SerializationException("Multi Type Arrays in JSON are not supported yet.");
else if (!arrayNode.get(i).isValueNode())
throw new SerializationException("Non primitive types in Arrays in JSON are not supported yet.");
}
for (int i = 0; i < arrayNode.size(); i++) {
ValueNode valueNode = (ValueNode) arrayNode.get(i);
writeJsonTree(currentPath, valueNode, jsonArrayType, out, true);
}
} else if (jsonNode.isValueNode()) {
ValueNode valueNode = (ValueNode) jsonNode;
if (!fieldMapping.containsField(currentPath)) {
fieldMapping.put(currentPath, delimiters[currentDelimiterIdx++], getNodeType(jsonNode));
} else {
DataType existingType = fieldMapping.getDataType(currentPath);
DataType newType = getNodeType(valueNode);
if (existingType != newType) {
DataType encapsulatingType = DataType.encapsulatingType(existingType, newType);
writeJsonTree(currentPath, valueNode, getNodeType(jsonNode), out, false);
}
}

private void writeJsonTree(String currentPath, ValueNode valueNode, DataType fieldMappingType, ByteArrayOutputStream out,
boolean isArray) throws SerializationException {
if (!fieldMapping.containsField(currentPath)) {
fieldMapping.put(currentPath, delimiters[currentDelimiterIdx++], fieldMappingType);
} else {
DataType existingType = fieldMapping.getDataType(currentPath);
DataType newType = getNodeType(valueNode);
if (existingType != newType) {
DataType encapsulatingType = DataType.encapsulatingType(existingType, newType);
if(isArray)
fieldMapping.updateType(currentPath, getArrayNodeType(encapsulatingType));
else
fieldMapping.updateType(currentPath, encapsulatingType);
}
}
try {
byte fieldByte = fieldMapping.getDelimiter(currentPath);
out.write(fieldByte);
out.write(valueNode.asText().getBytes());
out.write(fieldByte);
} catch (IOException e) {
throw new SerializationException(e.getMessage());
}
}
try {
byte fieldByte = fieldMapping.getDelimiter(currentPath);
out.write(fieldByte);
out.write(valueNode.asText().getBytes());
out.write(fieldByte);
} catch (IOException e) {
throw new SerializationException(e.getMessage());
}
}

private DataType getNodeType(JsonNode node) {
Expand All @@ -108,4 +129,12 @@ private DataType getNodeType(JsonNode node) {
throw new UnsupportedOperationException("JSON DataType not supported.");
}
}

private DataType getArrayNodeType(DataType fieldType) {
if (fieldType == DataType.STRING) return DataType.STRINGARRAY;
else if (fieldType == DataType.LONG) return DataType.LONGARRAY;
else if (fieldType == DataType.BYTE) return DataType.BYTEARRAY;
else if (fieldType == DataType.BOOLEAN) return DataType.BOOLARRAY;
else throw new UnsupportedOperationException("JSON DataType not supported.");
}
}
55 changes: 48 additions & 7 deletions serde/src/main/java/edu/berkeley/cs/succinct/object/deserializer/JsonDeserializer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,7 @@
import edu.berkeley.cs.succinct.PrimitiveDeserializer;
import edu.berkeley.cs.succinct.block.json.FieldMapping;

import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.*;

public class JsonDeserializer implements ObjectDeserializer<String> {

Expand Down Expand Up @@ -42,23 +40,53 @@ private String mapToJsonString(Map<String, Object> jsonMap) throws Deserializati
return jsonString;
}

private Map<String, Object> deserializeToMap(byte[] data) throws DeserializationException {
private Map<String, Object> deserializeToMap(byte[] data) throws DeserializationException, IllegalArgumentException {
Map<String, Object> jsonMap = new HashMap<String, Object>();

int curOffset = 0;
while (curOffset < data.length) {
byte curDelimiter = data[curOffset++];
String fieldKey = fieldMapping.getField(curDelimiter);
DataType fieldType = fieldMapping.getDataType(fieldKey);
byte[] fieldValueBytes = extractField(data, curOffset, curDelimiter);
Object fieldValue = PrimitiveDeserializer.deserializePrimitive(fieldValueBytes, fieldType);
Object fieldValue;
int totalLength;
if (isArray(fieldType)) {
ArrayList<byte []> fieldValuesBytes = extractArrayField(data, curOffset, curDelimiter);
final DataType typeinArray = getPrimitiveFromArray(fieldType);
ArrayList<Object> resultSet = new ArrayList<>();
totalLength = 0;
for (byte [] fieldValueBytes : fieldValuesBytes) {
resultSet.add(PrimitiveDeserializer.deserializePrimitive(fieldValueBytes, typeinArray));
totalLength += fieldValueBytes.length + 2; // Skip the field data, and the delimiter at end and start of next field
}
fieldValue = resultSet.toArray();
totalLength -= 1; // Remove last additional append of delimiter
}
else {
byte[] fieldValueBytes = extractField(data, curOffset, curDelimiter);
fieldValue = PrimitiveDeserializer.deserializePrimitive(fieldValueBytes, fieldType);
totalLength = fieldValueBytes.length + 1; // Skip the field data, and the end delimiter
}
add(jsonMap, fieldKey, fieldValue);
curOffset += (fieldValueBytes.length + 1); // Skip the field data, and the end delimiter
curOffset += totalLength;
}

return jsonMap;
}

private DataType getPrimitiveFromArray(DataType fieldType) throws IllegalArgumentException {
if (fieldType == DataType.STRINGARRAY) return DataType.STRING;
else if (fieldType == DataType.LONGARRAY) return DataType.LONG;
else if (fieldType == DataType.BYTEARRAY) return DataType.BYTE;
else if (fieldType == DataType.BOOLARRAY) return DataType.BOOLEAN;
else throw new IllegalArgumentException("Called getPrimitiveFromArray on a non Array DataType");
}

private Boolean isArray(DataType fieldType) {
return fieldType == DataType.STRINGARRAY || fieldType == DataType.LONGARRAY ||
fieldType == DataType.BYTEARRAY || fieldType == DataType.BOOLARRAY;
}

private void add(Map<String, Object> map, String key, Object value) {
if (key.contains(".")) {
String[] keySplits = key.split("\\.", 2);
Expand All @@ -84,4 +112,17 @@ private byte[] extractField(byte[] data, int startOffset, byte delimiter) {
return Arrays.copyOfRange(data, startOffset, i);
}

private ArrayList<byte []> extractArrayField(byte[] data, int startOffset, byte delimiter) {
int i = startOffset;
ArrayList<byte []> res = new ArrayList<>();
while (true) {
byte[] result = extractField(data, i, delimiter)
res.add(result);
i += result.length + 1;
if(data.length <= i || data[i] != delimiter) break;
else i++;
}
return res;
}

}

0 comments on commit e87df86

Please sign in to comment.