Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

WIP: Add support for Array-Type in JSON #42

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

WIP: Add support for Array-Type in JSON #42

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion serde/src/main/java/edu/berkeley/cs/succinct/DataType.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,11 @@ public enum DataType {
LONG(4),
FLOAT(5),
DOUBLE(6),
STRING(7);
STRING(7),
STRINGARRAY(8),
LONGARRAY(9),
BYTEARRAY(10),
BOOLARRAY(11);

private final int order;

Expand Down
63 changes: 46 additions & 17 deletions serde/src/main/java/edu/berkeley/cs/succinct/block/json/JsonBlockSerializer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.fasterxml.jackson.databind.node.ValueNode;
import edu.berkeley.cs.succinct.DataType;
Expand Down Expand Up @@ -67,28 +68,48 @@ private void flattenJsonTree(String currentPath, JsonNode jsonNode, ByteArrayOut
flattenJsonTree(pathPrefix + entry.getKey(), entry.getValue(), out);
}
} else if (jsonNode.isArray()) {
throw new SerializationException("Arrays in JSON are not supported yet.");
ArrayNode arrayNode = (ArrayNode) jsonNode;
DataType primitiveArrayType = getNodeType(arrayNode.get(0));
DataType jsonArrayType = getArrayNodeType(primitiveArrayType);
for (int i = 0; i < arrayNode.size(); i++) {
if (getNodeType(arrayNode.get(i)) != primitiveArrayType)
throw new SerializationException("Multi Type Arrays in JSON are not supported yet.");
else if (!arrayNode.get(i).isValueNode())
throw new SerializationException("Non primitive types in Arrays in JSON are not supported yet.");
}
for (int i = 0; i < arrayNode.size(); i++) {
ValueNode valueNode = (ValueNode) arrayNode.get(i);
writeJsonTree(currentPath, valueNode, jsonArrayType, out, true);
}
} else if (jsonNode.isValueNode()) {
ValueNode valueNode = (ValueNode) jsonNode;
if (!fieldMapping.containsField(currentPath)) {
fieldMapping.put(currentPath, delimiters[currentDelimiterIdx++], getNodeType(jsonNode));
} else {
DataType existingType = fieldMapping.getDataType(currentPath);
DataType newType = getNodeType(valueNode);
if (existingType != newType) {
DataType encapsulatingType = DataType.encapsulatingType(existingType, newType);
writeJsonTree(currentPath, valueNode, getNodeType(jsonNode), out, false);
}
}

private void writeJsonTree(String currentPath, ValueNode valueNode, DataType fieldMappingType, ByteArrayOutputStream out,
boolean isArray) throws SerializationException {
if (!fieldMapping.containsField(currentPath)) {
fieldMapping.put(currentPath, delimiters[currentDelimiterIdx++], fieldMappingType);
} else {
DataType existingType = fieldMapping.getDataType(currentPath);
DataType newType = getNodeType(valueNode);
if (existingType != newType) {
DataType encapsulatingType = DataType.encapsulatingType(existingType, newType);
if(isArray)
fieldMapping.updateType(currentPath, getArrayNodeType(encapsulatingType));
else
fieldMapping.updateType(currentPath, encapsulatingType);
}
}
try {
byte fieldByte = fieldMapping.getDelimiter(currentPath);
out.write(fieldByte);
out.write(valueNode.asText().getBytes());
out.write(fieldByte);
} catch (IOException e) {
throw new SerializationException(e.getMessage());
}
}
try {
byte fieldByte = fieldMapping.getDelimiter(currentPath);
out.write(fieldByte);
out.write(valueNode.asText().getBytes());
out.write(fieldByte);
} catch (IOException e) {
throw new SerializationException(e.getMessage());
}
}

private DataType getNodeType(JsonNode node) {
Expand All @@ -108,4 +129,12 @@ private DataType getNodeType(JsonNode node) {
throw new UnsupportedOperationException("JSON DataType not supported.");
}
}

private DataType getArrayNodeType(DataType fieldType) {
if (fieldType == DataType.STRING) return DataType.STRINGARRAY;
else if (fieldType == DataType.LONG) return DataType.LONGARRAY;
else if (fieldType == DataType.BYTE) return DataType.BYTEARRAY;
else if (fieldType == DataType.BOOLEAN) return DataType.BOOLARRAY;
else throw new UnsupportedOperationException("JSON DataType not supported.");
}
}
55 changes: 48 additions & 7 deletions serde/src/main/java/edu/berkeley/cs/succinct/object/deserializer/JsonDeserializer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,7 @@
import edu.berkeley.cs.succinct.PrimitiveDeserializer;
import edu.berkeley.cs.succinct.block.json.FieldMapping;

import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.*;

public class JsonDeserializer implements ObjectDeserializer<String> {

Expand Down Expand Up @@ -42,23 +40,53 @@ private String mapToJsonString(Map<String, Object> jsonMap) throws Deserializati
return jsonString;
}

private Map<String, Object> deserializeToMap(byte[] data) throws DeserializationException {
private Map<String, Object> deserializeToMap(byte[] data) throws DeserializationException, IllegalArgumentException {
Map<String, Object> jsonMap = new HashMap<String, Object>();

int curOffset = 0;
while (curOffset < data.length) {
byte curDelimiter = data[curOffset++];
String fieldKey = fieldMapping.getField(curDelimiter);
DataType fieldType = fieldMapping.getDataType(fieldKey);
byte[] fieldValueBytes = extractField(data, curOffset, curDelimiter);
Object fieldValue = PrimitiveDeserializer.deserializePrimitive(fieldValueBytes, fieldType);
Object fieldValue;
int totalLength;
if (isArray(fieldType)) {
ArrayList<byte []> fieldValuesBytes = extractArrayField(data, curOffset, curDelimiter);
final DataType typeinArray = getPrimitiveFromArray(fieldType);
ArrayList<Object> resultSet = new ArrayList<>();
totalLength = 0;
for (byte [] fieldValueBytes : fieldValuesBytes) {
resultSet.add(PrimitiveDeserializer.deserializePrimitive(fieldValueBytes, typeinArray));
totalLength += fieldValueBytes.length + 2; // Skip the field data, and the delimiter at end and start of next field
}
fieldValue = resultSet.toArray();
totalLength -= 1; // Remove last additional append of delimiter
}
else {
byte[] fieldValueBytes = extractField(data, curOffset, curDelimiter);
fieldValue = PrimitiveDeserializer.deserializePrimitive(fieldValueBytes, fieldType);
totalLength = fieldValueBytes.length + 1; // Skip the field data, and the end delimiter
}
add(jsonMap, fieldKey, fieldValue);
curOffset += (fieldValueBytes.length + 1); // Skip the field data, and the end delimiter
curOffset += totalLength;
}

return jsonMap;
}

private DataType getPrimitiveFromArray(DataType fieldType) throws IllegalArgumentException {
if (fieldType == DataType.STRINGARRAY) return DataType.STRING;
else if (fieldType == DataType.LONGARRAY) return DataType.LONG;
else if (fieldType == DataType.BYTEARRAY) return DataType.BYTE;
else if (fieldType == DataType.BOOLARRAY) return DataType.BOOLEAN;
else throw new IllegalArgumentException("Called getPrimitiveFromArray on a non Array DataType");
}

private Boolean isArray(DataType fieldType) {
return fieldType == DataType.STRINGARRAY || fieldType == DataType.LONGARRAY ||
fieldType == DataType.BYTEARRAY || fieldType == DataType.BOOLARRAY;
}

private void add(Map<String, Object> map, String key, Object value) {
if (key.contains(".")) {
String[] keySplits = key.split("\\.", 2);
Expand All @@ -84,4 +112,17 @@ private byte[] extractField(byte[] data, int startOffset, byte delimiter) {
return Arrays.copyOfRange(data, startOffset, i);
}

private ArrayList<byte []> extractArrayField(byte[] data, int startOffset, byte delimiter) {
int i = startOffset;
ArrayList<byte []> res = new ArrayList<>();
while (true) {
byte[] result = extractField(data, i, delimiter);
res.add(result);
i += result.length + 1;
if(data.length <= i || data[i] != delimiter) break;
else i++;
}
return res;
}

}