diff --git a/serde/src/main/java/edu/berkeley/cs/succinct/DataType.java b/serde/src/main/java/edu/berkeley/cs/succinct/DataType.java index 2250d38..dee2342 100644 --- a/serde/src/main/java/edu/berkeley/cs/succinct/DataType.java +++ b/serde/src/main/java/edu/berkeley/cs/succinct/DataType.java @@ -9,7 +9,11 @@ public enum DataType { LONG(4), FLOAT(5), DOUBLE(6), - STRING(7); + STRING(7), + STRINGARRAY(8), + LONGARRAY(9), + BYTEARRAY(10), + BOOLARRAY(11); private final int order; diff --git a/serde/src/main/java/edu/berkeley/cs/succinct/block/json/JsonBlockSerializer.java b/serde/src/main/java/edu/berkeley/cs/succinct/block/json/JsonBlockSerializer.java index f82d028..7303d5f 100644 --- a/serde/src/main/java/edu/berkeley/cs/succinct/block/json/JsonBlockSerializer.java +++ b/serde/src/main/java/edu/berkeley/cs/succinct/block/json/JsonBlockSerializer.java @@ -2,6 +2,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.databind.node.ValueNode; import edu.berkeley.cs.succinct.DataType; @@ -67,28 +68,48 @@ private void flattenJsonTree(String currentPath, JsonNode jsonNode, ByteArrayOut flattenJsonTree(pathPrefix + entry.getKey(), entry.getValue(), out); } } else if (jsonNode.isArray()) { - throw new SerializationException("Arrays in JSON are not supported yet."); + ArrayNode arrayNode = (ArrayNode) jsonNode; + DataType primitiveArrayType = getNodeType(arrayNode.get(0)); + DataType jsonArrayType = getArrayNodeType(primitiveArrayType); + for (int i = 0; i < arrayNode.size(); i++) { + if (getNodeType(arrayNode.get(i)) != primitiveArrayType) + throw new SerializationException("Multi Type Arrays in JSON are not supported yet."); + else if (!arrayNode.get(i).isValueNode()) + throw new SerializationException("Non primitive types in Arrays in JSON are not supported yet."); + } + for (int i = 0; i < arrayNode.size(); i++) { + ValueNode valueNode = (ValueNode) arrayNode.get(i); + writeJsonTree(currentPath, valueNode, jsonArrayType, out, true); + } } else if (jsonNode.isValueNode()) { ValueNode valueNode = (ValueNode) jsonNode; - if (!fieldMapping.containsField(currentPath)) { - fieldMapping.put(currentPath, delimiters[currentDelimiterIdx++], getNodeType(jsonNode)); - } else { - DataType existingType = fieldMapping.getDataType(currentPath); - DataType newType = getNodeType(valueNode); - if (existingType != newType) { - DataType encapsulatingType = DataType.encapsulatingType(existingType, newType); + writeJsonTree(currentPath, valueNode, getNodeType(jsonNode), out, false); + } + } + + private void writeJsonTree(String currentPath, ValueNode valueNode, DataType fieldMappingType, ByteArrayOutputStream out, + boolean isArray) throws SerializationException { + if (!fieldMapping.containsField(currentPath)) { + fieldMapping.put(currentPath, delimiters[currentDelimiterIdx++], fieldMappingType); + } else { + DataType existingType = fieldMapping.getDataType(currentPath); + DataType newType = getNodeType(valueNode); + if (existingType != newType) { + DataType encapsulatingType = DataType.encapsulatingType(existingType, newType); + if(isArray) + fieldMapping.updateType(currentPath, getArrayNodeType(encapsulatingType)); + else fieldMapping.updateType(currentPath, encapsulatingType); - } - } - try { - byte fieldByte = fieldMapping.getDelimiter(currentPath); - out.write(fieldByte); - out.write(valueNode.asText().getBytes()); - out.write(fieldByte); - } catch (IOException e) { - throw new SerializationException(e.getMessage()); } } + try { + byte fieldByte = fieldMapping.getDelimiter(currentPath); + out.write(fieldByte); + out.write(valueNode.asText().getBytes()); + out.write(fieldByte); + } catch (IOException e) { + throw new SerializationException(e.getMessage()); + } } private DataType getNodeType(JsonNode node) { @@ -108,4 +129,12 @@ private DataType getNodeType(JsonNode node) { throw new UnsupportedOperationException("JSON DataType not supported."); } } + + private DataType getArrayNodeType(DataType fieldType) { + if (fieldType == DataType.STRING) return DataType.STRINGARRAY; + else if (fieldType == DataType.LONG) return DataType.LONGARRAY; + else if (fieldType == DataType.BYTE) return DataType.BYTEARRAY; + else if (fieldType == DataType.BOOLEAN) return DataType.BOOLARRAY; + else throw new UnsupportedOperationException("JSON DataType not supported."); + } } diff --git a/serde/src/main/java/edu/berkeley/cs/succinct/object/deserializer/JsonDeserializer.java b/serde/src/main/java/edu/berkeley/cs/succinct/object/deserializer/JsonDeserializer.java index d797054..ec81c17 100644 --- a/serde/src/main/java/edu/berkeley/cs/succinct/object/deserializer/JsonDeserializer.java +++ b/serde/src/main/java/edu/berkeley/cs/succinct/object/deserializer/JsonDeserializer.java @@ -7,9 +7,7 @@ import edu.berkeley.cs.succinct.PrimitiveDeserializer; import edu.berkeley.cs.succinct.block.json.FieldMapping; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; +import java.util.*; public class JsonDeserializer implements ObjectDeserializer { @@ -42,7 +40,7 @@ private String mapToJsonString(Map jsonMap) throws Deserializati return jsonString; } - private Map deserializeToMap(byte[] data) throws DeserializationException { + private Map deserializeToMap(byte[] data) throws DeserializationException, IllegalArgumentException { Map jsonMap = new HashMap(); int curOffset = 0; @@ -50,15 +48,45 @@ private Map deserializeToMap(byte[] data) throws Deserialization byte curDelimiter = data[curOffset++]; String fieldKey = fieldMapping.getField(curDelimiter); DataType fieldType = fieldMapping.getDataType(fieldKey); - byte[] fieldValueBytes = extractField(data, curOffset, curDelimiter); - Object fieldValue = PrimitiveDeserializer.deserializePrimitive(fieldValueBytes, fieldType); + Object fieldValue; + int totalLength; + if (isArray(fieldType)) { + ArrayList fieldValuesBytes = extractArrayField(data, curOffset, curDelimiter); + final DataType typeinArray = getPrimitiveFromArray(fieldType); + ArrayList resultSet = new ArrayList<>(); + totalLength = 0; + for (byte [] fieldValueBytes : fieldValuesBytes) { + resultSet.add(PrimitiveDeserializer.deserializePrimitive(fieldValueBytes, typeinArray)); + totalLength += fieldValueBytes.length + 2; // Skip the field data, and the delimiter at end and start of next field + } + fieldValue = resultSet.toArray(); + totalLength -= 1; // Remove last additional append of delimiter + } + else { + byte[] fieldValueBytes = extractField(data, curOffset, curDelimiter); + fieldValue = PrimitiveDeserializer.deserializePrimitive(fieldValueBytes, fieldType); + totalLength = fieldValueBytes.length + 1; // Skip the field data, and the end delimiter + } add(jsonMap, fieldKey, fieldValue); - curOffset += (fieldValueBytes.length + 1); // Skip the field data, and the end delimiter + curOffset += totalLength; } return jsonMap; } + private DataType getPrimitiveFromArray(DataType fieldType) throws IllegalArgumentException { + if (fieldType == DataType.STRINGARRAY) return DataType.STRING; + else if (fieldType == DataType.LONGARRAY) return DataType.LONG; + else if (fieldType == DataType.BYTEARRAY) return DataType.BYTE; + else if (fieldType == DataType.BOOLARRAY) return DataType.BOOLEAN; + else throw new IllegalArgumentException("Called getPrimitiveFromArray on a non Array DataType"); + } + + private Boolean isArray(DataType fieldType) { + return fieldType == DataType.STRINGARRAY || fieldType == DataType.LONGARRAY || + fieldType == DataType.BYTEARRAY || fieldType == DataType.BOOLARRAY; + } + private void add(Map map, String key, Object value) { if (key.contains(".")) { String[] keySplits = key.split("\\.", 2); @@ -84,4 +112,17 @@ private byte[] extractField(byte[] data, int startOffset, byte delimiter) { return Arrays.copyOfRange(data, startOffset, i); } + private ArrayList extractArrayField(byte[] data, int startOffset, byte delimiter) { + int i = startOffset; + ArrayList res = new ArrayList<>(); + while (true) { + byte[] result = extractField(data, i, delimiter); + res.add(result); + i += result.length + 1; + if(data.length <= i || data[i] != delimiter) break; + else i++; + } + return res; + } + }