From 2d4772cfa678bab25c8715a263927c287fe824c8 Mon Sep 17 00:00:00 2001 From: Vadim Tsesko Date: Thu, 23 Nov 2023 20:51:01 +0300 Subject: [PATCH 01/20] Add bonus features (#290) --- README.md | 81 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/README.md b/README.md index 8e24e9d13..fe23b7b67 100644 --- a/README.md +++ b/README.md @@ -96,3 +96,84 @@ NB! Под "не блокирует" ниже понимается отсутс ### Report Когда всё будет готово, присылайте pull request в ветку `main` со своей реализацией на review. Не забывайте подтягивать **новые тесты и изменения**, **отвечать на комментарии в PR** и **исправлять замечания**! + +## Бонусные задания (deadline 2023-12-06 23:59:59 MSK) + +После реализации **всех** предыдущих этапов, необходимо выбрать одну из **незанятых** фич, описанных ниже, и предварительно обсудить с преподавателем предполагаемый способ реализации. + +При реализации фичи допускается изменение API `Dao`. + +Добавление тестов, демонстрирующих работоспособность реализации, является **обязательным**. + +### Feedback + +Развёрнутая **конструктивная** обратная связь по курсу: достоинства и недостатки курса, сложность тем, предложения по улучшению. + +### Autocompact + +Регулярный автоматический фоновый compaction. + +### Durability (WAL) + +Гарантирует durability (отсутствие потерь подтверджённых записей/удалений) даже в случае "падения" процесса за счёт того, что операции модификации подтверждаются только после того, как попадут в write-ahead-log. +При инициализации стораджа обнаруженные записи WAL "проигрываются" перед началом обслуживания новых операций. +Устаревшие WAL должны ротироваться, чтобы не занимать лишнее место на диске. + +Существуют разные подходы к реализации `sync()` на диск. + +### Reverse Iterator + +Текущий интерфейс `DAO` позволяет итерироваться по данным только в лексикографическом порядке. +Требуется реализовать возможность корректной итерации в обратном порядке, т.е. добавить метод `descendingGet(from, to)`. + +### Expiration + +Операция `upsert()` должна поддерживать опциональный параметр Time-To-Live (TTL) или время, после которого ячейка должна "пропасть". + +"Протухшие" ячейки не должны отдаваться клиентам и должны вычищаться при compaction. + +### Compression + +Необходимо реализовать блочную компрессию в файлах на диске (используя готовые реализации LZ4, Snappy или zstd) и не забыть про compaction. + +### Streaming + +Необходимо реализовать механизм для записи и чтения значений **больше** чем Java Heap, например, принимая `InputStream` и выдавая `OutputStream` в качестве значения. + +### Atomic Batches + +Необходимо реализовать возможность атомарного применения набора модификаций (upsert или remove). + +Например, можно принимать от клиента список модификаций, писать их в сериализованном виде в **отдельную таблицу** и удалять после успешного применения. +При инициализации стораджа должны "проигрываться" недоприменённые батчи. + +### Transactions + +Необходимо обеспечить возможность транзакционного выполнения набора любых операций (upsert/remove/get). +При возникновении конфликта (любой другой транзакции, работающей с теми же ключами несовместимым способом, т.е. не read/read конфликта) клиент должен получать `ConcurrentModificationException`. +Пример реализации -- [NewSQL](https://habr.com/ru/company/odnoklassniki/blog/417593/). + +### Bloom Filters + +Для каждой таблицы на диске необходимо поддерживать Bloom Filter для содержащихся в ней ключей, чтобы пропускать таблицы, гарантированно не содержащие запрашиваемых ключей. + +Очевидно, что это будет работать только в случае "точечных", а не range-запросов. + +### Column Families + +Поддержка независимых таблиц/keyspace/database/namespace/whatever. + +### Snapshots + +Получение слепка БД на текущий момент времени с возможностью чтения из него вне зависимости от "развития" основной БД. + +Здесь могут помочь hard links. + +### Custom Comparators + +Возможность указания клиентом пользовательского `Comparator` вместо лексикографического. + +### Real 64-bit + +* Поддержка файлов больше 2ГБ +* Модульные тесты для ключей/значений больше 2ГБ From 25d734355184a079ec2b683b88fb973272a5e90e Mon Sep 17 00:00:00 2001 From: Jaroslav Chernyshev <31724693+Jar-Cher@users.noreply.github.com> Date: Sat, 25 Nov 2023 13:17:59 +0300 Subject: [PATCH 02/20] =?UTF-8?q?HW4=20=D0=A7=D0=B5=D1=80=D0=BD=D1=8B?= =?UTF-8?q?=D1=88=D0=B5=D0=B2=20=D0=AF=D1=80=D0=BE=D1=81=D0=BB=D0=B0=D0=B2?= =?UTF-8?q?,=20=D0=9F=D0=BE=D0=BB=D0=B8=D1=82=D0=B5=D1=85,=20=D0=98=D0=92?= =?UTF-8?q?=D0=A2=20(#213)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Vadim Tsesko Co-authored-by: Vladislav Zybkin <47724127+zvladn7@users.noreply.github.com> --- .../chernyshevyaroslav/ChernyshevDao.java | 124 +++++++++++ .../itmo/chernyshevyaroslav/DiskStorage.java | 151 +++++++++++++ .../vk/itmo/chernyshevyaroslav/FileUtils.java | 205 ++++++++++++++++++ .../itmo/chernyshevyaroslav/InMemoryDao.java | 35 --- .../MemorySegmentComparator.java | 21 -- .../chernyshevyaroslav/MergeIterator.java | 136 ++++++++++++ ...Factory.java => ChernyshevDaoFactory.java} | 16 +- 7 files changed, 627 insertions(+), 61 deletions(-) create mode 100644 src/main/java/ru/vk/itmo/chernyshevyaroslav/ChernyshevDao.java create mode 100644 src/main/java/ru/vk/itmo/chernyshevyaroslav/DiskStorage.java create mode 100644 src/main/java/ru/vk/itmo/chernyshevyaroslav/FileUtils.java delete mode 100644 src/main/java/ru/vk/itmo/chernyshevyaroslav/InMemoryDao.java delete mode 100644 src/main/java/ru/vk/itmo/chernyshevyaroslav/MemorySegmentComparator.java create mode 100644 src/main/java/ru/vk/itmo/chernyshevyaroslav/MergeIterator.java rename src/main/java/ru/vk/itmo/test/chernyshevyaroslav/{InMemoryDaoFactory.java => ChernyshevDaoFactory.java} (63%) diff --git a/src/main/java/ru/vk/itmo/chernyshevyaroslav/ChernyshevDao.java b/src/main/java/ru/vk/itmo/chernyshevyaroslav/ChernyshevDao.java new file mode 100644 index 000000000..48da8b030 --- /dev/null +++ b/src/main/java/ru/vk/itmo/chernyshevyaroslav/ChernyshevDao.java @@ -0,0 +1,124 @@ +package ru.vk.itmo.chernyshevyaroslav; + +import ru.vk.itmo.Config; +import ru.vk.itmo.Dao; +import ru.vk.itmo.Entry; + +import java.io.IOException; +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collections; +import java.util.Comparator; +import java.util.Iterator; +import java.util.NavigableMap; +import java.util.concurrent.ConcurrentSkipListMap; + +public class ChernyshevDao implements Dao> { + + private static final String DATA_PATH = "data"; + private final Comparator comparator = ChernyshevDao::compare; + private final NavigableMap> storage = new ConcurrentSkipListMap<>(comparator); + private final Arena arena; + private final DiskStorage diskStorage; + private final Path path; + + public ChernyshevDao(Config config) throws IOException { + this.path = config.basePath().resolve(DATA_PATH); + Files.createDirectories(path); + + arena = Arena.ofShared(); + + this.diskStorage = new DiskStorage(FileUtils.loadOrRecover(path, arena)); + } + + static int compare(MemorySegment memorySegment1, MemorySegment memorySegment2) { + long mismatch = memorySegment1.mismatch(memorySegment2); + if (mismatch == -1) { + return 0; + } + + if (mismatch == memorySegment1.byteSize()) { + return -1; + } + + if (mismatch == memorySegment2.byteSize()) { + return 1; + } + byte b1 = memorySegment1.get(ValueLayout.JAVA_BYTE, mismatch); + byte b2 = memorySegment2.get(ValueLayout.JAVA_BYTE, mismatch); + return Byte.compareUnsigned(b1, b2); + } + + @Override + public Iterator> get(MemorySegment from, MemorySegment to) { + return diskStorage.range(getInMemory(from, to), from, to); + } + + private Iterator> getInMemory(MemorySegment from, MemorySegment to) { + if (from == null && to == null) { + return storage.values().iterator(); + } + if (from == null) { + return storage.headMap(to).values().iterator(); + } + if (to == null) { + return storage.tailMap(from).values().iterator(); + } + return storage.subMap(from, to).values().iterator(); + } + + @Override + public Entry get(MemorySegment key) { + Entry entry = storage.get(key); + if (entry != null) { + if (entry.value() == null) { + return null; + } + return entry; + } + + Iterator> iterator = diskStorage.range(Collections.emptyIterator(), key, null); + + if (!iterator.hasNext()) { + return null; + } + Entry next = iterator.next(); + if (compare(next.key(), key) == 0) { + return next; + } + return null; + } + + @Override + public void close() throws IOException { + if (!arena.scope().isAlive()) { + return; + } + + arena.close(); + + flush(); + } + + @Override + public void flush() throws IOException { + if (!storage.isEmpty()) { + FileUtils.save(path, storage.values(), false); + } + } + + @Override + public void compact() throws IOException { + flush(); + FileUtils.compact(path, this::all); + storage.clear(); + } + + @Override + public void upsert(Entry entry) { + storage.put(entry.key(), entry); + } +} diff --git a/src/main/java/ru/vk/itmo/chernyshevyaroslav/DiskStorage.java b/src/main/java/ru/vk/itmo/chernyshevyaroslav/DiskStorage.java new file mode 100644 index 000000000..031fc8076 --- /dev/null +++ b/src/main/java/ru/vk/itmo/chernyshevyaroslav/DiskStorage.java @@ -0,0 +1,151 @@ +package ru.vk.itmo.chernyshevyaroslav; + +import ru.vk.itmo.BaseEntry; +import ru.vk.itmo.Entry; + +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.Iterator; +import java.util.List; +import java.util.NoSuchElementException; + +public class DiskStorage { + + private final List segmentList; + + public DiskStorage(List segmentList) { + this.segmentList = segmentList; + } + + public Iterator> range( + Iterator> firstIterator, + MemorySegment from, + MemorySegment to) { + List>> iterators = new ArrayList<>(segmentList.size() + 1); + for (MemorySegment memorySegment : segmentList) { + iterators.add(iterator(memorySegment, from, to)); + } + iterators.add(firstIterator); + + return new MergeIterator<>(iterators, Comparator.comparing(Entry::key, ChernyshevDao::compare)) { + @Override + protected boolean skip(Entry memorySegmentEntry) { + return memorySegmentEntry.value() == null; + } + }; + } + + private static long indexOf(MemorySegment segment, MemorySegment key) { + long recordsCount = recordsCount(segment); + + long left = 0; + long right = recordsCount - 1; + while (left <= right) { + long mid = (left + right) >>> 1; + + long startOfKey = startOfKey(segment, mid); + long endOfKey = endOfKey(segment, mid); + long mismatch = MemorySegment.mismatch(segment, startOfKey, endOfKey, key, 0, key.byteSize()); + if (mismatch == -1) { + return mid; + } + + if (mismatch == key.byteSize()) { + right = mid - 1; + continue; + } + + if (mismatch == endOfKey - startOfKey) { + left = mid + 1; + continue; + } + + int b1 = Byte.toUnsignedInt(segment.get(ValueLayout.JAVA_BYTE, startOfKey + mismatch)); + int b2 = Byte.toUnsignedInt(key.get(ValueLayout.JAVA_BYTE, mismatch)); + if (b1 > b2) { + right = mid - 1; + } else { + left = mid + 1; + } + } + + return tombstone(left); + } + + private static long recordsCount(MemorySegment segment) { + long indexSize = indexSize(segment); + return indexSize / Long.BYTES / 2; + } + + private static long indexSize(MemorySegment segment) { + return segment.get(ValueLayout.JAVA_LONG_UNALIGNED, 0); + } + + static Iterator> iterator(MemorySegment page, MemorySegment from, MemorySegment to) { + long recordIndexFrom = from == null ? 0 : normalize(indexOf(page, from)); + long recordIndexTo = to == null ? recordsCount(page) : normalize(indexOf(page, to)); + long recordsCount = recordsCount(page); + + return new Iterator<>() { + long index = recordIndexFrom; + + @Override + public boolean hasNext() { + return index < recordIndexTo; + } + + @Override + public Entry next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + MemorySegment key = slice(page, startOfKey(page, index), endOfKey(page, index)); + long startOfValue = startOfValue(page, index); + MemorySegment value = + startOfValue < 0 + ? null + : slice(page, startOfValue, endOfValue(page, index, recordsCount)); + index++; + return new BaseEntry<>(key, value); + } + }; + } + + private static MemorySegment slice(MemorySegment page, long start, long end) { + return page.asSlice(start, end - start); + } + + private static long startOfKey(MemorySegment segment, long recordIndex) { + return segment.get(ValueLayout.JAVA_LONG_UNALIGNED, recordIndex * 2 * Long.BYTES); + } + + private static long endOfKey(MemorySegment segment, long recordIndex) { + return normalizedStartOfValue(segment, recordIndex); + } + + private static long normalizedStartOfValue(MemorySegment segment, long recordIndex) { + return normalize(startOfValue(segment, recordIndex)); + } + + private static long startOfValue(MemorySegment segment, long recordIndex) { + return segment.get(ValueLayout.JAVA_LONG_UNALIGNED, recordIndex * 2 * Long.BYTES + Long.BYTES); + } + + private static long endOfValue(MemorySegment segment, long recordIndex, long recordsCount) { + if (recordIndex < recordsCount - 1) { + return startOfKey(segment, recordIndex + 1); + } + return segment.byteSize(); + } + + static long tombstone(long offset) { + return 1L << 63 | offset; + } + + private static long normalize(long value) { + return value & ~(1L << 63); + } + +} diff --git a/src/main/java/ru/vk/itmo/chernyshevyaroslav/FileUtils.java b/src/main/java/ru/vk/itmo/chernyshevyaroslav/FileUtils.java new file mode 100644 index 000000000..e19b33c7d --- /dev/null +++ b/src/main/java/ru/vk/itmo/chernyshevyaroslav/FileUtils.java @@ -0,0 +1,205 @@ +package ru.vk.itmo.chernyshevyaroslav; + +import ru.vk.itmo.Entry; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.nio.channels.FileChannel; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.List; + +public final class FileUtils { + + private static final String INDEX_IDX = "index.idx"; + private static final String INDEX_TMP = "index.tmp"; + private static final String COMPACTION_TMP = "Compaction.tmp"; + + private FileUtils() { + throw new IllegalStateException("Utility class"); + } + + public static void save(Path storagePath, Iterable> iterable, boolean isCompaction) + throws IOException { + final Path indexTmp = storagePath.resolve(INDEX_TMP); + final Path indexFile = storagePath.resolve(INDEX_IDX); + + try { + Files.createFile(indexFile); + } catch (FileAlreadyExistsException ignored) { + // it is ok, actually it is normal state + } + List existedFiles = Files.readAllLines(indexFile, StandardCharsets.UTF_8); + + String newFileName = isCompaction ? COMPACTION_TMP : String.valueOf(existedFiles.size()); + + long dataSize = 0; + long count = 0; + for (Entry entry : iterable) { + dataSize += entry.key().byteSize(); + MemorySegment value = entry.value(); + if (value != null) { + dataSize += value.byteSize(); + } + count++; + } + long indexSize = count * 2 * Long.BYTES; + + try ( + FileChannel fileChannel = FileChannel.open( + storagePath.resolve(newFileName), + StandardOpenOption.WRITE, + StandardOpenOption.READ, + StandardOpenOption.CREATE + ); + Arena writeArena = Arena.ofConfined() + ) { + MemorySegment fileSegment = fileChannel.map( + FileChannel.MapMode.READ_WRITE, + 0, + indexSize + dataSize, + writeArena + ); + + // index: + // |key0_Start|value0_Start|key1_Start|value1_Start|key2_Start|value2_Start|... + // key0_Start = data start = end of index + long dataOffset = indexSize; + int indexOffset = 0; + for (Entry entry : iterable) { + fileSegment.set(ValueLayout.JAVA_LONG_UNALIGNED, indexOffset, dataOffset); + dataOffset += entry.key().byteSize(); + indexOffset += Long.BYTES; + + MemorySegment value = entry.value(); + if (value == null) { + fileSegment.set(ValueLayout.JAVA_LONG_UNALIGNED, indexOffset, DiskStorage.tombstone(dataOffset)); + } else { + fileSegment.set(ValueLayout.JAVA_LONG_UNALIGNED, indexOffset, dataOffset); + dataOffset += value.byteSize(); + } + indexOffset += Long.BYTES; + } + + // data: + // |key0|value0|key1|value1|... + dataOffset = indexSize; + for (Entry entry : iterable) { + MemorySegment key = entry.key(); + MemorySegment.copy(key, 0, fileSegment, dataOffset, key.byteSize()); + dataOffset += key.byteSize(); + + MemorySegment value = entry.value(); + if (value != null) { + MemorySegment.copy(value, 0, fileSegment, dataOffset, value.byteSize()); + dataOffset += value.byteSize(); + } + } + } + + Files.move(indexFile, indexTmp, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + + List list = new ArrayList<>(existedFiles.size() + 1); + list.addAll(existedFiles); + if (!isCompaction) { + list.add(newFileName); + } + Files.write( + indexFile, + list, + StandardOpenOption.WRITE, + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING + ); + + Files.delete(indexTmp); + } + + public static List loadOrRecover(Path storagePath, Arena arena) throws IOException { + if (Files.exists(storagePath.resolve(COMPACTION_TMP))) { + finalizeCompaction(storagePath); + } + Path indexTmp = storagePath.resolve(INDEX_TMP); + Path indexFile = storagePath.resolve(INDEX_IDX); + + if (Files.exists(indexTmp)) { + Files.move(indexTmp, indexFile, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + } else { + try { + Files.createFile(indexFile); + } catch (FileAlreadyExistsException ignored) { + // it is ok, actually it is normal state + } + } + + List existedFiles = Files.readAllLines(indexFile, StandardCharsets.UTF_8); + List result = new ArrayList<>(existedFiles.size()); + for (String fileName : existedFiles) { + Path file = storagePath.resolve(fileName); + try (FileChannel fileChannel = FileChannel.open(file, StandardOpenOption.READ, StandardOpenOption.WRITE)) { + MemorySegment fileSegment = fileChannel.map( + FileChannel.MapMode.READ_WRITE, + 0, + Files.size(file), + arena + ); + result.add(fileSegment); + } + } + + return result; + } + + public static void compact(Path storagePath, Iterable> iterable) throws IOException { + Path indexFile = storagePath.resolve(INDEX_IDX); + + try { + Files.createFile(indexFile); + } catch (FileAlreadyExistsException ignored) { + // it is ok, actually it is normal state + } + List existingFiles = Files.readAllLines(indexFile, StandardCharsets.UTF_8); + + if (existingFiles.isEmpty()) { + Files.delete(indexFile); + return; + } + save(storagePath, iterable, true); + finalizeCompaction(storagePath); + } + + private static void finalizeCompaction(Path storagePath) { + Path indexFile = storagePath.resolve(INDEX_IDX); + + try { + List existingFiles = Files.readAllLines(indexFile, StandardCharsets.UTF_8); + + for (String file : existingFiles) { + Files.deleteIfExists(storagePath.resolve(file)); + } + + Files.writeString( + indexFile, + "0", + StandardOpenOption.WRITE, + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING + ); + + Files.move(storagePath.resolve(COMPACTION_TMP), + storagePath.resolve("0"), + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } +} diff --git a/src/main/java/ru/vk/itmo/chernyshevyaroslav/InMemoryDao.java b/src/main/java/ru/vk/itmo/chernyshevyaroslav/InMemoryDao.java deleted file mode 100644 index 894abe181..000000000 --- a/src/main/java/ru/vk/itmo/chernyshevyaroslav/InMemoryDao.java +++ /dev/null @@ -1,35 +0,0 @@ -package ru.vk.itmo.chernyshevyaroslav; - -import ru.vk.itmo.Dao; -import ru.vk.itmo.Entry; -import java.lang.foreign.MemorySegment; -import java.util.Iterator; -import java.util.concurrent.ConcurrentSkipListMap; - -public class InMemoryDao implements Dao> { - private final ConcurrentSkipListMap> data = - new ConcurrentSkipListMap<>(new MemorySegmentComparator()); - - @Override - public Iterator> get(MemorySegment from, MemorySegment to) { - if (from == null && to == null) { - return data.values().iterator(); - } else if (from == null) { - return data.headMap(to).values().iterator(); - } else if (to == null) { - return data.tailMap(from).values().iterator(); - } else { - return data.subMap(from, to).values().iterator(); - } - } - - @Override - public Entry get(MemorySegment key) { - return data.get(key); - } - - @Override - public void upsert(Entry entry) { - data.put(entry.key(), entry); - } -} diff --git a/src/main/java/ru/vk/itmo/chernyshevyaroslav/MemorySegmentComparator.java b/src/main/java/ru/vk/itmo/chernyshevyaroslav/MemorySegmentComparator.java deleted file mode 100644 index b95b4ddc5..000000000 --- a/src/main/java/ru/vk/itmo/chernyshevyaroslav/MemorySegmentComparator.java +++ /dev/null @@ -1,21 +0,0 @@ -package ru.vk.itmo.chernyshevyaroslav; - -import java.lang.foreign.MemorySegment; -import java.lang.foreign.ValueLayout; -import java.util.Comparator; - -public final class MemorySegmentComparator implements Comparator { - @Override - public int compare(MemorySegment o1, MemorySegment o2) { - long offset = o1.mismatch(o2); - if (offset == -1) { - return 0; - } else if (o1.byteSize() == offset) { - return -1; - } else if (o2.byteSize() == offset) { - return 1; - } else { - return Byte.compareUnsigned(o1.get(ValueLayout.JAVA_BYTE, offset), o2.get(ValueLayout.JAVA_BYTE, offset)); - } - } -} diff --git a/src/main/java/ru/vk/itmo/chernyshevyaroslav/MergeIterator.java b/src/main/java/ru/vk/itmo/chernyshevyaroslav/MergeIterator.java new file mode 100644 index 000000000..2c85a0cc6 --- /dev/null +++ b/src/main/java/ru/vk/itmo/chernyshevyaroslav/MergeIterator.java @@ -0,0 +1,136 @@ +package ru.vk.itmo.chernyshevyaroslav; + +import java.util.Collection; +import java.util.Comparator; +import java.util.Iterator; +import java.util.NoSuchElementException; +import java.util.PriorityQueue; + +public abstract class MergeIterator implements Iterator { + + private final PriorityQueue> priorityQueue; + private final Comparator comparator; + private PeekIterator peek; + + private static class PeekIterator implements Iterator { + + public final int id; + private final Iterator delegate; + private T peek; + + private PeekIterator(int id, Iterator delegate) { + this.id = id; + this.delegate = delegate; + } + + @Override + public boolean hasNext() { + if (peek == null) { + return delegate.hasNext(); + } + return true; + } + + @Override + public T next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + T newNext = peek(); + this.peek = null; + return newNext; + } + + private T peek() { + if (peek == null) { + if (!delegate.hasNext()) { + return null; + } + peek = delegate.next(); + } + return peek; + } + } + + protected MergeIterator(Collection> iterators, Comparator comparator) { + this.comparator = comparator; + Comparator> peekComp = (o1, o2) -> comparator.compare(o1.peek(), o2.peek()); + priorityQueue = new PriorityQueue<>( + iterators.size(), + peekComp.thenComparing(o -> -o.id) + ); + + int id = 0; + for (Iterator iterator : iterators) { + if (iterator.hasNext()) { + priorityQueue.add(new PeekIterator<>(id++, iterator)); + } + } + } + + private PeekIterator peek() { + while (peek == null) { + peek = priorityQueue.poll(); + if (peek == null) { + return null; + } + + PeekIterator next = priorityQueue.peek(); + skipIdentical(next); + + if (peek.peek() == null) { + peek = null; + continue; + } + + if (skip(peek.peek())) { + peek.next(); + if (peek.hasNext()) { + priorityQueue.add(peek); + } + peek = null; + } + } + + return peek; + } + + private void skipIdentical(PeekIterator next) { + PeekIterator result = next; + while (result != null) { + + int compare = comparator.compare(peek.peek(), result.peek()); + PeekIterator poll = priorityQueue.peek(); + if ((compare != 0) || (poll == null)) { + break; + } + priorityQueue.remove(); + poll.next(); + if (poll.hasNext()) { + priorityQueue.add(poll); + } + result = priorityQueue.peek(); + } + } + + protected abstract boolean skip(T t); + + @Override + public boolean hasNext() { + return peek() != null; + } + + @Override + public T next() { + PeekIterator peekIterator = peek(); + if (peekIterator == null) { + throw new NoSuchElementException(); + } + T next = peekIterator.next(); + this.peek = null; + if (peekIterator.hasNext()) { + priorityQueue.add(peekIterator); + } + return next; + } +} diff --git a/src/main/java/ru/vk/itmo/test/chernyshevyaroslav/InMemoryDaoFactory.java b/src/main/java/ru/vk/itmo/test/chernyshevyaroslav/ChernyshevDaoFactory.java similarity index 63% rename from src/main/java/ru/vk/itmo/test/chernyshevyaroslav/InMemoryDaoFactory.java rename to src/main/java/ru/vk/itmo/test/chernyshevyaroslav/ChernyshevDaoFactory.java index 7ba91e6a4..7055ff62f 100644 --- a/src/main/java/ru/vk/itmo/test/chernyshevyaroslav/InMemoryDaoFactory.java +++ b/src/main/java/ru/vk/itmo/test/chernyshevyaroslav/ChernyshevDaoFactory.java @@ -1,22 +1,28 @@ package ru.vk.itmo.test.chernyshevyaroslav; +import ru.vk.itmo.Config; import ru.vk.itmo.Dao; import ru.vk.itmo.Entry; -import ru.vk.itmo.chernyshevyaroslav.InMemoryDao; +import ru.vk.itmo.chernyshevyaroslav.ChernyshevDao; import ru.vk.itmo.test.DaoFactory; + +import java.io.IOException; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; import java.nio.charset.StandardCharsets; -@DaoFactory -public class InMemoryDaoFactory implements DaoFactory.Factory> { +@DaoFactory(stage = 4, week = 6) +public class ChernyshevDaoFactory implements DaoFactory.Factory> { @Override - public Dao> createDao() { - return new InMemoryDao(); + public Dao> createDao(Config config) throws IOException { + return new ChernyshevDao(config); } @Override public String toString(MemorySegment memorySegment) { + if (memorySegment == null) { + return null; + } return new String(memorySegment.toArray(ValueLayout.JAVA_BYTE), StandardCharsets.UTF_8); } From d00956c2a851748b09acaa6d7c384dc60deccceb Mon Sep 17 00:00:00 2001 From: Andrey <70713475+Queenore@users.noreply.github.com> Date: Sat, 25 Nov 2023 13:22:47 +0300 Subject: [PATCH 03/20] =?UTF-8?q?=D0=A7=D0=B5=D1=88=D0=B5=D0=B2=20=D0=90?= =?UTF-8?q?=D0=BD=D0=B4=D1=80=D0=B5=D0=B9,=20=D0=9F=D0=BE=D0=BB=D0=B8?= =?UTF-8?q?=D1=82=D0=B5=D1=85,=20=D0=97=D0=B0=D0=B4=D0=B0=D0=BD=D0=B8?= =?UTF-8?q?=D0=B5=204=20(#201)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Vadim Tsesko Co-authored-by: Vladislav Zybkin <47724127+zvladn7@users.noreply.github.com> --- .../ru/vk/itmo/cheshevandrey/DiskStorage.java | 297 ++++++++++++++++++ .../ru/vk/itmo/cheshevandrey/InMemoryDao.java | 185 ----------- .../itmo/cheshevandrey/IterableStorage.java | 22 ++ .../vk/itmo/cheshevandrey/MergeIterator.java | 106 +++++++ .../vk/itmo/cheshevandrey/PeekIterator.java | 44 +++ .../vk/itmo/cheshevandrey/PersistentDao.java | 123 ++++++++ .../java/ru/vk/itmo/cheshevandrey/Tools.java | 56 ++++ .../test/cheshevandrey/InMemoryFactory.java | 6 +- 8 files changed, 651 insertions(+), 188 deletions(-) create mode 100644 src/main/java/ru/vk/itmo/cheshevandrey/DiskStorage.java delete mode 100644 src/main/java/ru/vk/itmo/cheshevandrey/InMemoryDao.java create mode 100644 src/main/java/ru/vk/itmo/cheshevandrey/IterableStorage.java create mode 100644 src/main/java/ru/vk/itmo/cheshevandrey/MergeIterator.java create mode 100644 src/main/java/ru/vk/itmo/cheshevandrey/PeekIterator.java create mode 100644 src/main/java/ru/vk/itmo/cheshevandrey/PersistentDao.java create mode 100644 src/main/java/ru/vk/itmo/cheshevandrey/Tools.java diff --git a/src/main/java/ru/vk/itmo/cheshevandrey/DiskStorage.java b/src/main/java/ru/vk/itmo/cheshevandrey/DiskStorage.java new file mode 100644 index 000000000..8905edf00 --- /dev/null +++ b/src/main/java/ru/vk/itmo/cheshevandrey/DiskStorage.java @@ -0,0 +1,297 @@ +package ru.vk.itmo.cheshevandrey; + +import ru.vk.itmo.BaseEntry; +import ru.vk.itmo.Entry; + +import java.io.IOException; +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.nio.channels.FileChannel; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.Iterator; +import java.util.List; +import java.util.NoSuchElementException; + +public class DiskStorage { + + private final List segmentList; + + private static final String INDEX_TMP_FILE_NAME = "index.tmp"; + private static final String INDEX_FILE_NAME = "index.idx"; + private static final String NEW_SSTABLE_FILE_NAME = "newSsTable.tmp"; + private static final String COMPACT_FILE_NAME = "compact.cmpct"; + + public DiskStorage(Arena arena, Path storagePath) throws IOException { + + Path indexFile = storagePath.resolve(INDEX_FILE_NAME); + try { + Files.createFile(indexFile); + } catch (FileAlreadyExistsException ignored) { + // it's ok. + } + + // Если существует скомпакченный файл, то ранее произошла ошибка в методе bringUpToDateState(). + // Следовательно, необходимо доделать компакт, приведя хранилище в актуальное состояние. + if (Files.exists(storagePath.resolve(COMPACT_FILE_NAME))) { + bringUpToDateState(storagePath); + } + + List fileNames = Files.readAllLines(indexFile); + int filesCount = fileNames.size(); + this.segmentList = new ArrayList<>(filesCount); + for (int i = 0; i < filesCount; i++) { + Path file = storagePath.resolve(String.valueOf(i)); + try (FileChannel fileChannel = FileChannel.open(file, StandardOpenOption.READ, StandardOpenOption.WRITE)) { + MemorySegment fileSegment = fileChannel.map( + FileChannel.MapMode.READ_WRITE, + 0, + Files.size(file), + arena + ); + segmentList.add(fileSegment); + } + } + } + + public Iterator> range( + Iterator> firstIterator, + MemorySegment from, + MemorySegment to) { + List>> iterators = new ArrayList<>(segmentList.size() + 1); + for (MemorySegment memorySegment : segmentList) { + iterators.add(iterator(memorySegment, from, to)); + } + iterators.add(firstIterator); + + return new MergeIterator<>(iterators, Comparator.comparing(Entry::key, PersistentDao::compare)) { + @Override + protected boolean skip(Entry memorySegmentEntry) { + return memorySegmentEntry.value() == null; + } + }; + } + + public static void compact( + Path storagePath, + DiskStorage diskStorage, + Iterable> iterableMemTable + ) throws IOException { + IterableStorage storage = new IterableStorage(iterableMemTable, diskStorage); + if (diskStorage.segmentList.size() <= 1 && !iterableMemTable.iterator().hasNext()) { + return; + } + + // После успешного выполнения ожидаем увидеть скомпакченый файл COMPACT_FILE_NAME. + save(storagePath, storage, true); + + bringUpToDateState(storagePath); + } + + public static void save(Path storagePath, Iterable> iterable, boolean isForCompact) + throws IOException { + Path newSsTablePath = storagePath.resolve(NEW_SSTABLE_FILE_NAME); + + long dataSize = 0; + long count = 0; + for (Entry entry : iterable) { + dataSize += entry.key().byteSize(); + MemorySegment value = entry.value(); + if (value != null) { + dataSize += value.byteSize(); + } + count++; + } + long indexSize = count * 2 * Long.BYTES; + + try ( + FileChannel fileChannel = FileChannel.open( + newSsTablePath, + StandardOpenOption.WRITE, + StandardOpenOption.READ, + StandardOpenOption.CREATE + ); + Arena writeArena = Arena.ofConfined() + ) { + MemorySegment fileSegment = fileChannel.map( + FileChannel.MapMode.READ_WRITE, + 0, + indexSize + dataSize, + writeArena + ); + + // index: + // |key0_Start|value0_Start|key1_Start|value1_Start|key2_Start|value2_Start|... + // key0_Start = data start = end of index + long dataOffset = indexSize; + int indexOffset = 0; + for (Entry entry : iterable) { + fileSegment.set(ValueLayout.JAVA_LONG_UNALIGNED, indexOffset, dataOffset); + dataOffset += entry.key().byteSize(); + indexOffset += Long.BYTES; + + MemorySegment value = entry.value(); + if (value == null) { + fileSegment.set(ValueLayout.JAVA_LONG_UNALIGNED, indexOffset, Tools.tombstone(dataOffset)); + } else { + fileSegment.set(ValueLayout.JAVA_LONG_UNALIGNED, indexOffset, dataOffset); + dataOffset += value.byteSize(); + } + indexOffset += Long.BYTES; + } + + // data: + // |key0|value0|key1|value1|... + dataOffset = indexSize; + for (Entry entry : iterable) { + MemorySegment key = entry.key(); + MemorySegment.copy(key, 0, fileSegment, dataOffset, key.byteSize()); + dataOffset += key.byteSize(); + + MemorySegment value = entry.value(); + if (value != null) { + MemorySegment.copy(value, 0, fileSegment, dataOffset, value.byteSize()); + dataOffset += value.byteSize(); + } + } + } + + if (isForCompact) { + Files.move( + newSsTablePath, + storagePath.resolve(COMPACT_FILE_NAME), + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING + ); + return; + } + + Path indexFile = storagePath.resolve(INDEX_FILE_NAME); + List fileNames = Files.readAllLines(indexFile); + String newFileName = String.valueOf(fileNames.size()); + fileNames.add(newFileName); + + updateIndex(storagePath, fileNames); + + Path newFilePath = storagePath.resolve(newFileName); + Files.move( + newSsTablePath, + newFilePath, + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING + ); + } + + private static void bringUpToDateState(Path storagePath) throws IOException { + List fileNames = Files.readAllLines(storagePath.resolve(INDEX_FILE_NAME)); + + for (int i = 1; i < fileNames.size(); i++) { + Files.delete(storagePath.resolve(String.valueOf(i))); + } + + String newFileName = "0"; + + updateIndex(storagePath, Collections.singletonList(newFileName)); + + Files.move( + storagePath.resolve(COMPACT_FILE_NAME), + storagePath.resolve(newFileName), + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING + ); + } + + private static void updateIndex(Path storagePath, List files) throws IOException { + Path indexTmpPath = storagePath.resolve(INDEX_TMP_FILE_NAME); + Path indexPath = storagePath.resolve(INDEX_FILE_NAME); + + Files.deleteIfExists(indexTmpPath); + Files.write( + indexTmpPath, + files, + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING + ); + + Files.move( + indexTmpPath, + indexPath, + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING + ); + } + + private static long indexOf(MemorySegment segment, MemorySegment key) { + long recordsCount = Tools.recordsCount(segment); + + long left = 0; + long right = recordsCount - 1; + while (left <= right) { + long mid = (left + right) >>> 1; + + long startOfKey = Tools.startOfKey(segment, mid); + long endOfKey = Tools.endOfKey(segment, mid); + long mismatch = MemorySegment.mismatch(segment, startOfKey, endOfKey, key, 0, key.byteSize()); + if (mismatch == -1) { + return mid; + } + + if (mismatch == key.byteSize()) { + right = mid - 1; + continue; + } + + if (mismatch == endOfKey - startOfKey) { + left = mid + 1; + continue; + } + + int b1 = Byte.toUnsignedInt(segment.get(ValueLayout.JAVA_BYTE, startOfKey + mismatch)); + int b2 = Byte.toUnsignedInt(key.get(ValueLayout.JAVA_BYTE, mismatch)); + if (b1 > b2) { + right = mid - 1; + } else { + left = mid + 1; + } + } + + return Tools.tombstone(left); + } + + private static Iterator> iterator(MemorySegment page, MemorySegment from, MemorySegment to) { + long recordIndexFrom = from == null ? 0 : Tools.normalize(indexOf(page, from)); + long recordIndexTo = to == null ? Tools.recordsCount(page) : Tools.normalize(indexOf(page, to)); + long recordsCount = Tools.recordsCount(page); + + return new Iterator<>() { + long index = recordIndexFrom; + + @Override + public boolean hasNext() { + return index < recordIndexTo; + } + + @Override + public Entry next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + MemorySegment key = Tools.slice(page, Tools.startOfKey(page, index), Tools.endOfKey(page, index)); + long startOfValue = Tools.startOfValue(page, index); + MemorySegment value = + startOfValue < 0 + ? null + : Tools.slice(page, startOfValue, Tools.endOfValue(page, index, recordsCount)); + index++; + return new BaseEntry<>(key, value); + } + }; + } +} diff --git a/src/main/java/ru/vk/itmo/cheshevandrey/InMemoryDao.java b/src/main/java/ru/vk/itmo/cheshevandrey/InMemoryDao.java deleted file mode 100644 index 90f5842b0..000000000 --- a/src/main/java/ru/vk/itmo/cheshevandrey/InMemoryDao.java +++ /dev/null @@ -1,185 +0,0 @@ -package ru.vk.itmo.cheshevandrey; - -import ru.vk.itmo.BaseEntry; -import ru.vk.itmo.Config; -import ru.vk.itmo.Dao; -import ru.vk.itmo.Entry; - -import java.io.IOException; -import java.lang.foreign.Arena; -import java.lang.foreign.MemorySegment; -import java.lang.foreign.ValueLayout; -import java.nio.channels.FileChannel; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardOpenOption; -import java.util.Iterator; -import java.util.NavigableMap; -import java.util.concurrent.ConcurrentSkipListMap; -import java.util.logging.Logger; - -public class InMemoryDao implements Dao> { - - private final Arena offHeapArena; - private final Path storagePath; - private static final String STORAGE_FILE_NAME = "output.sst"; - private static final Logger logger = Logger.getLogger(InMemoryDao.class.getName()); - private final NavigableMap> memTable = new ConcurrentSkipListMap<>( - this::compare - ); - - public InMemoryDao(Config config) throws IOException { - this.storagePath = config.basePath().resolve(STORAGE_FILE_NAME); - - if (!Files.exists(storagePath)) { - Files.createDirectories(storagePath.getParent()); - Files.createFile(storagePath); - } - - offHeapArena = Arena.ofConfined(); - } - - @Override - public Iterator> get(MemorySegment from, MemorySegment to) { - - if (from == null && to == null) { - return memTable.values().iterator(); - } - if (from == null) { - return memTable.headMap(to).values().iterator(); - } - if (to == null) { - return memTable.tailMap(from).values().iterator(); - } - return memTable.subMap(from, to).values().iterator(); - } - - @Override - public Entry get(MemorySegment key) { - - Entry entry = memTable.get(key); - if (entry != null) { - return entry; - } - - try (FileChannel channel = FileChannel.open( - storagePath, - StandardOpenOption.CREATE, - StandardOpenOption.READ - )) { - MemorySegment ssTable = channel.map( - FileChannel.MapMode.READ_ONLY, - 0, - channel.size(), - offHeapArena - ); - - if (ssTable.byteSize() == 0) { - return null; - } - - long offset = Long.BYTES; - long mismatch; - long operandSize; - long tableSize = readSizeFromSsTable(ssTable, 0); - while (offset < tableSize) { - - operandSize = readSizeFromSsTable(ssTable, offset); - offset += Long.BYTES; - - mismatch = MemorySegment.mismatch(ssTable, offset, offset + operandSize, key, 0, key.byteSize()); - offset += operandSize; - - operandSize = readSizeFromSsTable(ssTable, offset); - offset += Long.BYTES; - - if (mismatch == -1) { - return new BaseEntry<>( - key, - ssTable.asSlice(offset, operandSize) - ); - } - offset += operandSize; - } - } catch (IOException e) { - logger.severe("Ошибка при создании FileChannel: " + e.getMessage()); - } - - return null; - } - - private long readSizeFromSsTable(MemorySegment ssTable, long offset) { - return ssTable.get(ValueLayout.JAVA_LONG_UNALIGNED, offset); - } - - @Override - public void close() throws IOException { - - try ( - FileChannel channel = FileChannel.open( - storagePath, - StandardOpenOption.CREATE, - StandardOpenOption.READ, - StandardOpenOption.WRITE - ); - Arena closeArena = Arena.ofConfined() - ) { - long ssTableSize = 2L * Long.BYTES * memTable.size() + Long.BYTES; - for (Entry entry : memTable.values()) { - ssTableSize += entry.key().byteSize(); - ssTableSize += entry.value().byteSize(); - } - - MemorySegment ssTable = channel.map( - FileChannel.MapMode.READ_WRITE, - 0, - ssTableSize, - closeArena - ); - - ssTable.set(ValueLayout.JAVA_LONG_UNALIGNED, 0, ssTableSize); - - long offset = Long.BYTES; - for (Entry entry : memTable.values()) { - offset = storeAndGetOffset(ssTable, entry.key(), offset); - offset = storeAndGetOffset(ssTable, entry.value(), offset); - } - } finally { - offHeapArena.close(); - } - } - - private long storeAndGetOffset(MemorySegment ssTable, MemorySegment value, long offset) { - long newOffset = offset; - long valueSize = value.byteSize(); - - ssTable.set(ValueLayout.JAVA_LONG_UNALIGNED, newOffset, valueSize); - newOffset += Long.BYTES; - - MemorySegment.copy(value, 0, ssTable, newOffset, valueSize); - newOffset += valueSize; - - return newOffset; - } - - @Override - public void upsert(Entry entry) { - memTable.put(entry.key(), entry); - } - - private int compare(MemorySegment seg1, MemorySegment seg2) { - long mismatch = seg1.mismatch(seg2); - if (mismatch == -1) { - return 0; - } - if (mismatch == seg1.byteSize()) { - return -1; - } - if (mismatch == seg2.byteSize()) { - return 1; - } - byte b1 = seg1.get(ValueLayout.JAVA_BYTE, mismatch); - byte b2 = seg2.get(ValueLayout.JAVA_BYTE, mismatch); - return Byte.compare(b1, b2); - } -} diff --git a/src/main/java/ru/vk/itmo/cheshevandrey/IterableStorage.java b/src/main/java/ru/vk/itmo/cheshevandrey/IterableStorage.java new file mode 100644 index 000000000..d29e2a3f3 --- /dev/null +++ b/src/main/java/ru/vk/itmo/cheshevandrey/IterableStorage.java @@ -0,0 +1,22 @@ +package ru.vk.itmo.cheshevandrey; + +import ru.vk.itmo.Entry; + +import java.lang.foreign.MemorySegment; +import java.util.Iterator; + +public class IterableStorage implements Iterable> { + + Iterable> iterableMemTable; + DiskStorage diskStorage; + + IterableStorage(Iterable> iterableMemTable, DiskStorage diskStorage) { + this.iterableMemTable = iterableMemTable; + this.diskStorage = diskStorage; + } + + @Override + public Iterator> iterator() { + return diskStorage.range(iterableMemTable.iterator(), null, null); + } +} diff --git a/src/main/java/ru/vk/itmo/cheshevandrey/MergeIterator.java b/src/main/java/ru/vk/itmo/cheshevandrey/MergeIterator.java new file mode 100644 index 000000000..24cadec32 --- /dev/null +++ b/src/main/java/ru/vk/itmo/cheshevandrey/MergeIterator.java @@ -0,0 +1,106 @@ +package ru.vk.itmo.cheshevandrey; + +import java.util.Collection; +import java.util.Comparator; +import java.util.Iterator; +import java.util.NoSuchElementException; +import java.util.PriorityQueue; + +public class MergeIterator implements Iterator { + + private final PriorityQueue> priorityQueue; + private final Comparator comparator; + + PeekIterator peek; + + public MergeIterator(Collection> iterators, Comparator comparator) { + this.comparator = comparator; + Comparator> peekComp = (o1, o2) -> comparator.compare(o1.peek(), o2.peek()); + priorityQueue = new PriorityQueue<>( + iterators.size(), + peekComp.thenComparing(o -> -o.id) + ); + + int id = 0; + for (Iterator iterator : iterators) { + if (iterator.hasNext()) { + priorityQueue.add(new PeekIterator<>(id++, iterator)); + } + } + } + + private PeekIterator peek() { + while (peek == null) { + peek = priorityQueue.poll(); + if (peek == null) { + return null; + } + + updateQueueState(); + + if (peek.peek() == null) { + peek = null; + continue; + } + + if (skip(peek.peek())) { + peek.next(); + if (peek.hasNext()) { + priorityQueue.add(peek); + } + peek = null; + } + } + + return peek; + } + + private void updateQueueState() { + while (true) { + PeekIterator next = priorityQueue.peek(); + if (next == null) { + break; + } + + int compare = comparator.compare(peek.peek(), next.peek()); + if (compare == 0) { + pollNext(); + } else { + break; + } + } + } + + private void pollNext() { + PeekIterator poll = priorityQueue.poll(); + if (poll != null) { + poll.next(); + if (poll.hasNext()) { + priorityQueue.add(poll); + } + } + } + + protected boolean skip(T t) { + return t == null; + } + + @Override + public boolean hasNext() { + return peek() != null; + } + + @Override + public T next() { + PeekIterator currIterator = peek(); + if (currIterator == null) { + throw new NoSuchElementException(); + } + T next = currIterator.next(); + this.peek = null; + if (currIterator.hasNext()) { + priorityQueue.add(currIterator); + } + return next; + } +} diff --git a/src/main/java/ru/vk/itmo/cheshevandrey/PeekIterator.java b/src/main/java/ru/vk/itmo/cheshevandrey/PeekIterator.java new file mode 100644 index 000000000..bd9ab6526 --- /dev/null +++ b/src/main/java/ru/vk/itmo/cheshevandrey/PeekIterator.java @@ -0,0 +1,44 @@ +package ru.vk.itmo.cheshevandrey; + +import java.util.Iterator; +import java.util.NoSuchElementException; + +class PeekIterator implements Iterator { + + public final int id; + private final Iterator delegate; + private T currIterator; + + PeekIterator(int id, Iterator delegate) { + this.id = id; + this.delegate = delegate; + } + + @Override + public boolean hasNext() { + if (currIterator == null) { + return delegate.hasNext(); + } + return true; + } + + @Override + public T next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + T peek = peek(); + this.currIterator = null; + return peek; + } + + T peek() { + if (currIterator == null) { + if (!delegate.hasNext()) { + return null; + } + currIterator = delegate.next(); + } + return currIterator; + } +} diff --git a/src/main/java/ru/vk/itmo/cheshevandrey/PersistentDao.java b/src/main/java/ru/vk/itmo/cheshevandrey/PersistentDao.java new file mode 100644 index 000000000..7df4a85e6 --- /dev/null +++ b/src/main/java/ru/vk/itmo/cheshevandrey/PersistentDao.java @@ -0,0 +1,123 @@ +package ru.vk.itmo.cheshevandrey; + +import ru.vk.itmo.Config; +import ru.vk.itmo.Dao; +import ru.vk.itmo.Entry; + +import java.io.IOException; +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collections; +import java.util.Comparator; +import java.util.Iterator; +import java.util.NavigableMap; +import java.util.concurrent.ConcurrentSkipListMap; + +public class PersistentDao implements Dao> { + + private final Comparator comparator = PersistentDao::compare; + private final Arena arena; + private final DiskStorage diskStorage; + private final Path path; + + private NavigableMap> storage = new ConcurrentSkipListMap<>(comparator); + + public PersistentDao(Config config) throws IOException { + this.path = config.basePath().resolve("data"); + Files.createDirectories(path); + + arena = Arena.ofShared(); + + this.diskStorage = new DiskStorage(arena, path); + } + + static int compare(MemorySegment memorySegment1, MemorySegment memorySegment2) { + long mismatch = memorySegment1.mismatch(memorySegment2); + if (mismatch == -1) { + return 0; + } + + if (mismatch == memorySegment1.byteSize()) { + return -1; + } + + if (mismatch == memorySegment2.byteSize()) { + return 1; + } + byte b1 = memorySegment1.get(ValueLayout.JAVA_BYTE, mismatch); + byte b2 = memorySegment2.get(ValueLayout.JAVA_BYTE, mismatch); + return Byte.compare(b1, b2); + } + + @Override + public Iterator> get(MemorySegment from, MemorySegment to) { + return diskStorage.range(getInMemory(from, to).iterator(), from, to); + } + + private Iterable> getInMemory(MemorySegment from, MemorySegment to) { + if (from == null && to == null) { + return storage.values(); + } + if (from == null) { + return storage.headMap(to).values(); + } + if (to == null) { + return storage.tailMap(from).values(); + } + return storage.subMap(from, to).values(); + } + + @Override + public void upsert(Entry entry) { + storage.put(entry.key(), entry); + } + + @Override + public Entry get(MemorySegment key) { + Entry entry = storage.get(key); + if (entry != null) { + if (entry.value() == null) { + return null; + } + return entry; + } + + Iterator> iterator = diskStorage.range(Collections.emptyIterator(), key, null); + + if (!iterator.hasNext()) { + return null; + } + Entry next = iterator.next(); + if (compare(next.key(), key) == 0) { + return next; + } + return null; + } + + @Override + public void compact() throws IOException { + DiskStorage.compact(path, diskStorage, getInMemory(null, null)); + storage = new ConcurrentSkipListMap<>(comparator); + } + + @Override + public void flush() throws IOException { + DiskStorage.save(path, storage.values(), false); + } + + @Override + public void close() throws IOException { + if (!arena.scope().isAlive()) { + return; + } + + arena.close(); + + if (!storage.isEmpty()) { + flush(); + } + } +} diff --git a/src/main/java/ru/vk/itmo/cheshevandrey/Tools.java b/src/main/java/ru/vk/itmo/cheshevandrey/Tools.java new file mode 100644 index 000000000..3c77549c7 --- /dev/null +++ b/src/main/java/ru/vk/itmo/cheshevandrey/Tools.java @@ -0,0 +1,56 @@ +package ru.vk.itmo.cheshevandrey; + +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; + +public final class Tools { + + // to hide the implicit public constructor + private Tools() { + } + + static long recordsCount(MemorySegment segment) { + long indexSize = indexSize(segment); + return indexSize / Long.BYTES / 2; + } + + static long indexSize(MemorySegment segment) { + return segment.get(ValueLayout.JAVA_LONG_UNALIGNED, 0); + } + + static MemorySegment slice(MemorySegment page, long start, long end) { + return page.asSlice(start, end - start); + } + + static long startOfKey(MemorySegment segment, long recordIndex) { + return segment.get(ValueLayout.JAVA_LONG_UNALIGNED, recordIndex * 2 * Long.BYTES); + } + + static long endOfKey(MemorySegment segment, long recordIndex) { + return normalizedStartOfValue(segment, recordIndex); + } + + static long normalizedStartOfValue(MemorySegment segment, long recordIndex) { + return normalize(startOfValue(segment, recordIndex)); + } + + static long startOfValue(MemorySegment segment, long recordIndex) { + return segment.get(ValueLayout.JAVA_LONG_UNALIGNED, recordIndex * 2 * Long.BYTES + Long.BYTES); + } + + static long endOfValue(MemorySegment segment, long recordIndex, long recordsCount) { + if (recordIndex < recordsCount - 1) { + return startOfKey(segment, recordIndex + 1); + } + return segment.byteSize(); + } + + static long tombstone(long offset) { + return 1L << 63 | offset; + } + + static long normalize(long value) { + return value & ~(1L << 63); + } + +} diff --git a/src/main/java/ru/vk/itmo/test/cheshevandrey/InMemoryFactory.java b/src/main/java/ru/vk/itmo/test/cheshevandrey/InMemoryFactory.java index 2b61e0b4e..9732661ff 100644 --- a/src/main/java/ru/vk/itmo/test/cheshevandrey/InMemoryFactory.java +++ b/src/main/java/ru/vk/itmo/test/cheshevandrey/InMemoryFactory.java @@ -3,7 +3,7 @@ import ru.vk.itmo.Config; import ru.vk.itmo.Dao; import ru.vk.itmo.Entry; -import ru.vk.itmo.cheshevandrey.InMemoryDao; +import ru.vk.itmo.cheshevandrey.PersistentDao; import ru.vk.itmo.test.DaoFactory; import java.io.IOException; @@ -12,12 +12,12 @@ import static java.nio.charset.StandardCharsets.UTF_8; -@DaoFactory(stage = 2) +@DaoFactory(stage = 4) public class InMemoryFactory implements DaoFactory.Factory> { @Override public Dao> createDao(Config config) throws IOException { - return new InMemoryDao(config); + return new PersistentDao(config); } @Override From 3e6350ed901abccf4ac64b27c69f5609a4160604 Mon Sep 17 00:00:00 2001 From: Dmitry Osokin <114069284+osokindm@users.noreply.github.com> Date: Sat, 25 Nov 2023 16:23:31 +0300 Subject: [PATCH 04/20] =?UTF-8?q?=D1=82=D0=B5=D1=81=D1=82=D1=8B=20=D0=BD?= =?UTF-8?q?=D0=B0=20Windows=20(#152)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Vadim Tsesko Co-authored-by: Artyom Drozdov --- src/main/java/ru/vk/itmo/test/TestDao.java | 3 ++- src/test/java/ru/vk/itmo/PersistentTest.java | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/ru/vk/itmo/test/TestDao.java b/src/main/java/ru/vk/itmo/test/TestDao.java index 54c731a7d..66a4ebc35 100644 --- a/src/main/java/ru/vk/itmo/test/TestDao.java +++ b/src/main/java/ru/vk/itmo/test/TestDao.java @@ -29,7 +29,8 @@ class TestDao> implements Dao> } public Dao> reopen() throws IOException { - return new TestDao<>(factory, config); + delegate = factory.createDao(config); + return this; } @Override diff --git a/src/test/java/ru/vk/itmo/PersistentTest.java b/src/test/java/ru/vk/itmo/PersistentTest.java index 646dcae7a..9b5f2b349 100644 --- a/src/test/java/ru/vk/itmo/PersistentTest.java +++ b/src/test/java/ru/vk/itmo/PersistentTest.java @@ -45,6 +45,7 @@ void variability(Dao> dao) throws IOException { for (final Entry entry : entries) { assertSame(dao.get(entry.key()), entry); } + dao.close(); } @DaoTest(stage = 2) From cf15fba980fd7a3493474c428c3fdbb02d5deaf8 Mon Sep 17 00:00:00 2001 From: AndrewDanilin <76146421+AndrewDanilin@users.noreply.github.com> Date: Sat, 25 Nov 2023 23:22:01 +0300 Subject: [PATCH 05/20] =?UTF-8?q?HW4=20=D0=94=D0=B0=D0=BD=D0=B8=D0=BB?= =?UTF-8?q?=D0=B8=D0=BD=20=D0=90=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A4?= =?UTF-8?q?=D0=98=D0=A2=D0=B8=D0=9F=20M34341=20(#241)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Vadim Tsesko Co-authored-by: Daniil Ushkov <42135591+daniil-ushkov@users.noreply.github.com> Co-authored-by: Ilya Kriushenkov --- .../ru/vk/itmo/danilinandrew/DiskStorage.java | 304 ++++++++++++++++++ .../danilinandrew/DiskStorageWithCompact.java | 126 ++++++++ .../ru/vk/itmo/danilinandrew/InMemoryDao.java | 146 --------- .../MemorySegmentComparator.java | 30 -- .../vk/itmo/danilinandrew/MergeIterator.java | 142 ++++++++ .../ru/vk/itmo/danilinandrew/StorageDao.java | 121 +++++++ .../vk/itmo/test/danilinandrew/MyFactory.java | 9 +- 7 files changed, 698 insertions(+), 180 deletions(-) create mode 100644 src/main/java/ru/vk/itmo/danilinandrew/DiskStorage.java create mode 100644 src/main/java/ru/vk/itmo/danilinandrew/DiskStorageWithCompact.java delete mode 100644 src/main/java/ru/vk/itmo/danilinandrew/InMemoryDao.java delete mode 100644 src/main/java/ru/vk/itmo/danilinandrew/MemorySegmentComparator.java create mode 100644 src/main/java/ru/vk/itmo/danilinandrew/MergeIterator.java create mode 100644 src/main/java/ru/vk/itmo/danilinandrew/StorageDao.java diff --git a/src/main/java/ru/vk/itmo/danilinandrew/DiskStorage.java b/src/main/java/ru/vk/itmo/danilinandrew/DiskStorage.java new file mode 100644 index 000000000..9eb03f4b6 --- /dev/null +++ b/src/main/java/ru/vk/itmo/danilinandrew/DiskStorage.java @@ -0,0 +1,304 @@ +package ru.vk.itmo.danilinandrew; + +import ru.vk.itmo.BaseEntry; +import ru.vk.itmo.Entry; + +import java.io.IOException; +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.nio.channels.FileChannel; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.Iterator; +import java.util.List; +import java.util.NoSuchElementException; + +public class DiskStorage { + + private final List segmentList; + private static final String INDEX_FILE = "index.idx"; + private static final String TMP_FILE = "index.tmp"; + + public DiskStorage(List segmentList) { + this.segmentList = segmentList; + } + + public Iterator> range( + Iterator> firstIterator, + MemorySegment from, + MemorySegment to) { + List>> iterators = new ArrayList<>(segmentList.size() + 1); + for (MemorySegment memorySegment : segmentList) { + iterators.add(iterator(memorySegment, from, to)); + } + iterators.add(firstIterator); + + return new MergeIterator<>(iterators, Comparator.comparing(Entry::key, StorageDao::compare)) { + @Override + protected boolean skip(Entry memorySegmentEntry) { + return memorySegmentEntry.value() == null; + } + }; + } + + public void clearStorage(Path storagePath) throws IOException { + final Path indexFile = storagePath.resolve(INDEX_FILE); + final List existedFiles = Files.readAllLines(indexFile, StandardCharsets.UTF_8); + for (String fileName : existedFiles) { + Files.deleteIfExists(storagePath.resolve(fileName)); + } + } + + public static void save(Path storagePath, Iterable> iterable) + throws IOException { + final Path indexTmp = storagePath.resolve(TMP_FILE); + final Path indexFile = storagePath.resolve(INDEX_FILE); + + try { + Files.createFile(indexFile); + } catch (FileAlreadyExistsException ignored) { + // it is ok, actually it is normal state + } + List existedFiles = Files.readAllLines(indexFile, StandardCharsets.UTF_8); + + String newFileName = String.valueOf(existedFiles.size()); + + long dataSize = 0; + long count = 0; + for (Entry entry : iterable) { + dataSize += entry.key().byteSize(); + MemorySegment value = entry.value(); + if (value != null) { + dataSize += value.byteSize(); + } + count++; + } + long indexSize = count * 2 * Long.BYTES; + + try ( + FileChannel fileChannel = FileChannel.open( + storagePath.resolve(newFileName), + StandardOpenOption.WRITE, + StandardOpenOption.READ, + StandardOpenOption.CREATE + ); + Arena writeArena = Arena.ofConfined() + ) { + MemorySegment fileSegment = fileChannel.map( + FileChannel.MapMode.READ_WRITE, + 0, + indexSize + dataSize, + writeArena + ); + + // index: + // |key0_Start|value0_Start|key1_Start|value1_Start|key2_Start|value2_Start|... + // key0_Start = data start = end of index + long dataOffset = indexSize; + int indexOffset = 0; + for (Entry entry : iterable) { + fileSegment.set(ValueLayout.JAVA_LONG_UNALIGNED, indexOffset, dataOffset); + dataOffset += entry.key().byteSize(); + indexOffset += Long.BYTES; + + MemorySegment value = entry.value(); + if (value == null) { + fileSegment.set(ValueLayout.JAVA_LONG_UNALIGNED, indexOffset, tombstone(dataOffset)); + } else { + fileSegment.set(ValueLayout.JAVA_LONG_UNALIGNED, indexOffset, dataOffset); + dataOffset += value.byteSize(); + } + indexOffset += Long.BYTES; + } + + // data: + // |key0|value0|key1|value1|... + dataOffset = indexSize; + for (Entry entry : iterable) { + MemorySegment key = entry.key(); + MemorySegment.copy(key, 0, fileSegment, dataOffset, key.byteSize()); + dataOffset += key.byteSize(); + + MemorySegment value = entry.value(); + if (value != null) { + MemorySegment.copy(value, 0, fileSegment, dataOffset, value.byteSize()); + dataOffset += value.byteSize(); + } + } + } + + List list = new ArrayList<>(existedFiles.size() + 1); + list.addAll(existedFiles); + list.add(newFileName); + Files.write( + indexTmp, + list, + StandardOpenOption.WRITE, + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING + ); + + Files.move(indexTmp, indexFile, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + } + + public static List loadOrRecover(Path storagePath, Arena arena) throws IOException { + Path indexTmp = storagePath.resolve(TMP_FILE); + Path indexFile = storagePath.resolve(INDEX_FILE); + + // Не смог вынести названия в константы, потому что codeclimate фейлится по количеству строк этого файла + if (Files.exists(storagePath.resolve("0tmp"))) { + Files.move( + storagePath.resolve("0tmp"), + storagePath.resolve("0"), + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING + ); + } + + if (Files.exists(indexTmp)) { + Files.move(indexTmp, indexFile, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + } else { + try { + Files.createFile(indexFile); + } catch (FileAlreadyExistsException ignored) { + // it is ok, actually it is normal state + } + } + + List existedFiles = Files.readAllLines(indexFile, StandardCharsets.UTF_8); + List result = new ArrayList<>(existedFiles.size()); + for (String fileName : existedFiles) { + Path file = storagePath.resolve(fileName); + try (FileChannel fileChannel = FileChannel.open(file, StandardOpenOption.READ, StandardOpenOption.WRITE)) { + MemorySegment fileSegment = fileChannel.map( + FileChannel.MapMode.READ_WRITE, + 0, + Files.size(file), + arena + ); + result.add(fileSegment); + } + } + + return result; + } + + private static long indexOf(MemorySegment segment, MemorySegment key) { + long recordsCount = recordsCount(segment); + + long left = 0; + long right = recordsCount - 1; + while (left <= right) { + long mid = (left + right) >>> 1; + + long startOfKey = startOfKey(segment, mid); + long endOfKey = endOfKey(segment, mid); + long mismatch = MemorySegment.mismatch(segment, startOfKey, endOfKey, key, 0, key.byteSize()); + if (mismatch == -1) { + return mid; + } + + if (mismatch == key.byteSize()) { + right = mid - 1; + continue; + } + + if (mismatch == endOfKey - startOfKey) { + left = mid + 1; + continue; + } + + int b1 = Byte.toUnsignedInt(segment.get(ValueLayout.JAVA_BYTE, startOfKey + mismatch)); + int b2 = Byte.toUnsignedInt(key.get(ValueLayout.JAVA_BYTE, mismatch)); + if (b1 > b2) { + right = mid - 1; + } else { + left = mid + 1; + } + } + + return tombstone(left); + } + + private static long recordsCount(MemorySegment segment) { + long indexSize = indexSize(segment); + return indexSize / Long.BYTES / 2; + } + + private static long indexSize(MemorySegment segment) { + return segment.get(ValueLayout.JAVA_LONG_UNALIGNED, 0); + } + + private static Iterator> iterator(MemorySegment page, MemorySegment from, MemorySegment to) { + long recordIndexFrom = from == null ? 0 : normalize(indexOf(page, from)); + long recordIndexTo = to == null ? recordsCount(page) : normalize(indexOf(page, to)); + long recordsCount = recordsCount(page); + + return new Iterator<>() { + long index = recordIndexFrom; + + @Override + public boolean hasNext() { + return index < recordIndexTo; + } + + @Override + public Entry next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + MemorySegment key = slice(page, startOfKey(page, index), endOfKey(page, index)); + long startOfValue = startOfValue(page, index); + MemorySegment value = + startOfValue < 0 + ? null + : slice(page, startOfValue, endOfValue(page, index, recordsCount)); + index++; + return new BaseEntry<>(key, value); + } + }; + } + + private static MemorySegment slice(MemorySegment page, long start, long end) { + return page.asSlice(start, end - start); + } + + private static long startOfKey(MemorySegment segment, long recordIndex) { + return segment.get(ValueLayout.JAVA_LONG_UNALIGNED, recordIndex * 2 * Long.BYTES); + } + + private static long endOfKey(MemorySegment segment, long recordIndex) { + return normalizedStartOfValue(segment, recordIndex); + } + + private static long normalizedStartOfValue(MemorySegment segment, long recordIndex) { + return normalize(startOfValue(segment, recordIndex)); + } + + private static long startOfValue(MemorySegment segment, long recordIndex) { + return segment.get(ValueLayout.JAVA_LONG_UNALIGNED, recordIndex * 2 * Long.BYTES + Long.BYTES); + } + + private static long endOfValue(MemorySegment segment, long recordIndex, long recordsCount) { + if (recordIndex < recordsCount - 1) { + return startOfKey(segment, recordIndex + 1); + } + return segment.byteSize(); + } + + private static long tombstone(long offset) { + return 1L << 63 | offset; + } + + private static long normalize(long value) { + return value & ~(1L << 63); + } + +} diff --git a/src/main/java/ru/vk/itmo/danilinandrew/DiskStorageWithCompact.java b/src/main/java/ru/vk/itmo/danilinandrew/DiskStorageWithCompact.java new file mode 100644 index 000000000..8dff5a2b1 --- /dev/null +++ b/src/main/java/ru/vk/itmo/danilinandrew/DiskStorageWithCompact.java @@ -0,0 +1,126 @@ +package ru.vk.itmo.danilinandrew; + +import ru.vk.itmo.Entry; + +import java.io.IOException; +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.nio.channels.FileChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.util.Iterator; + +public class DiskStorageWithCompact { + private final DiskStorage diskStorage; + private static final String INDEX_FILE = "index.idx"; + private static final String TMP_FILE = "index.tmp"; + + private static final String TMP_COMPACTED_FILE = "0tmp"; + private static final String COMPACTED_FILE = "0"; + + public DiskStorageWithCompact(DiskStorage diskStorage) { + this.diskStorage = diskStorage; + } + + public Iterator> range( + Iterator> firstIterator, + MemorySegment from, + MemorySegment to + ) { + return diskStorage.range(firstIterator, from, to); + } + + public void compact( + Path storagePath, + Iterator> it1, + Iterator> it2 + ) throws IOException { + long sizeData = 0; + long sizeIndexes = 0; + while (it1.hasNext()) { + Entry entry = it1.next(); + if (entry.value() != null) { + sizeData += entry.key().byteSize(); + sizeData += entry.value().byteSize(); + sizeIndexes++; + } + } + + if (sizeIndexes == 0) { + return; + } + sizeIndexes *= 2 * Long.BYTES; + + final Path indexTmp = storagePath.resolve(TMP_FILE); + final Path indexFile = storagePath.resolve(INDEX_FILE); + + try ( + FileChannel fileChannel = FileChannel.open( + storagePath.resolve(TMP_COMPACTED_FILE), + StandardOpenOption.WRITE, + StandardOpenOption.READ, + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING + ); + Arena writeArena = Arena.ofConfined() + ) { + MemorySegment fileSegment = fileChannel.map( + FileChannel.MapMode.READ_WRITE, + 0, + sizeIndexes + sizeData, + writeArena + ); + long offsetIndexes = 0; + long currentIndex = sizeIndexes; + while (it2.hasNext()) { + Entry currentEntry = it2.next(); + if (currentEntry.value() != null) { + fileSegment.set(ValueLayout.JAVA_LONG_UNALIGNED, offsetIndexes, currentIndex); + offsetIndexes += Long.BYTES; + + MemorySegment.copy( + currentEntry.key(), + 0, + fileSegment, + currentIndex, + currentEntry.key().byteSize() + ); + currentIndex += currentEntry.key().byteSize(); + + fileSegment.set(ValueLayout.JAVA_LONG_UNALIGNED, offsetIndexes, currentIndex); + offsetIndexes += Long.BYTES; + + MemorySegment.copy( + currentEntry.value(), + 0, + fileSegment, + currentIndex, + currentEntry.value().byteSize() + ); + currentIndex += currentEntry.value().byteSize(); + } + } + + Files.writeString( + indexTmp, + COMPACTED_FILE, + StandardOpenOption.WRITE, + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING + ); + + diskStorage.clearStorage(storagePath); + + Files.move(indexTmp, indexFile, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + Files.move( + storagePath.resolve(TMP_COMPACTED_FILE), + storagePath.resolve(COMPACTED_FILE), + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING + ); + } + } +} diff --git a/src/main/java/ru/vk/itmo/danilinandrew/InMemoryDao.java b/src/main/java/ru/vk/itmo/danilinandrew/InMemoryDao.java deleted file mode 100644 index 71756157e..000000000 --- a/src/main/java/ru/vk/itmo/danilinandrew/InMemoryDao.java +++ /dev/null @@ -1,146 +0,0 @@ -package ru.vk.itmo.danilinandrew; - -import ru.vk.itmo.BaseEntry; -import ru.vk.itmo.Config; -import ru.vk.itmo.Dao; -import ru.vk.itmo.Entry; - -import java.io.IOException; -import java.lang.foreign.Arena; -import java.lang.foreign.MemorySegment; -import java.lang.foreign.ValueLayout; -import java.nio.channels.FileChannel; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardOpenOption; -import java.util.Iterator; -import java.util.concurrent.ConcurrentNavigableMap; -import java.util.concurrent.ConcurrentSkipListMap; - -public class InMemoryDao implements Dao> { - private final ConcurrentNavigableMap> data = - new ConcurrentSkipListMap<>(new MemorySegmentComparator()); - private final Path ssTablePath; - private final Arena readArena = Arena.ofConfined(); - private final MemorySegment mappedMemorySegment; - - public InMemoryDao(Config config) { - ssTablePath = config.basePath().resolve("data.txt"); - - MemorySegment tempMemorySegment; - try (FileChannel fileChannel = FileChannel.open(ssTablePath)) { - long size = Files.size(ssTablePath); - tempMemorySegment = fileChannel.map(FileChannel.MapMode.READ_ONLY, 0, size, readArena); - } catch (IOException e) { - tempMemorySegment = null; - } - - mappedMemorySegment = tempMemorySegment; - } - - @Override - public Entry get(MemorySegment key) { - Entry value = data.get(key); - - if (value != null) { - return value; - } - - if (mappedMemorySegment == null) { - return null; - } - - long offset = 0; - while (offset < mappedMemorySegment.byteSize()) { - long keySize = mappedMemorySegment.get(ValueLayout.JAVA_LONG_UNALIGNED, offset); - offset += Long.BYTES; - - if (keySize != key.byteSize()) { - offset += keySize; - long valueSize = mappedMemorySegment.get(ValueLayout.JAVA_LONG_UNALIGNED, offset); - offset += Long.BYTES + valueSize; - continue; - } - - MemorySegment readKey = mappedMemorySegment.asSlice(offset, keySize); - offset += keySize; - if (key.mismatch(readKey) == -1) { - long valueSize = mappedMemorySegment.get(ValueLayout.JAVA_LONG_UNALIGNED, offset); - offset += Long.BYTES; - MemorySegment readValue = mappedMemorySegment.asSlice(offset, valueSize); - - return new BaseEntry<>(key, readValue); - } - - } - - return null; - } - - @Override - public Iterator> get(MemorySegment from, MemorySegment to) { - if (from == null && to == null) { - return data.values().iterator(); - } - - if (from == null) { - return data.headMap(to).values().iterator(); - } - - if (to == null) { - return data.tailMap(from).values().iterator(); - } - - return data.subMap(from, to).values().iterator(); - } - - @Override - public void upsert(Entry entry) { - if (entry == null) { - return; - } - - data.put(entry.key(), entry); - } - - @Override - public void close() throws IOException { - readArena.close(); - - try (Arena writeArena = Arena.ofConfined()) { - long size = 0; - - for (Entry value : data.values()) { - size += value.key().byteSize() + value.value().byteSize(); - } - - size += 2L * Long.BYTES * size; - - try (FileChannel fileChannel = FileChannel.open( - ssTablePath, - StandardOpenOption.WRITE, - StandardOpenOption.TRUNCATE_EXISTING, - StandardOpenOption.CREATE, - StandardOpenOption.READ - )) { - MemorySegment page = fileChannel.map(FileChannel.MapMode.READ_WRITE, 0, size, writeArena); - - long offset = 0; - - for (Entry value : data.values()) { - page.set(ValueLayout.JAVA_LONG_UNALIGNED, offset, value.key().byteSize()); - offset += Long.BYTES; - - page.asSlice(offset).copyFrom(value.key()); - offset += value.key().byteSize(); - - page.set(ValueLayout.JAVA_LONG_UNALIGNED, offset, value.value().byteSize()); - offset += Long.BYTES; - - page.asSlice(offset).copyFrom(value.value()); - offset += value.value().byteSize(); - } - } - } - } -} diff --git a/src/main/java/ru/vk/itmo/danilinandrew/MemorySegmentComparator.java b/src/main/java/ru/vk/itmo/danilinandrew/MemorySegmentComparator.java deleted file mode 100644 index ed9a37ecf..000000000 --- a/src/main/java/ru/vk/itmo/danilinandrew/MemorySegmentComparator.java +++ /dev/null @@ -1,30 +0,0 @@ -package ru.vk.itmo.danilinandrew; - -import java.lang.foreign.MemorySegment; -import java.lang.foreign.ValueLayout; -import java.util.Comparator; - -public class MemorySegmentComparator implements Comparator { - - @Override - public int compare(MemorySegment o1, MemorySegment o2) { - long mismatch = o1.mismatch(o2); - - if (mismatch == -1) { - return 0; - } - - if (mismatch == o1.byteSize()) { - return -1; - } - - if (mismatch == o2.byteSize()) { - return 1; - } - - return Byte.compare( - o1.get(ValueLayout.JAVA_BYTE, mismatch), - o2.get(ValueLayout.JAVA_BYTE, mismatch) - ); - } -} diff --git a/src/main/java/ru/vk/itmo/danilinandrew/MergeIterator.java b/src/main/java/ru/vk/itmo/danilinandrew/MergeIterator.java new file mode 100644 index 000000000..b6b3b37cb --- /dev/null +++ b/src/main/java/ru/vk/itmo/danilinandrew/MergeIterator.java @@ -0,0 +1,142 @@ +package ru.vk.itmo.danilinandrew; + +import java.util.Collection; +import java.util.Comparator; +import java.util.Iterator; +import java.util.NoSuchElementException; +import java.util.PriorityQueue; + +public class MergeIterator implements Iterator { + + private final PriorityQueue> priorityQueue; + private final Comparator comparator; + + private PeekIterator peek; + + private static class PeekIterator implements Iterator { + + public final int id; + private final Iterator delegate; + private T peek; + + private PeekIterator(int id, Iterator delegate) { + this.id = id; + this.delegate = delegate; + } + + @Override + public boolean hasNext() { + if (peek == null) { + return delegate.hasNext(); + } + return true; + } + + @Override + public T next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + T curPeek = peek(); + this.peek = null; + return curPeek; + } + + private T peek() { + if (peek == null) { + if (!delegate.hasNext()) { + return null; + } + peek = delegate.next(); + } + return peek; + } + } + + public MergeIterator(Collection> iterators, Comparator comparator) { + this.comparator = comparator; + Comparator> peekComp = (o1, o2) -> comparator.compare(o1.peek(), o2.peek()); + priorityQueue = new PriorityQueue<>( + iterators.size(), + peekComp.thenComparing(o -> -o.id) + ); + + int id = 0; + for (Iterator iterator : iterators) { + if (iterator.hasNext()) { + priorityQueue.add(new PeekIterator<>(id++, iterator)); + } + } + } + + private PeekIterator peek() { + while (peek == null) { + peek = priorityQueue.poll(); + if (peek == null) { + return null; + } + + findElement(); + + if (peek.peek() == null) { + peek = null; + continue; + } + + if (skip(peek.peek())) { + peek.next(); + if (peek.hasNext()) { + priorityQueue.add(peek); + } + peek = null; + } + } + + return peek; + } + + private void findElement() { + while (true) { + PeekIterator next = priorityQueue.peek(); + if (next == null) { + break; + } + + int compare = comparator.compare(peek.peek(), next.peek()); + if (compare == 0) { + PeekIterator poll = priorityQueue.poll(); + if (poll != null) { + poll.next(); + if (poll.hasNext()) { + priorityQueue.add(poll); + } + } + } else { + break; + } + } + } + + protected boolean skip(T t) { + return t == null; + } + + @Override + public boolean hasNext() { + return peek() != null; + } + + @Override + public T next() { + PeekIterator curPeek = peek(); + if (curPeek == null) { + throw new NoSuchElementException(); + } + T next = curPeek.next(); + this.peek = null; + if (curPeek.hasNext()) { + priorityQueue.add(curPeek); + } + return next; + } +} diff --git a/src/main/java/ru/vk/itmo/danilinandrew/StorageDao.java b/src/main/java/ru/vk/itmo/danilinandrew/StorageDao.java new file mode 100644 index 000000000..59baef5a4 --- /dev/null +++ b/src/main/java/ru/vk/itmo/danilinandrew/StorageDao.java @@ -0,0 +1,121 @@ +package ru.vk.itmo.danilinandrew; + +import ru.vk.itmo.Config; +import ru.vk.itmo.Dao; +import ru.vk.itmo.Entry; + +import java.io.IOException; +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collections; +import java.util.Comparator; +import java.util.Iterator; +import java.util.NavigableMap; +import java.util.concurrent.ConcurrentSkipListMap; + +public class StorageDao implements Dao> { + + private final Comparator comparator = StorageDao::compare; + private final NavigableMap> storage = new ConcurrentSkipListMap<>(comparator); + private final Arena arena; + private final DiskStorageWithCompact diskStorage; + private final Path path; + + public StorageDao(Config config) throws IOException { + this.path = config.basePath().resolve("data"); + Files.createDirectories(path); + + arena = Arena.ofShared(); + + this.diskStorage = new DiskStorageWithCompact( + new DiskStorage(DiskStorage.loadOrRecover(path, arena)) + ); + } + + static int compare(MemorySegment memorySegment1, MemorySegment memorySegment2) { + long mismatch = memorySegment1.mismatch(memorySegment2); + if (mismatch == -1) { + return 0; + } + + if (mismatch == memorySegment1.byteSize()) { + return -1; + } + + if (mismatch == memorySegment2.byteSize()) { + return 1; + } + byte b1 = memorySegment1.get(ValueLayout.JAVA_BYTE, mismatch); + byte b2 = memorySegment2.get(ValueLayout.JAVA_BYTE, mismatch); + return Byte.compare(b1, b2); + } + + @Override + public Iterator> get(MemorySegment from, MemorySegment to) { + return diskStorage.range(getInMemory(from, to), from, to); + } + + private Iterator> getInMemory(MemorySegment from, MemorySegment to) { + if (from == null && to == null) { + return storage.values().iterator(); + } + if (from == null) { + return storage.headMap(to).values().iterator(); + } + if (to == null) { + return storage.tailMap(from).values().iterator(); + } + return storage.subMap(from, to).values().iterator(); + } + + @Override + public void upsert(Entry entry) { + storage.put(entry.key(), entry); + } + + @Override + public Entry get(MemorySegment key) { + Entry entry = storage.get(key); + if (entry != null) { + if (entry.value() == null) { + return null; + } + return entry; + } + + Iterator> iterator = diskStorage.range(Collections.emptyIterator(), key, null); + + if (!iterator.hasNext()) { + return null; + } + Entry next = iterator.next(); + if (compare(next.key(), key) == 0) { + return next; + } + return null; + } + + @Override + public void compact() throws IOException { + Iterator> allElementsIterator1 = all(); + Iterator> allElementsIterator2 = all(); + diskStorage.compact(path, allElementsIterator1, allElementsIterator2); + storage.clear(); + } + + @Override + public void close() throws IOException { + if (!arena.scope().isAlive()) { + return; + } + + arena.close(); + + if (!storage.isEmpty()) { + DiskStorage.save(path, storage.values()); + } + } +} diff --git a/src/main/java/ru/vk/itmo/test/danilinandrew/MyFactory.java b/src/main/java/ru/vk/itmo/test/danilinandrew/MyFactory.java index 0125a828c..9a48a2a56 100644 --- a/src/main/java/ru/vk/itmo/test/danilinandrew/MyFactory.java +++ b/src/main/java/ru/vk/itmo/test/danilinandrew/MyFactory.java @@ -3,14 +3,15 @@ import ru.vk.itmo.Config; import ru.vk.itmo.Dao; import ru.vk.itmo.Entry; -import ru.vk.itmo.danilinandrew.InMemoryDao; +import ru.vk.itmo.danilinandrew.StorageDao; import ru.vk.itmo.test.DaoFactory; +import java.io.IOException; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; import java.nio.charset.StandardCharsets; -@DaoFactory(stage = 2) +@DaoFactory(stage = 4) public class MyFactory implements DaoFactory.Factory> { @Override public String toString(MemorySegment memorySegment) { @@ -23,8 +24,8 @@ public MemorySegment fromString(String data) { } @Override - public Dao> createDao(Config config) { - return new InMemoryDao(config); + public Dao> createDao(Config config) throws IOException { + return new StorageDao(config); } @Override From e8663fc3a1861ed22d08d1f58c43e2106d2819b4 Mon Sep 17 00:00:00 2001 From: Vadim Ryabov <70646736+holeyko@users.noreply.github.com> Date: Sun, 26 Nov 2023 00:53:42 +0300 Subject: [PATCH 06/20] =?UTF-8?q?=D0=A0=D1=8F=D0=B1=D0=BE=D0=B2=20=D0=92?= =?UTF-8?q?=D0=B0=D0=B4=D0=B8=D0=BC=20=D0=A1=D0=B5=D1=80=D0=B3=D0=B5=D0=B5?= =?UTF-8?q?=D0=B2=D0=B8=D1=87,=20HW-4,=20=D0=98=D0=A2=D0=9C=D0=9E,=20M3335?= =?UTF-8?q?=20(#238)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Vadim Ryabov Co-authored-by: Vadim Tsesko Co-authored-by: Alexey Shik <58121508+AlexeyShik@users.noreply.github.com> Co-authored-by: fo-xi <60314037+fo-xi@users.noreply.github.com> Co-authored-by: Igor Lopatinskii <9440533@gmail.com> Co-authored-by: Dmitry Osokin <114069284+osokindm@users.noreply.github.com> Co-authored-by: Alexander Peskov <73636600+aosandy@users.noreply.github.com> Co-authored-by: Alexey <54590439+MegaVerkruzo@users.noreply.github.com> Co-authored-by: Nikita <31010564+y0f0@users.noreply.github.com> Co-authored-by: Pologov Nikita Evgenyevich Co-authored-by: Maxim Trofimov <57316041+trofik00777@users.noreply.github.com> Co-authored-by: croil <113173579+croil@users.noreply.github.com> Co-authored-by: PetyaVasya <37338356+PetyaVasya@users.noreply.github.com> Co-authored-by: Viktor <36002722+vitekkor@users.noreply.github.com> Co-authored-by: vitekkor Co-authored-by: atimofeyev Co-authored-by: Vadim Ershov <53083986+vadim01er@users.noreply.github.com> Co-authored-by: Timofeev Kirill Co-authored-by: Artyom Drozdov Co-authored-by: Kirill Timofeev Co-authored-by: Danil Mozzhevilov <38257473+Dmozze@users.noreply.github.com> Co-authored-by: dmozze Co-authored-by: Vladislav Kovalchuk <72710736+Dalvikk@users.noreply.github.com> Co-authored-by: Georgii Dalbeev <55684784+goshadalbeev@users.noreply.github.com> Co-authored-by: Ilya Kriushenkov --- .../vk/itmo/test/ryabovvadim/InMemoryDao.java | 138 +++++---- .../test/ryabovvadim/InMemoryDaoFactory.java | 2 +- .../ru/vk/itmo/test/ryabovvadim/SSTable.java | 266 +++++++++--------- .../iterators/EntrySkipNullsIterator.java | 42 +++ .../ryabovvadim/iterators/LazyIterator.java | 5 + .../test/ryabovvadim/utils/FileUtils.java | 20 +- .../ryabovvadim/utils/MemorySegmentUtils.java | 11 + .../test/ryabovvadim/utils/NumberUtils.java | 10 + 8 files changed, 288 insertions(+), 206 deletions(-) create mode 100644 src/main/java/ru/vk/itmo/test/ryabovvadim/iterators/EntrySkipNullsIterator.java diff --git a/src/main/java/ru/vk/itmo/test/ryabovvadim/InMemoryDao.java b/src/main/java/ru/vk/itmo/test/ryabovvadim/InMemoryDao.java index 392585fa2..74006fe9a 100644 --- a/src/main/java/ru/vk/itmo/test/ryabovvadim/InMemoryDao.java +++ b/src/main/java/ru/vk/itmo/test/ryabovvadim/InMemoryDao.java @@ -3,12 +3,14 @@ import ru.vk.itmo.Config; import ru.vk.itmo.Dao; import ru.vk.itmo.Entry; +import ru.vk.itmo.test.ryabovvadim.iterators.EntrySkipNullsIterator; import ru.vk.itmo.test.ryabovvadim.iterators.FutureIterator; import ru.vk.itmo.test.ryabovvadim.iterators.GatheringIterator; import ru.vk.itmo.test.ryabovvadim.iterators.LazyIterator; import ru.vk.itmo.test.ryabovvadim.iterators.PriorityIterator; import ru.vk.itmo.test.ryabovvadim.utils.FileUtils; import ru.vk.itmo.test.ryabovvadim.utils.MemorySegmentUtils; +import ru.vk.itmo.test.ryabovvadim.utils.NumberUtils; import java.io.IOException; import java.lang.foreign.Arena; @@ -36,7 +38,10 @@ public class InMemoryDao implements Dao> { private final ConcurrentNavigableMap> memoryTable = new ConcurrentSkipListMap<>(MemorySegmentUtils::compareMemorySegments); private final Config config; - private final List ssTables = new ArrayList<>(); + + private final NavigableSet ssTables = new TreeSet<>( + Comparator.comparingLong(SSTable::getId).reversed() + ); public InMemoryDao() throws IOException { this(null); @@ -49,29 +54,9 @@ public InMemoryDao(Config config) throws IOException { } if (Files.notExists(config.basePath())) { - Files.createDirectory(config.basePath()); - } - - NavigableSet dataFileNumbers = new TreeSet<>(Comparator.reverseOrder()); - Files.walkFileTree(config.basePath(), Set.of(), 1, new SimpleFileVisitor<>() { - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { - if (file.getFileName().toString().endsWith("." + DATA_FILE_EXT)) { - dataFileNumbers.add(Long.parseLong( - file.getFileName().toString().substring( - 0, - file.getFileName().toString().indexOf("." + DATA_FILE_EXT) - ) - )); - } - - return FileVisitResult.CONTINUE; - } - }); - - for (long number : dataFileNumbers) { - ssTables.add(new SSTable(config.basePath(), Long.toString(number), arena)); + Files.createDirectories(config.basePath()); } + updateSSTables(true); } @Override @@ -90,7 +75,7 @@ public Iterator> allFrom(MemorySegment from) { return all(); } - return makeIteratorWithSkipNulls(memoryTable.tailMap(from), from, null); + return makeIteratorWithSkipNulls(memoryTable.tailMap(from), load(from, null)); } @Override @@ -99,12 +84,12 @@ public Iterator> allTo(MemorySegment to) { return all(); } - return makeIteratorWithSkipNulls(memoryTable.headMap(to), null, to); + return makeIteratorWithSkipNulls(memoryTable.headMap(to), load(null, to)); } @Override public Iterator> all() { - return makeIteratorWithSkipNulls(memoryTable, null, null); + return makeIteratorWithSkipNulls(memoryTable, load(null, null)); } @Override @@ -116,7 +101,7 @@ public Iterator> get(MemorySegment from, MemorySegment to) return allFrom(from); } - return makeIteratorWithSkipNulls(memoryTable.subMap(from, to), from, to); + return makeIteratorWithSkipNulls(memoryTable.subMap(from, to), load(from, to)); } private Entry load(MemorySegment key) { @@ -155,22 +140,20 @@ private Entry handleDeletededEntry(Entry entry) { } private FutureIterator> makeIteratorWithSkipNulls( - Map> entries, - MemorySegment from, - MemorySegment to + Map> memoryEntries, + List>> loadedIterators ) { - List>> loadedIterators = load(from, to); - Iterator> entriesIterator = entries.values().iterator(); + Iterator> entriesIterator = memoryEntries.values().iterator(); + + if (loadedIterators.isEmpty()) { + return new EntrySkipNullsIterator(entriesIterator); + } int priority = 0; List>> priorityIterators = new ArrayList<>(); if (entriesIterator.hasNext()) { - priorityIterators.add(new PriorityIterator<>( - new LazyIterator<>(entriesIterator::next, entriesIterator::hasNext), - priority - )); - ++priority; + priorityIterators.add(new PriorityIterator<>(new LazyIterator<>(entriesIterator), priority++)); } for (FutureIterator> it : loadedIterators) { priorityIterators.add(new PriorityIterator<>(it, priority++)); @@ -179,37 +162,13 @@ private FutureIterator> makeIteratorWithSkipNulls( GatheringIterator> gatheringIterator = new GatheringIterator<>( priorityIterators, Comparator.comparing( - it -> ((PriorityIterator>) it).showNext().key(), + (PriorityIterator> it) -> it.showNext().key(), MemorySegmentUtils::compareMemorySegments - ).thenComparingInt(it -> ((PriorityIterator>) it).getPriority()), + ).thenComparingInt(PriorityIterator::getPriority), Comparator.comparing(Entry::key, MemorySegmentUtils::compareMemorySegments) ); - return new FutureIterator<>() { - @Override - public Entry showNext() { - skipNulls(); - return gatheringIterator.showNext(); - } - - @Override - public boolean hasNext() { - skipNulls(); - return gatheringIterator.hasNext(); - } - - @Override - public Entry next() { - skipNulls(); - return gatheringIterator.next(); - } - - private void skipNulls() { - while (gatheringIterator.hasNext() && gatheringIterator.showNext().value() == null) { - gatheringIterator.next(); - } - } - }; + return new EntrySkipNullsIterator(gatheringIterator); } @Override @@ -220,22 +179,57 @@ public void upsert(Entry entry) { @Override public void flush() throws IOException { if (existsPath() && !memoryTable.isEmpty()) { - String nameSavedTable = saveMemoryTable(config.basePath()); + long ssTableId = saveEntries(memoryTable.values()); memoryTable.clear(); - ssTables.add(new SSTable(config.basePath(), nameSavedTable, arena)); + ssTables.add(new SSTable(config.basePath(), ssTableId, arena)); + } + } + + @Override + public void compact() throws IOException { + if (existsPath()) { + saveEntries(this::all); + + for (SSTable ssTable : ssTables) { + ssTable.delete(); + } + ssTables.clear(); + memoryTable.clear(); + + updateSSTables(false); } } - private String saveMemoryTable(Path path) throws IOException { - FileUtils.createParentDirectories(config.basePath()); + private void updateSSTables(boolean isStartup) throws IOException { + Files.walkFileTree(config.basePath(), Set.of(), 1, new SimpleFileVisitor<>() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + if (FileUtils.hasExtension(file, DATA_FILE_EXT) && NumberUtils.isInteger( + FileUtils.extractFileName(file, DATA_FILE_EXT) + )) { + ssTables.add(new SSTable( + config.basePath(), + Long.parseLong(FileUtils.extractFileName(file, DATA_FILE_EXT)), + arena + )); + return FileVisitResult.CONTINUE; + } + if (isStartup) { + Files.deleteIfExists(file); + } + return FileVisitResult.CONTINUE; + } + }); + } + + private long saveEntries(Iterable> entries) throws IOException { long maxTableNumber = 0; for (SSTable ssTable : ssTables) { - maxTableNumber = Math.max(maxTableNumber, Long.parseLong(ssTable.getName())); + maxTableNumber = Math.max(maxTableNumber, ssTable.getId()); } - SSTable.save(path, Long.toString(maxTableNumber + 1), memoryTable.values(), arena); - - return Long.toString(maxTableNumber + 1); + SSTable.save(config.basePath(), maxTableNumber + 1, entries, arena); + return maxTableNumber + 1; } private boolean existsPath() { diff --git a/src/main/java/ru/vk/itmo/test/ryabovvadim/InMemoryDaoFactory.java b/src/main/java/ru/vk/itmo/test/ryabovvadim/InMemoryDaoFactory.java index 4d9adabea..636e6227b 100644 --- a/src/main/java/ru/vk/itmo/test/ryabovvadim/InMemoryDaoFactory.java +++ b/src/main/java/ru/vk/itmo/test/ryabovvadim/InMemoryDaoFactory.java @@ -11,7 +11,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; -@DaoFactory(stage = 3) +@DaoFactory(stage = 4) public class InMemoryDaoFactory implements DaoFactory.Factory> { @Override public String toString(MemorySegment memorySegment) { diff --git a/src/main/java/ru/vk/itmo/test/ryabovvadim/SSTable.java b/src/main/java/ru/vk/itmo/test/ryabovvadim/SSTable.java index 7ff1f5af1..953eec515 100644 --- a/src/main/java/ru/vk/itmo/test/ryabovvadim/SSTable.java +++ b/src/main/java/ru/vk/itmo/test/ryabovvadim/SSTable.java @@ -14,41 +14,31 @@ import java.lang.foreign.ValueLayout; import java.nio.channels.FileChannel; import java.nio.channels.FileChannel.MapMode; +import java.nio.file.Files; import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Collection; +import java.nio.file.StandardCopyOption; import java.util.Iterator; -import java.util.List; +import java.util.NoSuchElementException; import static java.nio.file.StandardOpenOption.CREATE; import static java.nio.file.StandardOpenOption.READ; +import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING; import static java.nio.file.StandardOpenOption.WRITE; public class SSTable { - private final String name; + private final Path parentPath; + private final long id; private final MemorySegment data; private final int countRecords; - private final List offsets; - public SSTable(Path prefix, String name, Arena arena) throws IOException { - this.name = name; - - Path dataFile = FileUtils.makePath(prefix, name, FileUtils.DATA_FILE_EXT); - Path offsetsFile = FileUtils.makePath(prefix, name, FileUtils.OFFSETS_FILE_EXT); + public SSTable(Path parentPath, long id, Arena arena) throws IOException { + this.parentPath = parentPath; + this.id = id; + Path dataFile = getDataFilePath(); try (FileChannel dataFileChannel = FileChannel.open(dataFile, READ)) { - try (FileChannel offsetsFileChannel = FileChannel.open(offsetsFile, READ)) { - this.data = dataFileChannel.map(MapMode.READ_ONLY, 0, dataFileChannel.size(), arena); - this.countRecords = (int) (offsetsFileChannel.size() / Long.BYTES); - this.offsets = new ArrayList<>(); - - MemorySegment offsetsSegment = offsetsFileChannel.map( - MapMode.READ_ONLY, 0, offsetsFileChannel.size(), arena - ); - for (int i = 0; i < countRecords; ++i) { - offsets.add(offsetsSegment.get(ValueLayout.JAVA_LONG, i * Long.BYTES)); - } - } + this.data = dataFileChannel.map(MapMode.READ_ONLY, 0, dataFileChannel.size(), arena); + this.countRecords = (int) (this.data.get(ValueLayout.JAVA_LONG, 0) / Long.BYTES); } } @@ -57,7 +47,7 @@ public Entry findEntry(MemorySegment key) { if (offsetIndex < 0) { return null; } - return new BaseEntry<>(key, readValue(getRecordInfo(offsets.get(offsetIndex)))); + return new BaseEntry<>(key, readValue(getRecordInfo(getOffset(offsetIndex)))); } public FutureIterator> findEntries(MemorySegment from, MemorySegment to) { @@ -72,7 +62,7 @@ public FutureIterator> findEntries(MemorySegment from, Memo toIndex = toOffsetIndex < 0 ? -toOffsetIndex : toOffsetIndex; } - Iterator offsetsIterator = offsets.subList(fromIndex, toIndex).iterator(); + Iterator offsetsIterator = getOffsetIterator(fromIndex, toIndex); return new LazyIterator<>( () -> { RecordInfo recordInfo = getRecordInfo(offsetsIterator.next()); @@ -82,19 +72,14 @@ public FutureIterator> findEntries(MemorySegment from, Memo ); } - public String getName() { - return name; - } - private int binSearchIndex(MemorySegment key, boolean lowerBound) { int l = -1; int r = countRecords; - while (l + 1 < r) { int mid = (l + r) / 2; - RecordInfo recordInfo = getRecordInfo(offsets.get(mid)); + RecordInfo recordInfo = getRecordInfo(getOffset(mid)); int compareResult = MemorySegmentUtils.compareMemorySegments( - data, recordInfo.getKeyOffset(), recordInfo.getValueOffset(), + data, recordInfo.keyOffset(), recordInfo.valueOffset(), key, 0, key.byteSize() ); @@ -111,150 +96,169 @@ private int binSearchIndex(MemorySegment key, boolean lowerBound) { } private RecordInfo getRecordInfo(long recordOffset) { - long curOffset = recordOffset; - ++curOffset; - - byte sizeInfo = data.get(ValueLayout.JAVA_BYTE, curOffset); - ++curOffset; + long curOffset = recordOffset + 1; + byte sizeInfo = data.get(ValueLayout.JAVA_BYTE, curOffset++); int keySizeSize = sizeInfo >> 4; int valueSizeSize = sizeInfo & 0xf; byte[] keySizeInBytes = new byte[keySizeSize]; for (int i = 0; i < keySizeSize; ++i) { - keySizeInBytes[i] = data.get(ValueLayout.JAVA_BYTE, curOffset); - ++curOffset; + keySizeInBytes[i] = data.get(ValueLayout.JAVA_BYTE, curOffset++); } byte[] valueSizeInBytes = new byte[valueSizeSize]; for (int i = 0; i < valueSizeSize; ++i) { - valueSizeInBytes[i] = data.get(ValueLayout.JAVA_BYTE, curOffset); - ++curOffset; + valueSizeInBytes[i] = data.get(ValueLayout.JAVA_BYTE, curOffset++); } long keySize = NumberUtils.fromBytes(keySizeInBytes); long valueSize = NumberUtils.fromBytes(valueSizeInBytes); byte meta = data.get(ValueLayout.JAVA_BYTE, recordOffset); - return new RecordInfo(meta, keySize, curOffset, valueSize, curOffset + keySize); } private MemorySegment readKey(RecordInfo recordInfo) { - return data.asSlice(recordInfo.getKeyOffset(), recordInfo.getKeySize()); + return data.asSlice(recordInfo.keyOffset(), recordInfo.keySize()); } private MemorySegment readValue(RecordInfo recordInfo) { - if (SSTableMeta.isRemovedValue(recordInfo.getMeta())) { + if (SSTableMeta.isRemovedValue(recordInfo.meta())) { return null; } - return data.asSlice(recordInfo.getValueOffset(), recordInfo.getValueSize()); + return data.asSlice(recordInfo.valueOffset(), recordInfo.valueSize()); + } + + private long getOffset(int index) { + return data.get(ValueLayout.JAVA_LONG, index * Long.BYTES); + } + + private Iterator getOffsetIterator(int fromIndex, int toIndex) { + return new Iterator<>() { + private int curIndex = fromIndex; + + @Override + public boolean hasNext() { + return curIndex < toIndex; + } + + @Override + public Long next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + return getOffset(curIndex++); + } + }; } public static void save( Path prefix, - String name, - Collection> entries, + long id, + Iterable> entries, Arena arena ) throws IOException { - Path dataFile = FileUtils.makePath(prefix, name, FileUtils.DATA_FILE_EXT); - Path offsetsFile = FileUtils.makePath(prefix, name, FileUtils.OFFSETS_FILE_EXT); - - try (FileChannel dataFileChannel = FileChannel.open(dataFile, CREATE, WRITE, READ)) { - try (FileChannel offsetsFileChannel = FileChannel.open(offsetsFile, CREATE, WRITE, READ)) { - long dataSize = 0; - for (Entry entry : entries) { - dataSize += 2; - MemorySegment key = entry.key(); - MemorySegment value = entry.value(); - - dataSize += NumberUtils.toBytes(key.byteSize()).length + key.byteSize(); - if (value != null) { - dataSize += NumberUtils.toBytes(value.byteSize()).length + value.byteSize(); - } - } + long dataSize = 0; + int countRecords = 0; + for (Entry entry : entries) { + ++countRecords; + dataSize += 2; + MemorySegment key = entry.key(); + MemorySegment value = entry.value(); + + dataSize += NumberUtils.toBytes(key.byteSize()).length + key.byteSize(); + if (value != null) { + dataSize += NumberUtils.toBytes(value.byteSize()).length + value.byteSize(); + } + } + + if (countRecords == 0) { + return; + } + + Path tmpDataFile = FileUtils.makePath(prefix, Long.toString(id), FileUtils.TMP_FILE_EXT); + try (FileChannel dataFileChannel = FileChannel.open(tmpDataFile, CREATE, WRITE, READ, TRUNCATE_EXISTING)) { + long dataOffset = (long) countRecords * Long.BYTES; + MemorySegment dataSegment = dataFileChannel.map( + MapMode.READ_WRITE, + 0, + dataSize + dataOffset, + arena + ); - MemorySegment dataSegment = dataFileChannel.map( - MapMode.READ_WRITE, 0, dataSize, arena - ); - MemorySegment offsetsSegment = offsetsFileChannel.map( - MapMode.READ_WRITE, 0, Long.BYTES * entries.size(), arena - ); - long dataSegmentOffset = 0; - long offsetsSegmentOffset = 0; - - for (Entry entry : entries) { - MemorySegment key = entry.key(); - MemorySegment value = entry.value(); - byte[] keySizeInBytes = NumberUtils.toBytes(key.byteSize()); - byte[] valueSizeInBytes = value == null - ? new byte[0] - : NumberUtils.toBytes(value.byteSize()); - - offsetsSegment.set(ValueLayout.JAVA_LONG, offsetsSegmentOffset, dataSegmentOffset); - offsetsSegmentOffset += Long.BYTES; - - byte meta = buildMeta(entry); - byte sizeInfo = (byte) ((keySizeInBytes.length << 4) | valueSizeInBytes.length); - - dataSegment.set(ValueLayout.JAVA_BYTE, dataSegmentOffset, meta); - dataSegmentOffset += 1; - dataSegment.set(ValueLayout.JAVA_BYTE, dataSegmentOffset, sizeInfo); - dataSegmentOffset += 1; - - MemorySegment.copy( - keySizeInBytes, - 0, - dataSegment, - ValueLayout.JAVA_BYTE, - dataSegmentOffset, - keySizeInBytes.length - ); - dataSegmentOffset += keySizeInBytes.length; - MemorySegment.copy( - valueSizeInBytes, - 0, - dataSegment, - ValueLayout.JAVA_BYTE, - dataSegmentOffset, - valueSizeInBytes.length - ); - dataSegmentOffset += valueSizeInBytes.length; - MemorySegment.copy(key, 0, dataSegment, dataSegmentOffset, key.byteSize()); - dataSegmentOffset += key.byteSize(); - if (value != null) { - MemorySegment.copy( - value, 0, dataSegment, dataSegmentOffset, value.byteSize() - ); - dataSegmentOffset += value.byteSize(); - } + int curEntryNumber = 0; + for (Entry entry : entries) { + dataSegment.set(ValueLayout.JAVA_LONG, curEntryNumber * Long.BYTES, dataOffset); + + MemorySegment key = entry.key(); + MemorySegment value = entry.value(); + byte[] keySizeInBytes = NumberUtils.toBytes(key.byteSize()); + byte[] valueSizeInBytes = value == null + ? new byte[0] + : NumberUtils.toBytes(value.byteSize()); + + byte meta = SSTableMeta.buildMeta(entry); + byte sizeInfo = (byte) ((keySizeInBytes.length << 4) | valueSizeInBytes.length); + dataSegment.set(ValueLayout.JAVA_BYTE, dataOffset++, meta); + dataSegment.set(ValueLayout.JAVA_BYTE, dataOffset++, sizeInfo); + + MemorySegmentUtils.copyByteArray(keySizeInBytes, dataSegment, dataOffset); + dataOffset += keySizeInBytes.length; + MemorySegmentUtils.copyByteArray(valueSizeInBytes, dataSegment, dataOffset); + dataOffset += valueSizeInBytes.length; + MemorySegment.copy(key, 0, dataSegment, dataOffset, key.byteSize()); + dataOffset += key.byteSize(); + if (value != null) { + MemorySegment.copy(value, 0, dataSegment, dataOffset, value.byteSize()); + dataOffset += value.byteSize(); } + + ++curEntryNumber; } } + + Path dataFile = FileUtils.makePath(prefix, Long.toString(id), FileUtils.DATA_FILE_EXT); + Files.move(tmpDataFile, dataFile, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); } - private static byte buildMeta(Entry entry) { - byte meta = 0; + public void delete() throws IOException { + Files.deleteIfExists(getDataFilePath()); + } - if (entry.value() == null) { - meta |= SSTableMeta.REMOVE_VALUE; - } - return meta; + private Path getDataFilePath() { + return FileUtils.makePath(parentPath, Long.toString(id), FileUtils.DATA_FILE_EXT); } - private static class SSTableMeta { + public long getId() { + return id; + } + + private static final class SSTableMeta { private static final byte REMOVE_VALUE = 0x1; - private static boolean isRemovedValue(byte meta) { + public static boolean isRemovedValue(byte meta) { return (meta & REMOVE_VALUE) == REMOVE_VALUE; } + + public static byte buildMeta(Entry entry) { + byte meta = 0; + + if (entry.value() == null) { + meta |= SSTableMeta.REMOVE_VALUE; + } + return meta; + } + + private SSTableMeta() { + } } - private static class RecordInfo { + private static final class RecordInfo { private final byte meta; private final long keySize; private final long keyOffset; private final long valueSize; private final long valueOffset; - public RecordInfo( + private RecordInfo( byte meta, long keySize, long keyOffset, @@ -268,23 +272,23 @@ public RecordInfo( this.valueOffset = valueOffset; } - public byte getMeta() { + public byte meta() { return meta; } - public long getKeySize() { + public long keySize() { return keySize; } - public long getKeyOffset() { + public long keyOffset() { return keyOffset; } - public long getValueSize() { + public long valueSize() { return valueSize; } - public long getValueOffset() { + public long valueOffset() { return valueOffset; } } diff --git a/src/main/java/ru/vk/itmo/test/ryabovvadim/iterators/EntrySkipNullsIterator.java b/src/main/java/ru/vk/itmo/test/ryabovvadim/iterators/EntrySkipNullsIterator.java new file mode 100644 index 000000000..c7216429d --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/ryabovvadim/iterators/EntrySkipNullsIterator.java @@ -0,0 +1,42 @@ +package ru.vk.itmo.test.ryabovvadim.iterators; + +import ru.vk.itmo.Entry; + +import java.lang.foreign.MemorySegment; +import java.util.Iterator; + +public class EntrySkipNullsIterator implements FutureIterator> { + private final FutureIterator> delegate; + + public EntrySkipNullsIterator(Iterator> delegate) { + this.delegate = new LazyIterator<>(delegate); + } + + public EntrySkipNullsIterator(FutureIterator> delegate) { + this.delegate = delegate; + } + + @Override + public Entry showNext() { + skipNulls(); + return delegate.showNext(); + } + + @Override + public boolean hasNext() { + skipNulls(); + return delegate.hasNext(); + } + + @Override + public Entry next() { + skipNulls(); + return delegate.next(); + } + + private void skipNulls() { + while (delegate.hasNext() && delegate.showNext().value() == null) { + delegate.next(); + } + } +} diff --git a/src/main/java/ru/vk/itmo/test/ryabovvadim/iterators/LazyIterator.java b/src/main/java/ru/vk/itmo/test/ryabovvadim/iterators/LazyIterator.java index b146f4be1..7a0b772a0 100644 --- a/src/main/java/ru/vk/itmo/test/ryabovvadim/iterators/LazyIterator.java +++ b/src/main/java/ru/vk/itmo/test/ryabovvadim/iterators/LazyIterator.java @@ -1,5 +1,6 @@ package ru.vk.itmo.test.ryabovvadim.iterators; +import java.util.Iterator; import java.util.function.Supplier; public class LazyIterator implements FutureIterator { @@ -7,6 +8,10 @@ public class LazyIterator implements FutureIterator { private final Supplier hasNextEntry; private T next; + public LazyIterator(Iterator iterator) { + this(iterator::next, iterator::hasNext); + } + public LazyIterator(Supplier getEntry, Supplier hasNextEntry) { this.loadEntry = getEntry; this.hasNextEntry = hasNextEntry; diff --git a/src/main/java/ru/vk/itmo/test/ryabovvadim/utils/FileUtils.java b/src/main/java/ru/vk/itmo/test/ryabovvadim/utils/FileUtils.java index 22da19507..bb747ba87 100644 --- a/src/main/java/ru/vk/itmo/test/ryabovvadim/utils/FileUtils.java +++ b/src/main/java/ru/vk/itmo/test/ryabovvadim/utils/FileUtils.java @@ -6,7 +6,8 @@ public final class FileUtils { public static final String DATA_FILE_EXT = "data"; - public static final String OFFSETS_FILE_EXT = "offsets"; + public static final String TMP_FILE_EXT = "tmp"; + private static final String FILE_EXTENSION_DELIMITER = "."; public static Path makePath(Path prefix, String name, String extension) { return Path.of(prefix.toString(), name + "." + extension); @@ -22,7 +23,22 @@ public static void createParentDirectories(Path path) throws IOException { Files.createDirectories(parent); } } - + + public static boolean hasExtension(Path path, String extension) { + return path.getFileName().toString().endsWith(FILE_EXTENSION_DELIMITER + extension); + } + + public static String extractFileName(Path path, String extension) { + String fullFileName = path.getFileName().toString(); + int index = fullFileName.indexOf(FILE_EXTENSION_DELIMITER + extension); + + if (index == -1) { + throw new IllegalArgumentException("File " + path + " doesn't have extension " + extension); + } + + return fullFileName.substring(0, index); + } + private FileUtils() { } } diff --git a/src/main/java/ru/vk/itmo/test/ryabovvadim/utils/MemorySegmentUtils.java b/src/main/java/ru/vk/itmo/test/ryabovvadim/utils/MemorySegmentUtils.java index 822cdb5c1..2cd3c4a63 100644 --- a/src/main/java/ru/vk/itmo/test/ryabovvadim/utils/MemorySegmentUtils.java +++ b/src/main/java/ru/vk/itmo/test/ryabovvadim/utils/MemorySegmentUtils.java @@ -50,6 +50,17 @@ public static int compareMemorySegments( ); } + public static void copyByteArray(byte[] src, MemorySegment dst, long offsetDst) { + MemorySegment.copy( + src, + 0, + dst, + JAVA_BYTE, + offsetDst, + src.length + ); + } + private MemorySegmentUtils() { } } diff --git a/src/main/java/ru/vk/itmo/test/ryabovvadim/utils/NumberUtils.java b/src/main/java/ru/vk/itmo/test/ryabovvadim/utils/NumberUtils.java index ce1f602c4..f099c0cc2 100644 --- a/src/main/java/ru/vk/itmo/test/ryabovvadim/utils/NumberUtils.java +++ b/src/main/java/ru/vk/itmo/test/ryabovvadim/utils/NumberUtils.java @@ -34,6 +34,16 @@ public static byte[] toBytes(long value) { return result; } + + public static boolean isInteger(String s) { + for (int i = 0; i < s.length(); ++i) { + if (!Character.isDigit(s.charAt(i))) { + return false; + } + } + + return true; + } private NumberUtils() { } From 344ab43ddc3488b7438245bb3b18b318f0411bbb Mon Sep 17 00:00:00 2001 From: Vladislav Kovalchuk <72710736+Dalvikk@users.noreply.github.com> Date: Sun, 26 Nov 2023 01:31:51 +0300 Subject: [PATCH 07/20] =?UTF-8?q?=D0=9A=D0=BE=D0=B2=D0=B0=D0=BB=D1=8C?= =?UTF-8?q?=D1=87=D1=83=D0=BA=20=D0=92=D0=BB=D0=B0=D0=B4=D0=B8=D1=81=D0=BB?= =?UTF-8?q?=D0=B0=D0=B2,=20=D0=98=D0=A2=D0=9C=D0=9E=20=D0=A4=D0=98=D0=A2?= =?UTF-8?q?=D0=B8=D0=9F,=20M33371,=20HW=204=20(#247)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Vladislav Kovalchuk Co-authored-by: Vadim Tsesko Co-authored-by: Alexey Shik <58121508+AlexeyShik@users.noreply.github.com> Co-authored-by: Ilya Kriushenkov --- .../AbstractBasedOnSSTableDao.java | 162 ++++++++++------- .../SSTableMemorySegmentWriter.java | 164 ++++++++++++++++++ .../kovalchukvladislav/model/TableInfo.java | 27 +++ .../MemorySegmentDaoFactory.java | 2 +- 4 files changed, 290 insertions(+), 65 deletions(-) create mode 100644 src/main/java/ru/vk/itmo/kovalchukvladislav/SSTableMemorySegmentWriter.java create mode 100644 src/main/java/ru/vk/itmo/kovalchukvladislav/model/TableInfo.java diff --git a/src/main/java/ru/vk/itmo/kovalchukvladislav/AbstractBasedOnSSTableDao.java b/src/main/java/ru/vk/itmo/kovalchukvladislav/AbstractBasedOnSSTableDao.java index 571eba8be..8d2fec029 100644 --- a/src/main/java/ru/vk/itmo/kovalchukvladislav/AbstractBasedOnSSTableDao.java +++ b/src/main/java/ru/vk/itmo/kovalchukvladislav/AbstractBasedOnSSTableDao.java @@ -4,28 +4,30 @@ import ru.vk.itmo.Entry; import ru.vk.itmo.kovalchukvladislav.model.DaoIterator; import ru.vk.itmo.kovalchukvladislav.model.EntryExtractor; +import ru.vk.itmo.kovalchukvladislav.model.TableInfo; import java.io.IOException; import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; -import java.lang.foreign.ValueLayout; import java.nio.channels.FileChannel; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; -import java.nio.file.OpenOption; import java.nio.file.Path; import java.nio.file.StandardOpenOption; import java.util.ArrayList; +import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.Objects; +import java.util.logging.Level; +import java.util.logging.Logger; public abstract class AbstractBasedOnSSTableDao> extends AbstractInMemoryDao { // =================================== // Constants // =================================== - private static final ValueLayout.OfLong LONG_LAYOUT = ValueLayout.JAVA_LONG_UNALIGNED; - private static final String OFFSETS_FILENAME_PREFIX = "offsets_"; private static final String METADATA_FILENAME = "metadata"; + private static final String OFFSETS_FILENAME_PREFIX = "offsets_"; private static final String DB_FILENAME_PREFIX = "db_"; // =================================== @@ -33,57 +35,91 @@ public abstract class AbstractBasedOnSSTableDao> extends A // =================================== private final Path basePath; - private final Path metadataPath; private final Arena arena = Arena.ofShared(); private final EntryExtractor extractor; + private final SSTableMemorySegmentWriter writer; // =================================== // Storages // =================================== - private final int storagesCount; + private int storagesCount; + private volatile boolean closed; private final List dbMappedSegments; private final List offsetMappedSegments; + private final Logger logger = Logger.getLogger(getClass().getSimpleName()); protected AbstractBasedOnSSTableDao(Config config, EntryExtractor extractor) throws IOException { super(extractor); + this.closed = false; + this.storagesCount = 0; this.extractor = extractor; this.basePath = Objects.requireNonNull(config.basePath()); + this.dbMappedSegments = new ArrayList<>(); + this.offsetMappedSegments = new ArrayList<>(); + reloadFilesAndMapToSegment(); + this.writer = new SSTableMemorySegmentWriter<>(basePath, DB_FILENAME_PREFIX, OFFSETS_FILENAME_PREFIX, + METADATA_FILENAME, extractor); + logger.setLevel(Level.OFF); // чтобы не засорять вывод в гитхабе, если такое возможно + } + + // =================================== + // Restoring state + // =================================== + private void reloadFilesAndMapToSegment() throws IOException { if (!Files.exists(basePath)) { Files.createDirectory(basePath); } - this.metadataPath = basePath.resolve(METADATA_FILENAME); + logger.info(() -> String.format("Reloading files from %s", basePath)); + List ssTableIds = getSSTableIds(); + for (String ssTableId : ssTableIds) { + readFileAndMapToSegment(ssTableId); + } + logger.info(() -> String.format("Reloaded %d files", storagesCount)); + } + + private void readFileAndMapToSegment(String timestamp) throws IOException { + Path dbPath = basePath.resolve(DB_FILENAME_PREFIX + timestamp); + Path offsetsPath = basePath.resolve(OFFSETS_FILENAME_PREFIX + timestamp); + if (!Files.exists(dbPath) || !Files.exists(offsetsPath)) { + logger.severe(() -> String.format("File under path %s or %s doesn't exists", dbPath, offsetsPath)); + return; + } + + logger.info(() -> String.format("Reading files with timestamp %s", timestamp)); - this.storagesCount = getCountFromMetadataOrCreate(); - this.dbMappedSegments = new ArrayList<>(storagesCount); - this.offsetMappedSegments = new ArrayList<>(storagesCount); + try (FileChannel dbChannel = FileChannel.open(dbPath, StandardOpenOption.READ); + FileChannel offsetChannel = FileChannel.open(offsetsPath, StandardOpenOption.READ)) { - for (int i = 0; i < storagesCount; i++) { - readFileAndMapToSegment(DB_FILENAME_PREFIX, i, dbMappedSegments); - readFileAndMapToSegment(OFFSETS_FILENAME_PREFIX, i, offsetMappedSegments); + MemorySegment db = dbChannel.map(FileChannel.MapMode.READ_ONLY, 0, Files.size(dbPath), arena); + MemorySegment offsets = offsetChannel.map(FileChannel.MapMode.READ_ONLY, 0, Files.size(offsetsPath), arena); + dbMappedSegments.add(db); + offsetMappedSegments.add(offsets); + storagesCount++; } + logger.info(() -> String.format("Successfully read files with %s timestamp", timestamp)); } - // =================================== - // Restoring state - // =================================== - private int getCountFromMetadataOrCreate() throws IOException { + private List getSSTableIds() throws IOException { + Path metadataPath = basePath.resolve(METADATA_FILENAME); if (!Files.exists(metadataPath)) { - Files.writeString(metadataPath, "0", StandardOpenOption.WRITE, StandardOpenOption.CREATE); - return 0; + return Collections.emptyList(); } - return Integer.parseInt(Files.readString(metadataPath)); + return Files.readAllLines(metadataPath, StandardCharsets.UTF_8); } - private void readFileAndMapToSegment(String filenamePrefix, int index, - List segments) throws IOException { - Path path = basePath.resolve(filenamePrefix + index); - try (FileChannel channel = FileChannel.open(path, StandardOpenOption.READ)) { + private Path[] getAllTablesPath() throws IOException { + List ssTableIds = getSSTableIds(); + int size = ssTableIds.size(); + Path[] files = new Path[2 * size]; - MemorySegment segment = channel.map(FileChannel.MapMode.READ_ONLY, 0, Files.size(path), arena); - segments.add(segment); + for (int i = 0; i < size; i++) { + String id = ssTableIds.get(i); + files[2 * i] = basePath.resolve(DB_FILENAME_PREFIX + id); + files[2 * i + 1] = basePath.resolve(OFFSETS_FILENAME_PREFIX + id); } + return files; } // =================================== @@ -126,64 +162,62 @@ private E findInStorages(D key) { } // =================================== - // Writing data + // Some utils // =================================== - private void writeData() throws IOException { - Path dbPath = basePath.resolve(DB_FILENAME_PREFIX + storagesCount); - Path offsetsPath = basePath.resolve(OFFSETS_FILENAME_PREFIX + storagesCount); - - OpenOption[] options = new OpenOption[] { - StandardOpenOption.READ, - StandardOpenOption.WRITE, - StandardOpenOption.TRUNCATE_EXISTING, - StandardOpenOption.CREATE - }; - - try (FileChannel db = FileChannel.open(dbPath, options); - FileChannel offsets = FileChannel.open(offsetsPath, options); - Arena confinedArena = Arena.ofConfined()) { - - long dbSize = getDAOBytesSize(); - long offsetsSize = (long) dao.size() * Long.BYTES; - MemorySegment fileSegment = db.map(FileChannel.MapMode.READ_WRITE, 0, dbSize, confinedArena); - MemorySegment offsetsSegment = offsets.map(FileChannel.MapMode.READ_WRITE, 0, offsetsSize, confinedArena); - - int i = 0; - long offset = 0; - for (E entry : dao.values()) { - offsetsSegment.setAtIndex(LONG_LAYOUT, i, offset); - i += 1; - offset = extractor.writeEntry(entry, fileSegment, offset); - } - fileSegment.load(); - offsetsSegment.load(); - } - } - private long getDAOBytesSize() { + private TableInfo getInMemoryDaoSizeInfo() { long size = 0; for (E entry : dao.values()) { size += extractor.size(entry); } - return size; + return new TableInfo(dao.size(), size); + } + + private TableInfo getSSTableDaoSizeInfo() { + Iterator allIterator = all(); + long entriesCount = 0; + long daoSize = 0; + + while (allIterator.hasNext()) { + E next = allIterator.next(); + entriesCount++; + daoSize += extractor.size(next); + } + + return new TableInfo(entriesCount, daoSize); } // =================================== // Flush and close // =================================== + @Override public synchronized void flush() throws IOException { - if (!dao.isEmpty()) { - writeData(); - Files.writeString(metadataPath, String.valueOf(storagesCount + 1)); + if (dao.isEmpty()) { + return; } + writer.flush(dao.values().iterator(), getInMemoryDaoSizeInfo()); } @Override public synchronized void close() throws IOException { + if (closed) { + return; + } + flush(); if (arena.scope().isAlive()) { arena.close(); } - flush(); + closed = true; + } + + @Override + public synchronized void compact() throws IOException { + if (storagesCount <= 1 && dao.isEmpty()) { + return; + } + Path[] oldTables = getAllTablesPath(); + writer.compact(all(), getSSTableDaoSizeInfo()); + writer.deleteUnusedFiles(oldTables); } } diff --git a/src/main/java/ru/vk/itmo/kovalchukvladislav/SSTableMemorySegmentWriter.java b/src/main/java/ru/vk/itmo/kovalchukvladislav/SSTableMemorySegmentWriter.java new file mode 100644 index 000000000..077a90c75 --- /dev/null +++ b/src/main/java/ru/vk/itmo/kovalchukvladislav/SSTableMemorySegmentWriter.java @@ -0,0 +1,164 @@ +package ru.vk.itmo.kovalchukvladislav; + +import ru.vk.itmo.Entry; +import ru.vk.itmo.kovalchukvladislav.model.EntryExtractor; +import ru.vk.itmo.kovalchukvladislav.model.TableInfo; + +import java.io.IOException; +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.nio.channels.FileChannel; +import java.nio.file.Files; +import java.nio.file.OpenOption; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.util.Comparator; +import java.util.Iterator; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.stream.Stream; + +public class SSTableMemorySegmentWriter> { + private static final Logger logger = Logger.getLogger(SSTableMemorySegmentWriter.class.getSimpleName()); + private static final OpenOption[] WRITE_OPTIONS = new OpenOption[] { + StandardOpenOption.READ, + StandardOpenOption.WRITE, + StandardOpenOption.TRUNCATE_EXISTING, + StandardOpenOption.CREATE + }; + + private static final StandardCopyOption[] MOVE_OPTIONS = new StandardCopyOption[] { + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING + }; + + private final Path basePath; + private final String metadataFilename; + private final String dbFilenamePrefix; + private final String offsetsFilenamePrefix; + private final EntryExtractor extractor; + + public SSTableMemorySegmentWriter(Path basePath, String dbFilenamePrefix, String offsetsFilenamePrefix, + String metadataFilename, EntryExtractor extractor) { + this.basePath = basePath; + this.dbFilenamePrefix = dbFilenamePrefix; + this.offsetsFilenamePrefix = offsetsFilenamePrefix; + this.metadataFilename = metadataFilename; + this.extractor = extractor; + logger.setLevel(Level.OFF); // чтобы не засорять вывод в гитхабе, если такое возможно + } + + public void compact(Iterator iterator, TableInfo info) throws IOException { + Path tempDirectory = Files.createTempDirectory(null); + String timestamp = String.valueOf(System.currentTimeMillis()); + + Path newSSTable = basePath.resolve(dbFilenamePrefix + timestamp); + Path newOffsetsTable = basePath.resolve(offsetsFilenamePrefix + timestamp); + Path tmpSSTable = tempDirectory.resolve(dbFilenamePrefix + timestamp); + Path tmpOffsetsTable = tempDirectory.resolve(offsetsFilenamePrefix + timestamp); + + logger.info(() -> String.format("Compacting started to dir %s, timestamp %s, info %s", + tempDirectory, timestamp, info)); + + try { + writeData(tempDirectory, timestamp, iterator, info); + Path tmpMetadata = addSSTableId(tempDirectory, timestamp); + Path newMetadata = basePath.resolve(metadataFilename); + + Files.move(tmpSSTable, newSSTable, MOVE_OPTIONS); + Files.move(tmpOffsetsTable, newOffsetsTable, MOVE_OPTIONS); + Files.move(tmpMetadata, newMetadata, MOVE_OPTIONS); + } catch (Exception e) { + deleteUnusedFiles(newSSTable, newOffsetsTable); + throw e; + } finally { + deleteUnusedFiles(tempDirectory); + } + logger.info(() -> String.format("Compacted to dir %s, timestamp %s", basePath, timestamp)); + } + + public void flush(Iterator iterator, TableInfo info) throws IOException { + Path tempDirectory = Files.createTempDirectory(null); + String timestamp = String.valueOf(System.currentTimeMillis()); + + Path newSSTable = basePath.resolve(dbFilenamePrefix + timestamp); + Path newOffsetsTable = basePath.resolve(offsetsFilenamePrefix + timestamp); + Path tmpSSTable = tempDirectory.resolve(dbFilenamePrefix + timestamp); + Path tmpOffsetsTable = tempDirectory.resolve(offsetsFilenamePrefix + timestamp); + + logger.info(() -> String.format("Flushing started to dir %s, timestamp %s, info %s", + tempDirectory, timestamp, info)); + try { + writeData(tempDirectory, timestamp, iterator, info); + + Files.move(tmpSSTable, newSSTable, MOVE_OPTIONS); + Files.move(tmpOffsetsTable, newOffsetsTable, MOVE_OPTIONS); + addSSTableId(basePath, timestamp); + } catch (Exception e) { + deleteUnusedFiles(newSSTable, newOffsetsTable); + throw e; + } finally { + deleteUnusedFilesInDirectory(tempDirectory); + } + logger.info(() -> String.format("Flushed to dir %s, timestamp %s", basePath, timestamp)); + } + + // Удаление ненужных файлов не является чем то критически важным + // Если произойдет исключение, лучше словить и вывести в лог, чем останавливать работу + public void deleteUnusedFiles(Path... files) { + for (Path file : files) { + try { + boolean deleted = Files.deleteIfExists(file); + if (deleted) { + logger.info(() -> String.format("File %s was deleted", file)); + } else { + logger.severe(() -> String.format("File %s not deleted", file)); + } + } catch (IOException e) { + logger.severe(() -> String.format("Error while deleting file %s: %s", file, e.getMessage())); + } + } + } + + private void deleteUnusedFilesInDirectory(Path directory) { + try (Stream files = Files.walk(directory)) { + Path[] array = files.sorted(Comparator.reverseOrder()).toArray(Path[]::new); + deleteUnusedFiles(array); + } catch (Exception e) { + logger.severe(() -> String.format("Error while deleting directory %s: %s", directory, e.getMessage())); + } + } + + private void writeData(Path path, String timestamp, Iterator daoIterator, TableInfo info) throws IOException { + Path dbPath = path.resolve(dbFilenamePrefix + timestamp); + Path offsetsPath = path.resolve(offsetsFilenamePrefix + timestamp); + + try (FileChannel db = FileChannel.open(dbPath, WRITE_OPTIONS); + FileChannel offsets = FileChannel.open(offsetsPath, WRITE_OPTIONS); + Arena arena = Arena.ofConfined()) { + + long offsetsSize = info.getRecordsCount() * Long.BYTES; + MemorySegment fileSegment = db.map(FileChannel.MapMode.READ_WRITE, 0, info.getRecordsSize(), arena); + MemorySegment offsetsSegment = offsets.map(FileChannel.MapMode.READ_WRITE, 0, offsetsSize, arena); + + int i = 0; + long offset = 0; + while (daoIterator.hasNext()) { + E entry = daoIterator.next(); + offsetsSegment.setAtIndex(ValueLayout.JAVA_LONG_UNALIGNED, i, offset); + offset = extractor.writeEntry(entry, fileSegment, offset); + i += 1; + } + + fileSegment.load(); + offsetsSegment.load(); + } + } + + private Path addSSTableId(Path path, String id) throws IOException { + return Files.writeString(path.resolve(metadataFilename), id + System.lineSeparator(), + StandardOpenOption.WRITE, StandardOpenOption.APPEND, StandardOpenOption.CREATE); + } +} diff --git a/src/main/java/ru/vk/itmo/kovalchukvladislav/model/TableInfo.java b/src/main/java/ru/vk/itmo/kovalchukvladislav/model/TableInfo.java new file mode 100644 index 000000000..2331fe563 --- /dev/null +++ b/src/main/java/ru/vk/itmo/kovalchukvladislav/model/TableInfo.java @@ -0,0 +1,27 @@ +package ru.vk.itmo.kovalchukvladislav.model; + +public class TableInfo { + private final long recordsCount; + private final long recordsSize; + + public TableInfo(long recordsCount, long recordsSize) { + this.recordsCount = recordsCount; + this.recordsSize = recordsSize; + } + + public long getRecordsCount() { + return recordsCount; + } + + public long getRecordsSize() { + return recordsSize; + } + + @Override + public String toString() { + return "TableInfo{" + + "recordsCount=" + recordsCount + + ", recordsSize=" + recordsSize + + '}'; + } +} diff --git a/src/main/java/ru/vk/itmo/test/kovalchukvladislav/MemorySegmentDaoFactory.java b/src/main/java/ru/vk/itmo/test/kovalchukvladislav/MemorySegmentDaoFactory.java index c96427e2f..a0d08eb31 100644 --- a/src/main/java/ru/vk/itmo/test/kovalchukvladislav/MemorySegmentDaoFactory.java +++ b/src/main/java/ru/vk/itmo/test/kovalchukvladislav/MemorySegmentDaoFactory.java @@ -12,7 +12,7 @@ import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; -@DaoFactory(stage = 3) +@DaoFactory(stage = 4) public class MemorySegmentDaoFactory implements DaoFactory.Factory> { private static final Charset CHARSET = StandardCharsets.UTF_8; private static final ValueLayout.OfByte VALUE_LAYOUT = ValueLayout.JAVA_BYTE; From ea606822b859e8869dbbc07728825dd5ec7c771b Mon Sep 17 00:00:00 2001 From: alexBlack01 Date: Wed, 29 Nov 2023 12:23:44 +0300 Subject: [PATCH 08/20] =?UTF-8?q?=D0=A2=D1=83=D0=B7=D0=B8=D0=BA=D0=BE?= =?UTF-8?q?=D0=B2=20=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B0=D0=BD=D0=B4=D1=80?= =?UTF-8?q?,=20=D0=9C=D0=B0=D0=B3=D0=B8=D1=81=D1=82=D1=80=D0=B0=D1=82?= =?UTF-8?q?=D1=83=D1=80=D0=B0=20=D0=98=D0=A2=D0=9C=D0=9E=20"=D0=A0=D0=B0?= =?UTF-8?q?=D1=81=D0=BF=D1=80=D0=B5=D0=B4=D0=B5=D0=BB=D0=B5=D0=BD=D0=BD?= =?UTF-8?q?=D1=8B=D0=B5=20=D0=B2=D0=B5=D0=B1-=D1=81=D0=B5=D1=80=D0=B2?= =?UTF-8?q?=D0=B8=D1=81=D1=8B",=204=20=D0=BB=D0=B0=D0=B1=D0=BE=D1=80=D0=B0?= =?UTF-8?q?=D1=82=D0=BE=D1=80=D0=BD=D0=B0=D1=8F=20(#216)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Vadim Tsesko Co-authored-by: Artyom Drozdov Co-authored-by: atimofeyev --- .../test/tuzikovalexandr/DaoFactoryImpl.java | 2 +- .../itmo/tuzikovalexandr/InMemoryDaoImpl.java | 49 +++- .../MemorySegmentComparator.java | 9 +- .../vk/itmo/tuzikovalexandr/PeekIterator.java | 45 +++ .../itmo/tuzikovalexandr/RangeIterator.java | 101 +++++++ .../ru/vk/itmo/tuzikovalexandr/SSTable.java | 268 ++++++++++++++---- .../ru/vk/itmo/tuzikovalexandr/Utils.java | 75 +++++ 7 files changed, 483 insertions(+), 66 deletions(-) create mode 100644 src/main/java/ru/vk/itmo/tuzikovalexandr/PeekIterator.java create mode 100644 src/main/java/ru/vk/itmo/tuzikovalexandr/RangeIterator.java create mode 100644 src/main/java/ru/vk/itmo/tuzikovalexandr/Utils.java diff --git a/src/main/java/ru/vk/itmo/test/tuzikovalexandr/DaoFactoryImpl.java b/src/main/java/ru/vk/itmo/test/tuzikovalexandr/DaoFactoryImpl.java index 84ce44a04..b99eeb29b 100644 --- a/src/main/java/ru/vk/itmo/test/tuzikovalexandr/DaoFactoryImpl.java +++ b/src/main/java/ru/vk/itmo/test/tuzikovalexandr/DaoFactoryImpl.java @@ -11,7 +11,7 @@ import java.lang.foreign.ValueLayout; import java.nio.charset.StandardCharsets; -@DaoFactory(stage = 2) +@DaoFactory(stage = 4) public class DaoFactoryImpl implements DaoFactory.Factory> { @Override diff --git a/src/main/java/ru/vk/itmo/tuzikovalexandr/InMemoryDaoImpl.java b/src/main/java/ru/vk/itmo/tuzikovalexandr/InMemoryDaoImpl.java index 41af88086..9bbb76dc3 100644 --- a/src/main/java/ru/vk/itmo/tuzikovalexandr/InMemoryDaoImpl.java +++ b/src/main/java/ru/vk/itmo/tuzikovalexandr/InMemoryDaoImpl.java @@ -5,38 +5,60 @@ import ru.vk.itmo.Entry; import java.io.IOException; +import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.Iterator; import java.util.NavigableMap; import java.util.concurrent.ConcurrentSkipListMap; public class InMemoryDaoImpl implements Dao> { private final NavigableMap> memory = - new ConcurrentSkipListMap<>(new MemorySegmentComparator()); + new ConcurrentSkipListMap<>(MemorySegmentComparator::compare); private final SSTable ssTable; + private final Arena arena; + private final Path path; public InMemoryDaoImpl(Config config) throws IOException { - this.ssTable = new SSTable(config); + this.path = config.basePath().resolve("data"); + Files.createDirectories(path); + + arena = Arena.ofShared(); + + this.ssTable = new SSTable(SSTable.loadData(path, arena)); } @Override public Iterator> get(MemorySegment from, MemorySegment to) { + return ssTable.range(getInMemory(from, to), from, to); + } + public Iterator> getInMemory(MemorySegment from, MemorySegment to) { if (from == null && to == null) { return memory.values().iterator(); } else if (from == null) { return memory.headMap(to, false).values().iterator(); } else if (to == null) { - return memory.subMap(from, true, memory.lastKey(), false).values().iterator(); - } else { - return memory.subMap(from, true, to, false).values().iterator(); + return memory.tailMap(from, true).values().iterator(); } + + return memory.subMap(from, true, to, false).values().iterator(); } @Override public Entry get(MemorySegment key) { - var entry = memory.get(key); - return entry == null ? ssTable.readData(key) : entry; + Entry entry = memory.get(key); + + if (entry == null) { + entry = ssTable.readData(key); + } + + if (entry != null && entry.value() == null) { + return null; + } + + return entry; } @Override @@ -50,16 +72,21 @@ public Iterator> all() { } @Override - public void flush() throws IOException { - throw new UnsupportedOperationException(""); + public void compact() throws IOException { + ssTable.compactData(path, () -> get(null, null)); + memory.clear(); } @Override public void close() throws IOException { - if (memory.isEmpty()) { + if (!arena.scope().isAlive()) { return; } - ssTable.saveMemData(memory.values()); + arena.close(); + + if (!memory.isEmpty()) { + ssTable.saveMemData(path, memory.values()); + } } } diff --git a/src/main/java/ru/vk/itmo/tuzikovalexandr/MemorySegmentComparator.java b/src/main/java/ru/vk/itmo/tuzikovalexandr/MemorySegmentComparator.java index 3a7f22f38..59b7691b5 100644 --- a/src/main/java/ru/vk/itmo/tuzikovalexandr/MemorySegmentComparator.java +++ b/src/main/java/ru/vk/itmo/tuzikovalexandr/MemorySegmentComparator.java @@ -2,12 +2,13 @@ import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; -import java.util.Comparator; -public class MemorySegmentComparator implements Comparator { +public final class MemorySegmentComparator { - @Override - public int compare(MemorySegment o1, MemorySegment o2) { + private MemorySegmentComparator() { + } + + public static int compare(MemorySegment o1, MemorySegment o2) { long offset = o1.mismatch(o2); if (offset == -1) { return 0; diff --git a/src/main/java/ru/vk/itmo/tuzikovalexandr/PeekIterator.java b/src/main/java/ru/vk/itmo/tuzikovalexandr/PeekIterator.java new file mode 100644 index 000000000..680045853 --- /dev/null +++ b/src/main/java/ru/vk/itmo/tuzikovalexandr/PeekIterator.java @@ -0,0 +1,45 @@ +package ru.vk.itmo.tuzikovalexandr; + +import java.util.Iterator; +import java.util.NoSuchElementException; + +public class PeekIterator implements Iterator { + + private final int priority; + private T currentEntry; + private final Iterator iterator; + + public PeekIterator(Iterator iterator, int priority) { + this.priority = priority; + this.iterator = iterator; + } + + @Override + public boolean hasNext() { + return currentEntry != null || iterator.hasNext(); + } + + @Override + public T next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + T next = peek(); + currentEntry = null; + return next; + } + + public T peek() { + if (currentEntry == null) { + if (!iterator.hasNext()) { + return null; + } + currentEntry = iterator.next(); + } + return currentEntry; + } + + public int getPriority() { + return priority; + } +} diff --git a/src/main/java/ru/vk/itmo/tuzikovalexandr/RangeIterator.java b/src/main/java/ru/vk/itmo/tuzikovalexandr/RangeIterator.java new file mode 100644 index 000000000..08c287481 --- /dev/null +++ b/src/main/java/ru/vk/itmo/tuzikovalexandr/RangeIterator.java @@ -0,0 +1,101 @@ +package ru.vk.itmo.tuzikovalexandr; + +import java.util.Collection; +import java.util.Comparator; +import java.util.Iterator; +import java.util.NoSuchElementException; +import java.util.PriorityQueue; +import java.util.Queue; + +public abstract class RangeIterator implements Iterator { + + private final Queue> iterators; + private final Comparator comparator; + private PeekIterator peekIterator; + + protected RangeIterator(Collection> peekIterators, Comparator comparator) { + this.comparator = comparator; + Comparator> peekComp = (o1, o2) -> comparator.compare(o1.peek(), o2.peek()); + iterators = new PriorityQueue<>(peekIterators.size(), peekComp.thenComparing(o -> -o.getPriority())); + + int priority = 0; + for (Iterator iterator : peekIterators) { + if (iterator.hasNext()) { + iterators.add(new PeekIterator<>(iterator, priority++)); + } + } + } + + private PeekIterator peek() { + while (peekIterator == null) { + peekIterator = iterators.poll(); + if (peekIterator == null) { + return null; + } + + skipOldEntries(); + + skipNullEntries(); + } + + return peekIterator; + } + + private void skipOldEntries() { + while (true) { + PeekIterator next = iterators.peek(); + if (next == null) { + break; + } + + int compare = comparator.compare(peekIterator.peek(), next.peek()); + if (compare == 0) { + PeekIterator poll = iterators.poll(); + if (poll != null) { + poll.next(); + if (poll.hasNext()) { + iterators.add(poll); + } + } + } else { + break; + } + } + } + + private void skipNullEntries() { + if (peekIterator.peek() == null) { + peekIterator = null; + return; + } + + if (skip(peekIterator.peek())) { + peekIterator.next(); + if (peekIterator.hasNext()) { + iterators.add(peekIterator); + } + peekIterator = null; + } + } + + protected abstract boolean skip(T entry); + + @Override + public boolean hasNext() { + return peek() != null; + } + + @Override + public T next() { + PeekIterator localPeekIterator = peek(); + if (localPeekIterator == null) { + throw new NoSuchElementException(); + } + T next = localPeekIterator.next(); + this.peekIterator = null; + if (localPeekIterator.hasNext()) { + iterators.add(localPeekIterator); + } + return next; + } +} diff --git a/src/main/java/ru/vk/itmo/tuzikovalexandr/SSTable.java b/src/main/java/ru/vk/itmo/tuzikovalexandr/SSTable.java index f7c4fae7d..37101f2ad 100644 --- a/src/main/java/ru/vk/itmo/tuzikovalexandr/SSTable.java +++ b/src/main/java/ru/vk/itmo/tuzikovalexandr/SSTable.java @@ -1,7 +1,6 @@ package ru.vk.itmo.tuzikovalexandr; import ru.vk.itmo.BaseEntry; -import ru.vk.itmo.Config; import ru.vk.itmo.Entry; import java.io.IOException; @@ -9,11 +8,19 @@ import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; import java.nio.channels.FileChannel; +import java.nio.charset.StandardCharsets; +import java.nio.file.DirectoryStream; +import java.nio.file.FileAlreadyExistsException; import java.nio.file.Files; import java.nio.file.OpenOption; import java.nio.file.Path; +import java.nio.file.StandardCopyOption; import java.nio.file.StandardOpenOption; -import java.util.Collection; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.Iterator; +import java.util.List; +import java.util.NoSuchElementException; import java.util.Set; import static java.nio.channels.FileChannel.MapMode.READ_ONLY; @@ -25,85 +32,246 @@ public class SSTable { StandardOpenOption.CREATE, StandardOpenOption.READ, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING ); - private final Path filePath; - private final Arena readArena; - private final MemorySegment readSegment; - private static final String FILE_PATH = "data"; - public SSTable(Config config) throws IOException { - this.filePath = config.basePath().resolve(FILE_PATH); + private static final String FILE_PREFIX = "data_"; + private static final String OFFSET_PREFIX = "offset_"; + private static final String INDEX_FILE = "index.idx"; + private static final String INDEX_TMP = "index.tmp"; + private final List> files; - readArena = Arena.ofConfined(); + public SSTable(List> files) throws IOException { + this.files = files; + } - if (Files.notExists(filePath)) { - readSegment = null; - return; + public Iterator> range( + Iterator> firstIterator, + MemorySegment from, + MemorySegment to) { + List>> iterators = new ArrayList<>(files.size() + 1); + for (Entry entry : files) { + iterators.add(readDataFromTo(entry.key(), entry.value(), from, to)); } + iterators.add(firstIterator); + return new RangeIterator<>(iterators, Comparator.comparing(Entry::key, MemorySegmentComparator::compare)) { + @Override + protected boolean skip(Entry memorySegmentEntry) { + return memorySegmentEntry.value() == null; + } + }; + } - try (FileChannel fc = FileChannel.open(filePath, StandardOpenOption.READ)) { - readSegment = fc.map(READ_ONLY, 0, Files.size(filePath), readArena); + // storage format: offsetFile |keyOffset|valueOffset| dataFile |key|value| + public void saveMemData(Path basePath, Iterable> entries) throws IOException { + final Path indexTmp = basePath.resolve(INDEX_TMP); + final Path indexFile = basePath.resolve(INDEX_FILE); + + try { + Files.createFile(indexFile); + } catch (FileAlreadyExistsException ignored) { + // it is ok, actually it is normal state } - } + List existedFiles = Files.readAllLines(indexFile, StandardCharsets.UTF_8); + + String newFileName = getNewFileIndex(existedFiles); - public void saveMemData(Collection> entries) throws IOException { - if (!readArena.scope().isAlive()) { - return; + int countOffsets = 0; + long offsetData = 0; + long memorySize = 0; + for (Entry entry : entries) { + memorySize += entry.key().byteSize(); + if (entry.value() != null) { + memorySize += entry.value().byteSize(); + } + if (entry.value() == null) { + memorySize += Long.BYTES; + } + countOffsets++; } - readArena.close(); + long[] offsets = new long[countOffsets * 2]; - long offset = 0L; - long memorySize = entries.stream().mapToLong( - entry -> entry.key().byteSize() + entry.value().byteSize() - ).sum() + Long.BYTES * entries.size() * 2L; + int index = 0; - try (FileChannel fc = FileChannel.open(filePath, openOptions)) { + try (FileChannel fcData = FileChannel.open(basePath.resolve(FILE_PREFIX + newFileName), openOptions); + FileChannel fcOffset = FileChannel.open(basePath.resolve(OFFSET_PREFIX + newFileName), openOptions); + Arena writeArena = Arena.ofConfined()) { - MemorySegment writeSegment = fc.map(READ_WRITE, 0, memorySize, Arena.ofConfined()); + MemorySegment writeSegmentData = fcData.map(READ_WRITE, 0, memorySize, writeArena); + MemorySegment writeSegmentOffset = fcOffset.map( + READ_WRITE, 0, (long) offsets.length * Long.BYTES, writeArena + ); for (Entry entry : entries) { - writeSegment.set(ValueLayout.JAVA_LONG_UNALIGNED, offset, entry.key().byteSize()); - offset += Long.BYTES; - writeSegment.asSlice(offset).copyFrom(entry.key()); - offset += entry.key().byteSize(); - - writeSegment.set(ValueLayout.JAVA_LONG_UNALIGNED, offset, entry.value().byteSize()); - offset += Long.BYTES; - writeSegment.asSlice(offset).copyFrom(entry.value()); - offset += entry.value().byteSize(); + MemorySegment key = entry.key(); + offsets[index] = offsetData; + MemorySegment.copy(key, 0, writeSegmentData, offsetData, entry.key().byteSize()); + offsetData += key.byteSize(); + + MemorySegment value = entry.value(); + offsets[index + 1] = offsetData; + if (value != null) { + MemorySegment.copy(value, 0, writeSegmentData, offsetData, entry.value().byteSize()); + offsetData += value.byteSize(); + } + if (value == null) { + writeSegmentData.set(ValueLayout.JAVA_LONG_UNALIGNED, offsetData, -1L); + offsetData += Long.BYTES; + } + + index += 2; } + + MemorySegment.copy( + MemorySegment.ofArray(offsets), ValueLayout.JAVA_LONG, 0, + writeSegmentOffset, ValueLayout.JAVA_LONG,0, offsets.length + ); } + + Files.move(indexFile, indexTmp, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + + List list = new ArrayList<>(existedFiles.size() + 1); + list.addAll(existedFiles); + list.add(newFileName); + Files.write( + indexFile, + list, + StandardOpenOption.WRITE, + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING + ); + + Files.delete(indexTmp); } public Entry readData(MemorySegment key) { - if (readSegment == null) { + if (files == null || key == null) { return null; } - long offset = 0L; + for (int i = files.size() - 1; i >= 0; i--) { + MemorySegment offsetSegment = files.get(i).key(); + MemorySegment dataSegment = files.get(i).value(); + + long offsetResult = Utils.binarySearch(key, offsetSegment, dataSegment); + + if (offsetResult >= 0) { + return Utils.getEntryByKeyOffset(offsetResult, offsetSegment, dataSegment); + } + } + + return null; + } + + public Iterator> readDataFromTo(MemorySegment offsetSegment, MemorySegment dataSegment, + MemorySegment from, MemorySegment to) { + long start = from == null ? 0 : Math.abs(Utils.binarySearch(from, offsetSegment, dataSegment)); + long end = to == null ? offsetSegment.byteSize() - Long.BYTES * 2 : + Math.abs(Utils.binarySearch(to, offsetSegment, dataSegment)) - Long.BYTES * 2; + + return new Iterator<>() { + long currentOffset = start; + + @Override + public boolean hasNext() { + return currentOffset <= end; + } + + @Override + public Entry next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + + Entry currentEntry = + Utils.getEntryByKeyOffset(currentOffset, offsetSegment, dataSegment); + + currentOffset += Long.BYTES * 2; + return currentEntry; + } + }; + } + + public void compactData(Path storagePath, Iterable> iterator) throws IOException { + saveMemData(storagePath, iterator); + + final Path indexTmp = storagePath.resolve(INDEX_TMP); + final Path indexFile = storagePath.resolve(INDEX_FILE); + + List existedFiles = Files.readAllLines(indexFile, StandardCharsets.UTF_8); + Files.move(indexFile, indexTmp, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + + String lastFileIndex = existedFiles.getLast(); + + deleteOldFiles(storagePath, lastFileIndex); + + Files.writeString( + indexFile, + lastFileIndex, + StandardOpenOption.WRITE, + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING + ); - while (offset < readSegment.byteSize()) { - long keySize = readSegment.get(ValueLayout.JAVA_LONG_UNALIGNED, offset); - offset += Long.BYTES; + Files.delete(indexTmp); + } + + public void deleteOldFiles(Path storagePath, String lastFileIndex) throws IOException { + String lastFileNameOffset = OFFSET_PREFIX + lastFileIndex; + String lastFileNameData = FILE_PREFIX + lastFileIndex; + + try (DirectoryStream fileStream = Files.newDirectoryStream(storagePath)) { + for (Path path : fileStream) { + String fileName = path.getFileName().toString(); + if (!fileName.equals(INDEX_FILE) && !fileName.equals(lastFileNameData) + && !fileName.equals(lastFileNameOffset) && !fileName.equals(INDEX_TMP)) { + Files.delete(path); + } + } + } + } - long valueSize = readSegment.get(ValueLayout.JAVA_LONG_UNALIGNED, offset + keySize); + public static List> loadData(Path storagePath, Arena arena) throws IOException { + Path indexTmp = storagePath.resolve(INDEX_TMP); + Path indexFile = storagePath.resolve(INDEX_FILE); - if (keySize != key.byteSize()) { - offset += keySize + valueSize + Long.BYTES; - continue; + if (Files.exists(indexTmp)) { + Files.move(indexTmp, indexFile, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + } else { + try { + Files.createFile(indexFile); + } catch (FileAlreadyExistsException ignored) { + // it is ok, actually it is normal state } + } + + List existedFiles = Files.readAllLines(indexFile, StandardCharsets.UTF_8); + List> result = new ArrayList<>(existedFiles.size()); + for (String fileName : existedFiles) { + Path offsetFullPath = storagePath.resolve(OFFSET_PREFIX + fileName); + Path fileFullPath = storagePath.resolve(FILE_PREFIX + fileName); - MemorySegment keySegment = readSegment.asSlice(offset, keySize); - offset += keySize + Long.BYTES; + try (FileChannel fcOffset = FileChannel.open(offsetFullPath, StandardOpenOption.READ); + FileChannel fcData = FileChannel.open(fileFullPath, StandardOpenOption.READ)) { + MemorySegment readSegmentOffset = fcOffset.map( + READ_ONLY, 0, Files.size(offsetFullPath), arena + ); + MemorySegment readSegmentData = fcData.map( + READ_ONLY, 0, Files.size(fileFullPath), arena + ); - if (key.mismatch(keySegment) == -1) { - MemorySegment valueSegment = readSegment.asSlice(offset, valueSize); - return new BaseEntry<>(keySegment, valueSegment); + result.add(new BaseEntry<>(readSegmentOffset, readSegmentData)); } + } - offset += valueSize; + return result; + } + + private String getNewFileIndex(List existedFiles) { + if (existedFiles.isEmpty()) { + return "1"; } - return null; + int lastIndex = Integer.parseInt(existedFiles.getLast()); + return String.valueOf(lastIndex + 1); } } diff --git a/src/main/java/ru/vk/itmo/tuzikovalexandr/Utils.java b/src/main/java/ru/vk/itmo/tuzikovalexandr/Utils.java new file mode 100644 index 000000000..0c86d843d --- /dev/null +++ b/src/main/java/ru/vk/itmo/tuzikovalexandr/Utils.java @@ -0,0 +1,75 @@ +package ru.vk.itmo.tuzikovalexandr; + +import ru.vk.itmo.BaseEntry; +import ru.vk.itmo.Entry; + +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; + +public final class Utils { + private Utils() { + } + + public static Entry getEntryByKeyOffset( + long offsetResult, MemorySegment offsetSegment, MemorySegment dataSegment) { + + long offset = offsetResult + Long.BYTES; + long valueOffset = offsetSegment.get(ValueLayout.JAVA_LONG, offset); + + MemorySegment valueSegment; + offset += Long.BYTES; + if (offset >= offsetSegment.byteSize()) { + valueSegment = dataSegment.asSlice(valueOffset); + + } else { + long valueSize = offsetSegment.get(ValueLayout.JAVA_LONG, offset) - valueOffset; + + valueSegment = dataSegment.asSlice(valueOffset, valueSize); + } + + if (valueSegment.byteSize() == Long.BYTES && valueSegment.get(ValueLayout.JAVA_LONG_UNALIGNED, 0) == -1) { + valueSegment = null; + } + + long keyOffset = offsetSegment.get(ValueLayout.JAVA_LONG, offsetResult); + long keySize = valueOffset - keyOffset; + MemorySegment keySegment = dataSegment.asSlice(keyOffset, keySize); + + return new BaseEntry<>(keySegment, valueSegment); + } + + public static long binarySearch(MemorySegment key, MemorySegment offsetSegment, + MemorySegment dataSegment) { + long left = 0; + long right = offsetSegment.byteSize() / Long.BYTES - 1; + + while (left <= right) { + + long middle = (right - left) / 2 + left; + + long offset = middle * Long.BYTES * 2; + if (offset >= offsetSegment.byteSize()) { + return -left * Long.BYTES * 2; + } + + long keyOffset = offsetSegment.get(ValueLayout.JAVA_LONG, offset); + + offset = middle * Long.BYTES * 2 + Long.BYTES; + long keySize = offsetSegment.get(ValueLayout.JAVA_LONG, offset) - keyOffset; + + MemorySegment keySegment = dataSegment.asSlice(keyOffset, keySize); + + int result = MemorySegmentComparator.compare(keySegment, key); + + if (result < 0) { + left = middle + 1; + } else if (result > 0) { + right = middle - 1; + } else { + return middle * Long.BYTES * 2; + } + } + + return -left * Long.BYTES * 2; + } +} From 08397bd4dfb1988e8152d282dbdd9dda6cdf1478 Mon Sep 17 00:00:00 2001 From: Alexey <71148093+lehatheslayer@users.noreply.github.com> Date: Wed, 29 Nov 2023 12:24:40 +0300 Subject: [PATCH 09/20] =?UTF-8?q?=D0=A0=D0=B5=D1=88=D0=B5=D1=82=D0=BD?= =?UTF-8?q?=D0=B8=D0=BA=D0=BE=D0=B2=20=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B5?= =?UTF-8?q?=D0=B9,=20=D0=9C=D0=B0=D0=B3=D0=B8=D1=81=D1=82=D1=80=D0=B0?= =?UTF-8?q?=D1=82=D1=83=D1=80=D0=B0=20=D0=98=D0=A2=D0=9C=D0=9E=20"=D0=A0?= =?UTF-8?q?=D0=B0=D1=81=D0=BF=D1=80=D0=B5=D0=B4=D0=B5=D0=BB=D0=B5=D0=BD?= =?UTF-8?q?=D0=BD=D1=8B=D0=B5=20=D0=B2=D0=B5=D0=B1-=D1=81=D0=B5=D1=80?= =?UTF-8?q?=D0=B2=D0=B8=D1=81=D1=8B",=20=D0=9B=D0=B0=D0=B14=20(#228)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Vadim Tsesko --- .../vk/itmo/reshetnikovaleksei/DaoImpl.java | 97 ++++++++++ .../itmo/reshetnikovaleksei/InMemoryDao.java | 58 ------ .../MemorySegmentComparator.java | 58 ++++++ .../vk/itmo/reshetnikovaleksei/SSTable.java | 81 ++++++++ .../reshetnikovaleksei/SSTableManager.java | 182 ++++++++++++++++++ .../iterators/MergeIterator.java | 84 ++++++++ .../iterators/PeekingIterator.java | 51 +++++ .../iterators/SSTableIterator.java | 79 ++++++++ .../reshetnikovaleksei/DaoFactoryImpl.java | 13 +- 9 files changed, 642 insertions(+), 61 deletions(-) create mode 100644 src/main/java/ru/vk/itmo/reshetnikovaleksei/DaoImpl.java delete mode 100644 src/main/java/ru/vk/itmo/reshetnikovaleksei/InMemoryDao.java create mode 100644 src/main/java/ru/vk/itmo/reshetnikovaleksei/MemorySegmentComparator.java create mode 100644 src/main/java/ru/vk/itmo/reshetnikovaleksei/SSTable.java create mode 100644 src/main/java/ru/vk/itmo/reshetnikovaleksei/SSTableManager.java create mode 100644 src/main/java/ru/vk/itmo/reshetnikovaleksei/iterators/MergeIterator.java create mode 100644 src/main/java/ru/vk/itmo/reshetnikovaleksei/iterators/PeekingIterator.java create mode 100644 src/main/java/ru/vk/itmo/reshetnikovaleksei/iterators/SSTableIterator.java diff --git a/src/main/java/ru/vk/itmo/reshetnikovaleksei/DaoImpl.java b/src/main/java/ru/vk/itmo/reshetnikovaleksei/DaoImpl.java new file mode 100644 index 000000000..fa65481d7 --- /dev/null +++ b/src/main/java/ru/vk/itmo/reshetnikovaleksei/DaoImpl.java @@ -0,0 +1,97 @@ +package ru.vk.itmo.reshetnikovaleksei; + +import ru.vk.itmo.Config; +import ru.vk.itmo.Dao; +import ru.vk.itmo.Entry; +import ru.vk.itmo.reshetnikovaleksei.iterators.MergeIterator; +import ru.vk.itmo.reshetnikovaleksei.iterators.PeekingIterator; + +import java.io.IOException; +import java.lang.foreign.MemorySegment; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.ConcurrentNavigableMap; +import java.util.concurrent.ConcurrentSkipListMap; + +public class DaoImpl implements Dao> { + private final ConcurrentNavigableMap> memoryTable; + private final SSTableManager ssTableManager; + + public DaoImpl(Config config) throws IOException { + this.memoryTable = new ConcurrentSkipListMap<>(MemorySegmentComparator.getInstance()); + this.ssTableManager = new SSTableManager(config); + } + + @Override + public Entry get(MemorySegment key) { + Iterator> iterator = allIterators(key, null); + + if (iterator.hasNext()) { + Entry result = iterator.next(); + if (MemorySegmentComparator.getInstance().compare(key, result.key()) == 0) { + return result.value() == null ? null : result; + } + } + + return null; + } + + @Override + public Iterator> get(MemorySegment from, MemorySegment to) { + return allIterators(from, to); + } + + @Override + public void upsert(Entry entry) { + memoryTable.put(entry.key(), entry); + } + + @Override + public void close() throws IOException { + if (!memoryTable.isEmpty()) { + ssTableManager.save(memoryTable.values()); + memoryTable.clear(); + } + + ssTableManager.close(); + } + + @Override + public void compact() throws IOException { + ssTableManager.compact(() -> get(null, null)); + memoryTable.clear(); + } + + private Iterator> allIterators(MemorySegment from, MemorySegment to) { + List iterators = new ArrayList<>(); + + Iterator> memoryIterator = memoryIterator(from, to); + Iterator> filesIterator = ssTableManager.get(from, to); + + iterators.add(new PeekingIterator(memoryIterator, 1)); + iterators.add(new PeekingIterator(filesIterator, 0)); + + return new PeekingIterator( + MergeIterator.merge( + iterators, + MemorySegmentComparator.getInstance() + ) + ); + } + + private Iterator> memoryIterator(MemorySegment from, MemorySegment to) { + if (from == null && to == null) { + return memoryTable.values().iterator(); + } + + if (from == null) { + return memoryTable.headMap(to).values().iterator(); + } + if (to == null) { + return memoryTable.tailMap(from).values().iterator(); + } + + return memoryTable.subMap(from, to).values().iterator(); + } +} diff --git a/src/main/java/ru/vk/itmo/reshetnikovaleksei/InMemoryDao.java b/src/main/java/ru/vk/itmo/reshetnikovaleksei/InMemoryDao.java deleted file mode 100644 index d393b0ea8..000000000 --- a/src/main/java/ru/vk/itmo/reshetnikovaleksei/InMemoryDao.java +++ /dev/null @@ -1,58 +0,0 @@ -package ru.vk.itmo.reshetnikovaleksei; - -import ru.vk.itmo.Dao; -import ru.vk.itmo.Entry; - -import java.lang.foreign.MemorySegment; -import java.lang.foreign.ValueLayout; -import java.util.Iterator; -import java.util.concurrent.ConcurrentSkipListMap; - -public class InMemoryDao implements Dao> { - private final ConcurrentSkipListMap> map; - - public InMemoryDao() { - this.map = new ConcurrentSkipListMap<>((a, b) -> { - var offset = a.mismatch(b); - - if (offset == -1) { - return 0; - } else if (offset == a.byteSize()) { - return -1; - } else if (offset == b.byteSize()) { - return 1; - } else { - return Byte.compare( - a.get(ValueLayout.JAVA_BYTE, offset), - b.get(ValueLayout.JAVA_BYTE, offset) - ); - } - }); - } - - @Override - public Iterator> get(MemorySegment from, MemorySegment to) { - if (from == null && to == null) { - return map.values().iterator(); - } - - if (from == null) { - return map.headMap(to).values().iterator(); - } - if (to == null) { - return map.tailMap(from).values().iterator(); - } - - return map.subMap(from, to).values().iterator(); - } - - @Override - public Entry get(MemorySegment key) { - return map.get(key); - } - - @Override - public void upsert(Entry entry) { - map.put(entry.key(), entry); - } -} diff --git a/src/main/java/ru/vk/itmo/reshetnikovaleksei/MemorySegmentComparator.java b/src/main/java/ru/vk/itmo/reshetnikovaleksei/MemorySegmentComparator.java new file mode 100644 index 000000000..30a2b08a3 --- /dev/null +++ b/src/main/java/ru/vk/itmo/reshetnikovaleksei/MemorySegmentComparator.java @@ -0,0 +1,58 @@ +package ru.vk.itmo.reshetnikovaleksei; + +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.util.Comparator; + +public final class MemorySegmentComparator implements Comparator { + private static MemorySegmentComparator instance; + + private MemorySegmentComparator() { + } + + public static synchronized MemorySegmentComparator getInstance() { + if (instance == null) { + instance = new MemorySegmentComparator(); + } + + return instance; + } + + @Override + public int compare(MemorySegment a, MemorySegment b) { + var offset = a.mismatch(b); + + if (offset == -1) { + return 0; + } else if (offset == a.byteSize()) { + return -1; + } else if (offset == b.byteSize()) { + return 1; + } else { + return Byte.compare( + a.get(ValueLayout.JAVA_BYTE, offset), + b.get(ValueLayout.JAVA_BYTE, offset) + ); + } + } + + public int compare(MemorySegment a, MemorySegment b, long fromOffset, long toOffset) { + long mismatch = MemorySegment.mismatch( + b, fromOffset, toOffset, + a, 0, a.byteSize() + ); + + if (mismatch == -1) { + return 0; + } else if (mismatch == a.byteSize()) { + return -1; + } else if (mismatch == b.byteSize()) { + return 1; + } + + return Byte.compare( + a.get(ValueLayout.JAVA_BYTE, mismatch), + b.get(ValueLayout.JAVA_BYTE, mismatch + fromOffset) + ); + } +} diff --git a/src/main/java/ru/vk/itmo/reshetnikovaleksei/SSTable.java b/src/main/java/ru/vk/itmo/reshetnikovaleksei/SSTable.java new file mode 100644 index 000000000..4d4ec2d03 --- /dev/null +++ b/src/main/java/ru/vk/itmo/reshetnikovaleksei/SSTable.java @@ -0,0 +1,81 @@ +package ru.vk.itmo.reshetnikovaleksei; + +import ru.vk.itmo.Entry; +import ru.vk.itmo.reshetnikovaleksei.iterators.SSTableIterator; + +import java.io.IOException; +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.nio.channels.FileChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.Iterator; + +public class SSTable { + public static final String DATA_PREFIX = "data-"; + public static final String DATA_TMP = "data.tmp"; + public static final String INDEX_PREFIX = "index-"; + public static final String INDEX_TMP = "index.tmp"; + + private final MemorySegment dataSegment; + private final MemorySegment indexSegment; + private final Path dataPath; + private final Path indexPath; + + public SSTable(Path basePath, Arena arena, long idx) throws IOException { + this.dataPath = basePath.resolve(DATA_PREFIX + idx); + this.indexPath = basePath.resolve(INDEX_PREFIX + idx); + + try (FileChannel dataChannel = FileChannel.open(dataPath, StandardOpenOption.READ)) { + this.dataSegment = dataChannel.map(FileChannel.MapMode.READ_ONLY, 0, dataChannel.size(), arena); + } + try (FileChannel indexChannel = FileChannel.open(indexPath, StandardOpenOption.READ)) { + this.indexSegment = indexChannel.map(FileChannel.MapMode.READ_ONLY, 0, indexChannel.size(), arena); + } + } + + public Iterator> iterator(MemorySegment from, MemorySegment to) { + long indexFrom; + + if (from == null) { + indexFrom = 0; + } else { + indexFrom = getIndexOffsetByKey(from); + } + + return new SSTableIterator(indexFrom, to, dataSegment, indexSegment); + } + + public void deleteFiles() throws IOException { + Files.deleteIfExists(dataPath); + Files.deleteIfExists(indexPath); + } + + private long getIndexOffsetByKey(MemorySegment key) { + long l = 0; + long r = indexSegment.byteSize() / Long.BYTES - 1; + + while (l <= r) { + long m = l + (r - l) / 2; + + long dataOffset = indexSegment.get(ValueLayout.JAVA_LONG_UNALIGNED, m * Long.BYTES); + long currKeySize = dataSegment.get(ValueLayout.JAVA_LONG_UNALIGNED, dataOffset); + dataOffset += Long.BYTES; + + long comparing = MemorySegmentComparator.getInstance().compare( + key, dataSegment, dataOffset, dataOffset + currKeySize); + if (comparing > 0) { + l = m + 1; + } else if (comparing < 0) { + r = m - 1; + } else { + return m * Long.BYTES; + } + + } + + return l * Long.BYTES; + } +} diff --git a/src/main/java/ru/vk/itmo/reshetnikovaleksei/SSTableManager.java b/src/main/java/ru/vk/itmo/reshetnikovaleksei/SSTableManager.java new file mode 100644 index 000000000..c3fc3519a --- /dev/null +++ b/src/main/java/ru/vk/itmo/reshetnikovaleksei/SSTableManager.java @@ -0,0 +1,182 @@ +package ru.vk.itmo.reshetnikovaleksei; + +import ru.vk.itmo.Config; +import ru.vk.itmo.Entry; +import ru.vk.itmo.reshetnikovaleksei.iterators.MergeIterator; +import ru.vk.itmo.reshetnikovaleksei.iterators.PeekingIterator; + +import java.io.IOException; +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.nio.channels.FileChannel; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.stream.Stream; + +import static ru.vk.itmo.reshetnikovaleksei.SSTable.DATA_PREFIX; +import static ru.vk.itmo.reshetnikovaleksei.SSTable.DATA_TMP; +import static ru.vk.itmo.reshetnikovaleksei.SSTable.INDEX_PREFIX; +import static ru.vk.itmo.reshetnikovaleksei.SSTable.INDEX_TMP; + +public class SSTableManager implements AutoCloseable { + + private final Arena arena; + private final Path basePath; + private final List ssTables; + + private int lastIdx; + private boolean isClosed; + + public SSTableManager(Config config) throws IOException { + this.arena = Arena.ofShared(); + this.basePath = config.basePath(); + this.ssTables = new ArrayList<>(); + + this.lastIdx = 0; + this.isClosed = false; + + if (!Files.exists(basePath)) { + return; + } + + long filesCount; + try (Stream filesStream = Files.list(basePath)) { + filesCount = filesStream.count(); + } + + for (int i = 0; i < filesCount; i++) { + try { + ssTables.add(new SSTable(basePath, arena, i)); + } catch (IOException e) { + lastIdx = i; + } + } + } + + public Iterator> get(MemorySegment key) { + return get(key, null); + } + + public Iterator> get(MemorySegment from, MemorySegment to) { + List iterators = new ArrayList<>(); + + int priority = 1; + for (SSTable ssTable : ssTables) { + iterators.add(new PeekingIterator(ssTable.iterator(from, to), priority)); + priority++; + } + + return MergeIterator.merge(iterators, MemorySegmentComparator.getInstance()); + } + + public void save(Iterable> entries) throws IOException { + Path tmpDataPath = basePath.resolve(DATA_TMP); + Path tmpIndexPath = basePath.resolve(INDEX_TMP); + + Path dataPath = basePath.resolve(DATA_PREFIX + lastIdx); + Path indexPath = basePath.resolve(INDEX_PREFIX + lastIdx); + + try ( + FileChannel dataChannel = FileChannel.open( + tmpDataPath, + StandardOpenOption.READ, + StandardOpenOption.CREATE, + StandardOpenOption.WRITE + ); + FileChannel indexChannel = FileChannel.open( + tmpIndexPath, + StandardOpenOption.READ, + StandardOpenOption.CREATE, + StandardOpenOption.WRITE + ); + Arena writeDataArena = Arena.ofConfined() + ) { + long dataSize = Long.BYTES; + long indexSize = 0; + for (Entry entry : entries) { + dataSize += entry.key().byteSize() + + (entry.value() == null ? 0 : entry.value().byteSize()) + + 2 * Long.BYTES; + indexSize += Long.BYTES; + } + + long dataOffset = 0; + long indexOffset = 0; + + MemorySegment dataSegment = dataChannel.map( + FileChannel.MapMode.READ_WRITE, dataOffset, dataSize, writeDataArena); + dataSegment.set(ValueLayout.JAVA_LONG_UNALIGNED, dataOffset, dataSize - Long.BYTES); + dataOffset += Long.BYTES; + + MemorySegment indexSegment = indexChannel.map( + FileChannel.MapMode.READ_WRITE, indexOffset, indexSize, writeDataArena); + + for (Entry entry : entries) { + indexSegment.set(ValueLayout.JAVA_LONG_UNALIGNED, indexOffset, dataOffset); + indexOffset += Long.BYTES; + + dataSegment.set(ValueLayout.JAVA_LONG_UNALIGNED, dataOffset, entry.key().byteSize()); + dataOffset += Long.BYTES; + MemorySegment.copy(entry.key(), 0, dataSegment, dataOffset, entry.key().byteSize()); + dataOffset += entry.key().byteSize(); + + if (entry.value() == null) { + dataSegment.set(ValueLayout.JAVA_LONG_UNALIGNED, dataOffset, -1); + dataOffset += Long.BYTES; + } else { + dataSegment.set(ValueLayout.JAVA_LONG_UNALIGNED, dataOffset, entry.value().byteSize()); + dataOffset += Long.BYTES; + MemorySegment.copy(entry.value(), 0, dataSegment, dataOffset, entry.value().byteSize()); + dataOffset += entry.value().byteSize(); + } + } + } + + moveDataFromTmpToReal(tmpDataPath, dataPath); + moveDataFromTmpToReal(tmpIndexPath, indexPath); + } + + public void compact(Iterable> entries) throws IOException { + save(entries); + + Path dataPath = basePath.resolve(DATA_PREFIX + lastIdx); + Path indexPath = basePath.resolve(INDEX_PREFIX + lastIdx); + deleteAllFiles(); + + moveDataFromTmpToReal(dataPath, basePath.resolve(DATA_PREFIX + lastIdx)); + moveDataFromTmpToReal(indexPath, basePath.resolve(INDEX_PREFIX + lastIdx)); + } + + @Override + public void close() { + if (!isClosed) { + arena.close(); + isClosed = true; + } + } + + private void moveDataFromTmpToReal(Path tmpFilePath, Path realFilePath) throws IOException { + try { + Files.createFile(realFilePath); + } catch (FileAlreadyExistsException ignored) { + // do nothing + } + + Files.move(tmpFilePath, realFilePath, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + } + + private void deleteAllFiles() throws IOException { + for (SSTable ssTable : ssTables) { + ssTable.deleteFiles(); + } + + lastIdx = 0; + } +} diff --git a/src/main/java/ru/vk/itmo/reshetnikovaleksei/iterators/MergeIterator.java b/src/main/java/ru/vk/itmo/reshetnikovaleksei/iterators/MergeIterator.java new file mode 100644 index 000000000..a2cedd7e2 --- /dev/null +++ b/src/main/java/ru/vk/itmo/reshetnikovaleksei/iterators/MergeIterator.java @@ -0,0 +1,84 @@ +package ru.vk.itmo.reshetnikovaleksei.iterators; + +import ru.vk.itmo.Entry; + +import java.lang.foreign.MemorySegment; +import java.util.Collections; +import java.util.Comparator; +import java.util.Iterator; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.PriorityQueue; +import java.util.Queue; + +public final class MergeIterator implements Iterator> { + private final Queue queue; + + private MergeIterator(Queue queue) { + this.queue = queue; + } + + public static Iterator> merge( + List iterators, Comparator comparator) { + if (iterators.isEmpty()) { + return Collections.emptyIterator(); + } + + Queue queue = new PriorityQueue<>( + iterators.size(), + Comparator.comparing((PeekingIterator iter) -> iter.peek().key(), comparator) + .thenComparing(PeekingIterator::priority, Comparator.reverseOrder()) + ); + + for (PeekingIterator iterator : iterators) { + if (iterator.hasNext()) { + queue.add(iterator); + } + } + + return new MergeIterator(queue); + } + + @Override + public boolean hasNext() { + skipDeletedEntry(); + return !queue.isEmpty(); + } + + @Override + public Entry next() { + if (!hasNext()) { + throw new NoSuchElementException("no next element"); + } + + PeekingIterator currIterator = queue.remove(); + Entry currEntry = currIterator.next(); + removeOldEntryByKey(currEntry.key()); + if (currIterator.hasNext()) { + queue.add(currIterator); + } + + return currEntry; + } + + private void skipDeletedEntry() { + while (!queue.isEmpty() && queue.peek().peek().value() == null) { + PeekingIterator currentItr = queue.remove(); + Entry current = currentItr.next(); + removeOldEntryByKey(current.key()); + if (currentItr.hasNext()) { + queue.add(currentItr); + } + } + } + + private void removeOldEntryByKey(MemorySegment key) { + while (!queue.isEmpty() && queue.peek().peek().key().mismatch(key) == -1) { + PeekingIterator currentItr = queue.remove(); + currentItr.next(); + if (currentItr.hasNext()) { + queue.add(currentItr); + } + } + } +} diff --git a/src/main/java/ru/vk/itmo/reshetnikovaleksei/iterators/PeekingIterator.java b/src/main/java/ru/vk/itmo/reshetnikovaleksei/iterators/PeekingIterator.java new file mode 100644 index 000000000..a69e29d1f --- /dev/null +++ b/src/main/java/ru/vk/itmo/reshetnikovaleksei/iterators/PeekingIterator.java @@ -0,0 +1,51 @@ +package ru.vk.itmo.reshetnikovaleksei.iterators; + +import ru.vk.itmo.Entry; + +import java.lang.foreign.MemorySegment; +import java.util.Iterator; + +public class PeekingIterator implements Iterator> { + private final Iterator> iterator; + private final int priority; + + private Entry next; + + public PeekingIterator(Iterator> iterator) { + this(iterator, 0); + } + + public PeekingIterator(Iterator> iterator, int priority) { + this.iterator = iterator; + this.priority = priority; + + if (iterator.hasNext()) { + next = iterator.next(); + } + } + + @Override + public boolean hasNext() { + return next != null || iterator.hasNext(); + } + + @Override + public Entry next() { + Entry toReturn = peek(); + next = null; + + return toReturn; + } + + public int priority() { + return priority; + } + + public Entry peek() { + if (next == null) { + next = iterator.next(); + } + + return next; + } +} diff --git a/src/main/java/ru/vk/itmo/reshetnikovaleksei/iterators/SSTableIterator.java b/src/main/java/ru/vk/itmo/reshetnikovaleksei/iterators/SSTableIterator.java new file mode 100644 index 000000000..505e9a5fe --- /dev/null +++ b/src/main/java/ru/vk/itmo/reshetnikovaleksei/iterators/SSTableIterator.java @@ -0,0 +1,79 @@ +package ru.vk.itmo.reshetnikovaleksei.iterators; + +import ru.vk.itmo.BaseEntry; +import ru.vk.itmo.Entry; +import ru.vk.itmo.reshetnikovaleksei.MemorySegmentComparator; + +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.util.Iterator; +import java.util.NoSuchElementException; + +public class SSTableIterator implements Iterator> { + + private final MemorySegment dataSegment; + private final MemorySegment indexSegment; + private final MemorySegment to; + + private long indexOffset; + private long currentKeyOffset; + private long currentKeySize; + + public SSTableIterator(long indexOffset, MemorySegment to, MemorySegment dataSegment, MemorySegment indexSegment) { + this.indexOffset = indexOffset; + this.to = to; + this.dataSegment = dataSegment; + this.indexSegment = indexSegment; + + this.currentKeyOffset = -1; + this.currentKeySize = -1; + } + + @Override + public boolean hasNext() { + if (indexOffset == indexSegment.byteSize()) { + return false; + } + + if (to == null) { + return true; + } + currentKeyOffset = indexSegment.get(ValueLayout.JAVA_LONG_UNALIGNED, indexOffset); + currentKeySize = dataSegment.get(ValueLayout.JAVA_LONG_UNALIGNED, currentKeyOffset); + long fromOffset = currentKeyOffset + Long.BYTES; + + return MemorySegmentComparator.getInstance() + .compare(to, dataSegment, fromOffset, fromOffset + currentKeySize) > 0; + } + + @Override + public Entry next() { + if (!hasNext()) { + throw new NoSuchElementException("No next element"); + } + + long keyOffset; + long keySize; + if (currentKeyOffset == -1 || currentKeySize == -1) { + keyOffset = indexSegment.get(ValueLayout.JAVA_LONG_UNALIGNED, indexOffset); + keySize = dataSegment.get(ValueLayout.JAVA_LONG_UNALIGNED, keyOffset); + } else { + keyOffset = currentKeyOffset; + keySize = currentKeySize; + } + indexOffset += Long.BYTES; + keyOffset += Long.BYTES; + MemorySegment key = dataSegment.asSlice(keyOffset, keySize); + keyOffset += keySize; + + long valueSize = dataSegment.get(ValueLayout.JAVA_LONG_UNALIGNED, keyOffset); + MemorySegment value; + if (valueSize == -1) { + value = null; + } else { + value = dataSegment.asSlice(keyOffset + Long.BYTES, valueSize); + } + + return new BaseEntry<>(key, value); + } +} diff --git a/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/DaoFactoryImpl.java b/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/DaoFactoryImpl.java index c9ecc4f07..8ae493ef1 100644 --- a/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/DaoFactoryImpl.java +++ b/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/DaoFactoryImpl.java @@ -1,15 +1,17 @@ package ru.vk.itmo.test.reshetnikovaleksei; +import ru.vk.itmo.Config; import ru.vk.itmo.Dao; import ru.vk.itmo.Entry; -import ru.vk.itmo.reshetnikovaleksei.InMemoryDao; +import ru.vk.itmo.reshetnikovaleksei.DaoImpl; import ru.vk.itmo.test.DaoFactory; +import java.io.IOException; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; import java.nio.charset.StandardCharsets; -@DaoFactory +@DaoFactory(stage = 4) public class DaoFactoryImpl implements DaoFactory.Factory> { @Override public String toString(MemorySegment memorySegment) { @@ -32,6 +34,11 @@ public Entry fromBaseEntry(Entry baseEntry) { @Override public Dao> createDao() { - return new InMemoryDao(); + return null; + } + + @Override + public Dao> createDao(Config config) throws IOException { + return new DaoImpl(config); } } From 88f827d583cc38e7542cf8943905921e048b2725 Mon Sep 17 00:00:00 2001 From: Eugene Kachmar <66039322+Jenshen30@users.noreply.github.com> Date: Sun, 3 Dec 2023 11:07:05 +0300 Subject: [PATCH 10/20] =?UTF-8?q?=D0=9A=D0=B0=D1=87=D0=BC=D0=B0=D1=80=20?= =?UTF-8?q?=D0=95=D0=B2=D0=B3=D0=B5=D0=BD=D0=B8=D0=B9,=20=D0=98=D0=A2?= =?UTF-8?q?=D0=9C=D0=9E=20=D0=A4=D0=98=D0=A2=D0=B8=D0=9F=20=D0=9C33331,=20?= =?UTF-8?q?HW4=20(#232)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Jenshen30 Co-authored-by: Vadim Tsesko Co-authored-by: Alexey Shik <58121508+AlexeyShik@users.noreply.github.com> --- .../itmo/test/kachmareugene/DaoFactory.java | 6 +- .../test/kachmareugene/DaoWithCompaction.java | 21 +++ .../itmo/test/kachmareugene/InMemoryDao.java | 17 +- .../kachmareugene/MemSegComparatorNull.java | 3 - .../test/kachmareugene/SSTableIterable.java | 28 +++ .../test/kachmareugene/SSTableIterator.java | 32 ++-- .../test/kachmareugene/SSTableRowInfo.java | 14 +- .../kachmareugene/SSTablesController.java | 172 ++++++++++-------- .../ru/vk/itmo/test/kachmareugene/Utils.java | 75 ++++++++ 9 files changed, 261 insertions(+), 107 deletions(-) create mode 100644 src/main/java/ru/vk/itmo/test/kachmareugene/DaoWithCompaction.java create mode 100644 src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterable.java create mode 100644 src/main/java/ru/vk/itmo/test/kachmareugene/Utils.java diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/DaoFactory.java b/src/main/java/ru/vk/itmo/test/kachmareugene/DaoFactory.java index 740ee6d7a..bba99fa88 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/DaoFactory.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/DaoFactory.java @@ -9,7 +9,7 @@ import java.lang.foreign.ValueLayout; import java.nio.charset.StandardCharsets; -@ru.vk.itmo.test.DaoFactory(stage = 3, week = 100) +@ru.vk.itmo.test.DaoFactory(stage = 4, week = 1111) public class DaoFactory implements ru.vk.itmo.test.DaoFactory.Factory> { @Override public String toString(MemorySegment memorySegment) { @@ -34,11 +34,11 @@ public Entry fromBaseEntry(Entry baseEntry) { @Override public Dao> createDao() { - return new InMemoryDao(); + return new DaoWithCompaction(); } @Override public Dao> createDao(Config config) throws IOException { - return new InMemoryDao(config); + return new DaoWithCompaction(config); } } diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/DaoWithCompaction.java b/src/main/java/ru/vk/itmo/test/kachmareugene/DaoWithCompaction.java new file mode 100644 index 000000000..b18717438 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/DaoWithCompaction.java @@ -0,0 +1,21 @@ +package ru.vk.itmo.test.kachmareugene; + +import ru.vk.itmo.Config; +import java.io.IOException; + +public class DaoWithCompaction extends InMemoryDao { + public DaoWithCompaction() { + super(); + } + + public DaoWithCompaction(Config conf) { + super(conf); + } + + @Override + public void compact() throws IOException { + controller.dumpIterator(new SSTableIterable(getMemTable().values(), controller, null, null)); + closeMemTable(); + controller.deleteAllOldFiles(); + } +} diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java b/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java index 030f3ab6b..516bb75d5 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java @@ -17,7 +17,7 @@ public class InMemoryDao implements Dao> { private final SortedMap> mp = new ConcurrentSkipListMap<>(memorySegmentComparatorImpl); - private final SSTablesController controller; + protected final SSTablesController controller; public InMemoryDao() { this.controller = new SSTablesController(new MemSegComparatorNull()); @@ -53,6 +53,7 @@ public Entry get(MemorySegment key) { if (value != null) { return value.value() == null ? null : value; } + var res = controller.getRow(controller.searchInSStables(key)); if (res == null) { return null; @@ -67,7 +68,19 @@ public void upsert(Entry entry) { @Override public void close() throws IOException { - controller.dumpMemTableToSStable(mp); + try { + controller.dumpIterator(mp.values()); + } finally { + mp.clear(); + } + } + + protected void closeMemTable() { mp.clear(); } + + protected SortedMap> getMemTable() { + return mp; + } + } diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/MemSegComparatorNull.java b/src/main/java/ru/vk/itmo/test/kachmareugene/MemSegComparatorNull.java index a7e1e6b6f..eaff57f3c 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/MemSegComparatorNull.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/MemSegComparatorNull.java @@ -5,9 +5,6 @@ public class MemSegComparatorNull extends MemorySegmentComparator { @Override public int compare(MemorySegment segment1, MemorySegment segment2) { - if (segment1 == null && segment2 == null) { - throw new IllegalArgumentException("Incomparable null and null"); - } if (segment1 == null) { return -1; } diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterable.java b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterable.java new file mode 100644 index 000000000..9c875c2c2 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterable.java @@ -0,0 +1,28 @@ +package ru.vk.itmo.test.kachmareugene; + +import ru.vk.itmo.Entry; + +import java.lang.foreign.MemorySegment; +import java.util.Collection; +import java.util.Iterator; + +public class SSTableIterable implements Iterable> { + private final Collection> memTable; + private final SSTablesController controller; + private final MemorySegment from; + private final MemorySegment to; + + public SSTableIterable(Collection> it, SSTablesController controller, + MemorySegment from, MemorySegment to) { + this.memTable = it; + this.controller = controller; + + this.from = from; + this.to = to; + } + + @Override + public Iterator> iterator() { + return new SSTableIterator(memTable.iterator(), controller, from, to); + } +} diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java index 6e096bc94..6d53ce9b3 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java @@ -23,7 +23,7 @@ public class SSTableIterator implements Iterator> { public SSTableIterator(Iterator> it, SSTablesController controller, MemorySegment from, MemorySegment to) { - this.memTableIterator = it; + memTableIterator = it; this.controller = controller; this.from = from; @@ -32,24 +32,26 @@ public SSTableIterator(Iterator> it, SSTablesController con positioningIterator(); } - private void insertNew(final SSTableRowInfo info) { - var curInfo = info; - Entry kv = controller.getRow(curInfo); + private void insertNew(SSTableRowInfo info) { + Entry kv = controller.getRow(info); - while (kv != null) { - SSTableRowInfo old = mp.putIfAbsent(kv.key(), curInfo); - if (old == null) { - return; - } + if (kv == null) { + return; + } - SSTableRowInfo oldInfo = old.ssTableInd > curInfo.ssTableInd ? curInfo : old; - SSTableRowInfo newInfo = old.ssTableInd < curInfo.ssTableInd ? curInfo : old; + if (!mp.containsKey(kv.key())) { + mp.put(kv.key(), info); + return; + } + SSTableRowInfo old = mp.get(kv.key()); - mp.put(controller.getRow(newInfo).key(), newInfo); + SSTableRowInfo oldInfo = old.ssTableInd > info.ssTableInd ? info : old; + SSTableRowInfo newInfo = old.ssTableInd < info.ssTableInd ? info : old; - curInfo = controller.getNextInfo(oldInfo, to); - kv = controller.getRow(curInfo); - } + mp.put(controller.getRow(newInfo).key(), newInfo); + + // tail recursion + insertNew(controller.getNextInfo(oldInfo, to)); } private void positioningIterator() { diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableRowInfo.java b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableRowInfo.java index 29716eb73..309a26400 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableRowInfo.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableRowInfo.java @@ -1,12 +1,12 @@ package ru.vk.itmo.test.kachmareugene; public class SSTableRowInfo { - final long keyOffset; - final long valueOffset; - final long keySize; - final long rowShift; + long keyOffset; + long valueOffset; + long keySize; + long rowShift; private final long valueSize; - final int ssTableInd; + int ssTableInd; public SSTableRowInfo(long keyOffset, long keySize, long valueOffset, long valueSize, int ssTableInd, long rowShift) { @@ -25,4 +25,8 @@ public boolean isDeletedData() { public long getValueSize() { return valueSize; } + + public long totalShift() { + return keyOffset + keySize + valueSize; + } } diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java index 7dfec8800..7341db9e3 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java @@ -8,38 +8,39 @@ import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; import java.nio.channels.FileChannel; +import java.nio.file.FileAlreadyExistsException; import java.nio.file.Files; +import java.nio.file.OpenOption; import java.nio.file.Path; +import java.nio.file.StandardCopyOption; import java.nio.file.StandardOpenOption; -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.time.format.DateTimeFormatter; import java.util.ArrayList; -import java.util.Collections; import java.util.Comparator; +import java.util.Iterator; import java.util.List; -import java.util.SortedMap; +import java.util.Set; import java.util.stream.Stream; +import static ru.vk.itmo.test.kachmareugene.Utils.getValueOrNull; + public class SSTablesController { private final Path ssTablesDir; - private static final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd-HH-mm-ss.SSSSS"); private final List ssTables = new ArrayList<>(); - private final List ssTablesIndexes = new ArrayList<>(); + private final List ssTablesPaths = new ArrayList<>(); private static final String SS_TABLE_COMMON_PREF = "ssTable"; - - // index format: (long) keyOffset, (long) keyLen, (long) valueOffset, (long) valueLen + // index format: (long) keyOffset, (long) keyLen, (long) valueOffset, (long) valueLen private static final long ONE_LINE_SIZE = 4 * Long.BYTES; - private static final String INDEX_COMMON_PREF = "index"; + private static final Set options = Set.of(StandardOpenOption.WRITE, StandardOpenOption.READ, + StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); private final Arena arenaForReading = Arena.ofShared(); + private boolean isClosedArena; private final Comparator segComp; public SSTablesController(Path dir, Comparator com) { this.ssTablesDir = dir; this.segComp = com; - openFiles(dir, SS_TABLE_COMMON_PREF, ssTables); - openFiles(dir, INDEX_COMMON_PREF, ssTablesIndexes); + ssTablesPaths.addAll(openFiles(dir, SS_TABLE_COMMON_PREF, ssTables)); } public SSTablesController(Comparator com) { @@ -47,7 +48,7 @@ public SSTablesController(Comparator com) { this.segComp = com; } - private void openFiles(Path dir, String fileNamePref, List storage) { + private List openFiles(Path dir, String fileNamePref, List storage) { try { Files.createDirectories(dir); } catch (IOException e) { @@ -56,7 +57,7 @@ private void openFiles(Path dir, String fileNamePref, List storag try (Stream tabels = Files.find(dir, 1, (path, ignore) -> path.getFileName().toString().startsWith(fileNamePref))) { final List list = new ArrayList<>(tabels.toList()); - Collections.sort(list); + Utils.sortByNames(list, fileNamePref); list.forEach(t -> { try (FileChannel channel = FileChannel.open(t, StandardOpenOption.READ)) { storage.add(channel.map(FileChannel.MapMode.READ_ONLY, 0, channel.size(), arenaForReading)); @@ -64,33 +65,33 @@ private void openFiles(Path dir, String fileNamePref, List storag throw new UncheckedIOException(e); } }); + return list; } catch (IOException e) { throw new UncheckedIOException(e); } } - private boolean greaterThen(MemorySegment mappedIndex, long lineInBytesOffset, - MemorySegment mappedData, MemorySegment key) { - long offset = mappedIndex.get(ValueLayout.JAVA_LONG, lineInBytesOffset); - long size = mappedIndex.get(ValueLayout.JAVA_LONG, lineInBytesOffset + Long.BYTES); - return segComp.compare(key, mappedData.asSlice(offset, size)) > 0; + private boolean greaterThen(long keyOffset, long keySize, + MemorySegment mapped, MemorySegment key) { + + return segComp.compare(key, mapped.asSlice(keyOffset, keySize)) > 0; } //Gives offset for line in index file - private long searchKeyInFile(MemorySegment mappedIndex, MemorySegment mappedData, MemorySegment key) { + private long searchKeyInFile(int ind, MemorySegment mapped, MemorySegment key) { long l = -1; - long r = mappedIndex.byteSize() / ONE_LINE_SIZE; + long r = getNumberOfEntries(mapped); while (r - l > 1) { long mid = (l + r) / 2; - - if (greaterThen(mappedIndex, mid * ONE_LINE_SIZE, mappedData, key)) { + SSTableRowInfo info = createRowInfo(ind, mid); + if (greaterThen(info.keyOffset, info.keySize, mapped, key)) { l = mid; } else { r = mid; } } - return r == (mappedIndex.byteSize() / ONE_LINE_SIZE) ? -1 : r; + return r == getNumberOfEntries(mapped) ? -1 : r; } //return - List ordered form the latest created sstable to the first. @@ -98,7 +99,10 @@ public List firstGreaterKeys(MemorySegment key) { List ans = new ArrayList<>(); for (int i = ssTables.size() - 1; i >= 0; i--) { - long entryIndexesLine = searchKeyInFile(ssTablesIndexes.get(i), ssTables.get(i), key); + long entryIndexesLine = 0; + if (key != null) { + entryIndexesLine = searchKeyInFile(i, ssTables.get(i), key); + } if (entryIndexesLine < 0) { continue; } @@ -107,18 +111,18 @@ public List firstGreaterKeys(MemorySegment key) { return ans; } - private SSTableRowInfo createRowInfo(int ind, long rowShift) { - long start = ssTablesIndexes.get(ind).get(ValueLayout.JAVA_LONG, rowShift * ONE_LINE_SIZE); - long size = ssTablesIndexes.get(ind).get(ValueLayout.JAVA_LONG, rowShift * ONE_LINE_SIZE + Long.BYTES); + private SSTableRowInfo createRowInfo(int ind, final long rowIndex) { + long start = ssTables.get(ind).get(ValueLayout.JAVA_LONG_UNALIGNED, rowIndex * ONE_LINE_SIZE + Long.BYTES); + long size = ssTables.get(ind).get(ValueLayout.JAVA_LONG_UNALIGNED, rowIndex * ONE_LINE_SIZE + Long.BYTES * 2); - long start1 = ssTablesIndexes.get(ind).get(ValueLayout.JAVA_LONG,rowShift * ONE_LINE_SIZE + Long.BYTES * 2); - long size1 = ssTablesIndexes.get(ind).get(ValueLayout.JAVA_LONG,rowShift * ONE_LINE_SIZE + Long.BYTES * 3); - return new SSTableRowInfo(start, size, start1, size1, ind, rowShift); + long start1 = ssTables.get(ind).get(ValueLayout.JAVA_LONG_UNALIGNED,rowIndex * ONE_LINE_SIZE + Long.BYTES * 3); + long size1 = ssTables.get(ind).get(ValueLayout.JAVA_LONG_UNALIGNED,rowIndex * ONE_LINE_SIZE + Long.BYTES * 4); + return new SSTableRowInfo(start, size, start1, size1, ind, rowIndex); } public SSTableRowInfo searchInSStables(MemorySegment key) { - for (int i = ssTablesIndexes.size() - 1; i >= 0; i--) { - long ind = searchKeyInFile(ssTablesIndexes.get(i), ssTables.get(i), key); + for (int i = ssTables.size() - 1; i >= 0; i--) { + long ind = searchKeyInFile(i, ssTables.get(i), key); if (ind >= 0) { return createRowInfo(i, ind); } @@ -145,7 +149,7 @@ public Entry getRow(SSTableRowInfo info) { * Ignores deleted values. */ public SSTableRowInfo getNextInfo(SSTableRowInfo info, MemorySegment maxKey) { - for (long t = info.rowShift + 1; t < ssTablesIndexes.get(info.ssTableInd).byteSize() / ONE_LINE_SIZE; t++) { + for (long t = info.rowShift + 1; t < getNumberOfEntries(ssTables.get(info.ssTableInd)); t++) { var inf = createRowInfo(info.ssTableInd, t); Entry row = getRow(inf); @@ -153,78 +157,88 @@ public SSTableRowInfo getNextInfo(SSTableRowInfo info, MemorySegment maxKey) { return inf; } } - return null; } - private long dumpLong(MemorySegment mapped, long value, long offset) { - mapped.set(ValueLayout.JAVA_LONG, offset, value); - return offset + Long.BYTES; + private long getNumberOfEntries(MemorySegment memSeg) { + return memSeg.get(ValueLayout.JAVA_LONG_UNALIGNED, 0); } - private long dumpSegment(MemorySegment mapped, MemorySegment data, long offset) { - MemorySegment.copy(data, 0, mapped, offset, data.byteSize()); - return offset + data.byteSize(); - } - - public void dumpMemTableToSStable(SortedMap> mp) throws IOException { + public void dumpIterator(Iterable> iter) throws IOException { + Iterator> iter1 = iter.iterator(); - if (ssTablesDir == null || mp.isEmpty()) { - arenaForReading.close(); + if (ssTablesDir == null || !iter1.hasNext()) { + closeArena(); return; } - LocalDateTime time = LocalDateTime.now(ZoneId.systemDefault()); + String suff = String.valueOf(Utils.getMaxNumberOfFile(ssTablesDir, SS_TABLE_COMMON_PREF) + 1); + + final Path tmpFile = ssTablesDir.resolve("data.tmp"); + final Path targetFile = ssTablesDir.resolve(SS_TABLE_COMMON_PREF + suff); + + try { + Files.createFile(ssTablesDir.resolve(SS_TABLE_COMMON_PREF + suff)); + } catch (FileAlreadyExistsException ignored) { + // it is ok, actually it is normal state + } + try (FileChannel ssTableChannel = - FileChannel.open(ssTablesDir.resolve(SS_TABLE_COMMON_PREF + formatter.format(time)), - StandardOpenOption.WRITE, StandardOpenOption.READ, StandardOpenOption.CREATE); - FileChannel indexChannel = - FileChannel.open(ssTablesDir.resolve(INDEX_COMMON_PREF + formatter.format(time)), - StandardOpenOption.WRITE, StandardOpenOption.READ, StandardOpenOption.CREATE); + FileChannel.open(tmpFile, options); Arena saveArena = Arena.ofConfined()) { long ssTableLenght = 0L; - long indexLength = mp.size() * ONE_LINE_SIZE; + long indexLength = 0L; - for (var kv : mp.values()) { - ssTableLenght += - kv.key().byteSize() + getValueOrNull(kv).byteSize(); + while (iter1.hasNext()) { + var seg = iter1.next(); + ssTableLenght += seg.key().byteSize() + getValueOrNull(seg).byteSize(); + indexLength += ONE_LINE_SIZE; } - long currOffsetSSTable = 0L; - long currOffsetIndex = 0L; MemorySegment mappedSSTable = ssTableChannel.map( - FileChannel.MapMode.READ_WRITE, currOffsetSSTable, ssTableLenght, saveArena); + FileChannel.MapMode.READ_WRITE, currOffsetSSTable, ssTableLenght + indexLength + Long.BYTES, + saveArena); - MemorySegment mappedIndex = indexChannel.map( - FileChannel.MapMode.READ_WRITE, currOffsetIndex, indexLength, saveArena); + currOffsetSSTable = Utils.dumpLong(mappedSSTable, indexLength / ONE_LINE_SIZE, currOffsetSSTable); - for (var kv : mp.values()) { - currOffsetIndex = dumpLong(mappedIndex, currOffsetSSTable, currOffsetIndex); - currOffsetSSTable = dumpSegment(mappedSSTable, kv.key(), currOffsetSSTable); - currOffsetIndex = dumpLong(mappedIndex, kv.key().byteSize(), currOffsetIndex); + long shiftForData = indexLength + Long.BYTES; + + for (Entry kv : iter) { + // key offset + currOffsetSSTable = Utils.dumpLong(mappedSSTable, shiftForData, currOffsetSSTable); + // key length + currOffsetSSTable = Utils.dumpLong(mappedSSTable, kv.key().byteSize(), currOffsetSSTable); + shiftForData += kv.key().byteSize(); + + // value offset + currOffsetSSTable = Utils.dumpLong(mappedSSTable, shiftForData, currOffsetSSTable); + // value length + currOffsetSSTable = Utils.dumpLong(mappedSSTable, Utils.rightByteSize(kv), currOffsetSSTable); + shiftForData += getValueOrNull(kv).byteSize(); + } - currOffsetIndex = dumpLong(mappedIndex, currOffsetSSTable, currOffsetIndex); - currOffsetSSTable = dumpSegment(mappedSSTable, getValueOrNull(kv), currOffsetSSTable); - currOffsetIndex = dumpLong(mappedIndex, rightByteSize(kv), currOffsetIndex); + for (Entry kv : iter) { + currOffsetSSTable = Utils.dumpSegment(mappedSSTable, kv.key(), currOffsetSSTable); + currOffsetSSTable = Utils.dumpSegment(mappedSSTable, getValueOrNull(kv), currOffsetSSTable); } + + Files.move(tmpFile, targetFile, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + } finally { - arenaForReading.close(); + closeArena(); } } - private long rightByteSize(Entry memSeg) { - if (memSeg.value() == null) { - return -1; + private void closeArena() { + if (!isClosedArena) { + arenaForReading.close(); } - return memSeg.value().byteSize(); + isClosedArena = true; } - private MemorySegment getValueOrNull(Entry kv) { - MemorySegment value = kv.value(); - if (kv.value() == null) { - value = MemorySegment.NULL; - } - return value; + public void deleteAllOldFiles() throws IOException { + closeArena(); + Utils.deleteFiles(ssTablesPaths); } } diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/Utils.java b/src/main/java/ru/vk/itmo/test/kachmareugene/Utils.java new file mode 100644 index 000000000..77886d9d3 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/Utils.java @@ -0,0 +1,75 @@ +package ru.vk.itmo.test.kachmareugene; + +import ru.vk.itmo.Entry; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Comparator; +import java.util.List; +import java.util.stream.Stream; + +public final class Utils { + private Utils() { + + } + + public static long dumpLong(MemorySegment mapped, long value, long offset) { + mapped.set(ValueLayout.JAVA_LONG, offset, value); + return offset + Long.BYTES; + } + + public static long dumpSegment(MemorySegment mapped, MemorySegment data, long offset) { + MemorySegment.copy(data, 0, mapped, offset, data.byteSize()); + return offset + data.byteSize(); + } + + public static long getNumberFromFileName(Path pathToFile, String prefix) { + return Long.parseLong(pathToFile.getFileName().toString().substring(prefix.length())); + } + + public static void sortByNames(List l, String prefix) { + l.sort(Comparator.comparingLong(s -> getNumberFromFileName(s, prefix))); + } + + public static long getMaxNumberOfFile(Path dir, String prefix) { + try (Stream tabels = Files.find(dir, 1, + (path, ignore) -> path.getFileName().toString().startsWith(prefix))) { + final List list = tabels.toList(); + long maxi = 0; + for (Path p : list) { + if (getNumberFromFileName(p, prefix) > maxi) { + maxi = getNumberFromFileName(p, prefix); + } + } + return maxi; + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + public static void deleteFiles(List files) throws IOException { + for (Path file : files) { + Files.deleteIfExists(file); + } + files.clear(); + } + + public static long rightByteSize(Entry memSeg) { + if (memSeg.value() == null) { + return -1; + } + return memSeg.value().byteSize(); + } + + public static MemorySegment getValueOrNull(Entry kv) { + MemorySegment value = kv.value(); + if (kv.value() == null) { + value = MemorySegment.NULL; + } + return value; + } +} From 1fd5c14f5604a9bde2aca4779b00b7097797ffa5 Mon Sep 17 00:00:00 2001 From: Ilya Abramov <45769461+IlyaAbramovv@users.noreply.github.com> Date: Mon, 4 Dec 2023 03:28:49 +0300 Subject: [PATCH 11/20] =?UTF-8?q?=D0=90=D0=B1=D1=80=D0=B0=D0=BC=D0=BE?= =?UTF-8?q?=D0=B2=20=D0=98=D0=BB=D1=8C=D1=8F,=20=D0=98=D0=A2=D0=9C=D0=9E?= =?UTF-8?q?=20=D0=A4=D0=98=D0=A2=D0=B8=D0=9F,=20=D0=9C33341,=20HW4=20(#218?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Vadim Tsesko Co-authored-by: Daniil Ushkov <42135591+daniil-ushkov@users.noreply.github.com> --- .../java/ru/vk/itmo/abramovilya/DaoImpl.java | 217 +++---------- .../ru/vk/itmo/abramovilya/DaoIterator.java | 46 +-- .../java/ru/vk/itmo/abramovilya/Storage.java | 293 ++++++++++++++++++ .../itmo/abramovilya/StorageFileWriter.java | 149 +++++++++ .../ru/vk/itmo/abramovilya/StorageWriter.java | 96 ------ .../itmo/test/abramovilya/DaoFactoryImpl.java | 2 +- 6 files changed, 493 insertions(+), 310 deletions(-) create mode 100644 src/main/java/ru/vk/itmo/abramovilya/Storage.java create mode 100644 src/main/java/ru/vk/itmo/abramovilya/StorageFileWriter.java delete mode 100644 src/main/java/ru/vk/itmo/abramovilya/StorageWriter.java diff --git a/src/main/java/ru/vk/itmo/abramovilya/DaoImpl.java b/src/main/java/ru/vk/itmo/abramovilya/DaoImpl.java index d3ade0446..2648082c5 100644 --- a/src/main/java/ru/vk/itmo/abramovilya/DaoImpl.java +++ b/src/main/java/ru/vk/itmo/abramovilya/DaoImpl.java @@ -1,6 +1,5 @@ package ru.vk.itmo.abramovilya; -import ru.vk.itmo.BaseEntry; import ru.vk.itmo.Config; import ru.vk.itmo.Dao; import ru.vk.itmo.Entry; @@ -9,66 +8,23 @@ import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; -import java.nio.channels.FileChannel; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardOpenOption; -import java.util.ArrayList; import java.util.Iterator; -import java.util.List; import java.util.concurrent.ConcurrentNavigableMap; import java.util.concurrent.ConcurrentSkipListMap; public class DaoImpl implements Dao> { private final ConcurrentNavigableMap> map = new ConcurrentSkipListMap<>(DaoImpl::compareMemorySegments); - private final Path storagePath; private final Arena arena = Arena.ofShared(); - private static final String SSTABLE_BASE_NAME = "storage"; - private static final String INDEX_BASE_NAME = "table"; - private final Path metaFilePath; - private final List sstableFileChannels = new ArrayList<>(); - private final List sstableMappedList = new ArrayList<>(); - private final List indexFileChannels = new ArrayList<>(); - private final List indexMappedList = new ArrayList<>(); + private final Storage storage; public DaoImpl(Config config) throws IOException { - storagePath = config.basePath(); - - Files.createDirectories(storagePath); - metaFilePath = storagePath.resolve("meta"); - if (!Files.exists(metaFilePath)) { - Files.createFile(metaFilePath); - Files.writeString(metaFilePath, "0", StandardOpenOption.WRITE); - } - - int totalSSTables = Integer.parseInt(Files.readString(metaFilePath)); - for (int sstableNum = 0; sstableNum < totalSSTables; sstableNum++) { - Path sstablePath = storagePath.resolve(SSTABLE_BASE_NAME + sstableNum); - Path indexPath = storagePath.resolve(INDEX_BASE_NAME + sstableNum); - - FileChannel sstableFileChannel = FileChannel.open(sstablePath, StandardOpenOption.READ); - sstableFileChannels.add(sstableFileChannel); - MemorySegment sstableMapped = - sstableFileChannel.map(FileChannel.MapMode.READ_ONLY, 0, Files.size(sstablePath), arena); - sstableMappedList.add(sstableMapped); - - FileChannel indexFileChannel = FileChannel.open(indexPath, StandardOpenOption.READ); - indexFileChannels.add(indexFileChannel); - MemorySegment indexMapped = - indexFileChannel.map(FileChannel.MapMode.READ_ONLY, 0, Files.size(indexPath), arena); - indexMappedList.add(indexMapped); - } + this.storage = new Storage(config, arena); } @Override public Iterator> get(MemorySegment from, MemorySegment to) { - return new DaoIterator(getTotalSStables(), from, to, sstableMappedList, indexMappedList, map); - } - - @Override - public void upsert(Entry entry) { - map.put(entry.key(), entry); + return new DaoIterator(storage.getTotalSStables(), from, to, storage, map); } @Override @@ -80,116 +36,29 @@ public Entry get(MemorySegment key) { } return null; } - - int totalSStables = getTotalSStables(); - for (int sstableNum = totalSStables; sstableNum >= 0; sstableNum--) { - var foundEntry = seekForValueInFile(key, sstableNum); - if (foundEntry != null) { - if (foundEntry.value() != null) { - return foundEntry; - } - return null; - } - } - return null; + return storage.get(key); } - private Entry seekForValueInFile(MemorySegment key, int sstableNum) { - if (sstableNum >= sstableFileChannels.size()) { - return null; - } - - MemorySegment storageMapped = sstableMappedList.get(sstableNum); - MemorySegment indexMapped = indexMappedList.get(sstableNum); - - int foundIndex = upperBound(key, storageMapped, indexMapped, indexMapped.byteSize()); - long keyStorageOffset = getKeyStorageOffset(indexMapped, foundIndex); - long foundKeySize = storageMapped.get(ValueLayout.JAVA_LONG_UNALIGNED, keyStorageOffset); - keyStorageOffset += Long.BYTES; - - if (MemorySegment.mismatch(key, - 0, - key.byteSize(), - storageMapped, - keyStorageOffset, - keyStorageOffset + foundKeySize) == -1) { - return getEntryFromIndexFile(storageMapped, indexMapped, foundIndex); - } - return null; - } - - static int upperBound(MemorySegment key, MemorySegment storageMapped, MemorySegment indexMapped, long indexSize) { - int l = -1; - int r = indexMapped.get(ValueLayout.JAVA_INT_UNALIGNED, indexSize - Long.BYTES - Integer.BYTES); - - while (r - l > 1) { - int m = (r + l) / 2; - long keyStorageOffset = getKeyStorageOffset(indexMapped, m); - long keySize = storageMapped.get(ValueLayout.JAVA_LONG_UNALIGNED, keyStorageOffset); - keyStorageOffset += Long.BYTES; - - if (compareMemorySegmentsUsingOffset(key, storageMapped, keyStorageOffset, keySize) > 0) { - l = m; - } else { - r = m; - } - } - return r; - } - - static long getKeyStorageOffset(MemorySegment indexMapped, int entryNum) { - return indexMapped.get( - ValueLayout.JAVA_LONG_UNALIGNED, - (long) (Integer.BYTES + Long.BYTES) * entryNum + Integer.BYTES - ); + @Override + public void upsert(Entry entry) { + map.put(entry.key(), entry); } - private Entry getEntryFromIndexFile(MemorySegment storageMapped, - MemorySegment indexMapped, - int entryNum) { - long offsetInStorageFile = indexMapped.get( - ValueLayout.JAVA_LONG_UNALIGNED, - (long) (Integer.BYTES + Long.BYTES) * entryNum + Integer.BYTES - ); - - long keySize = storageMapped.get(ValueLayout.JAVA_LONG_UNALIGNED, offsetInStorageFile); - offsetInStorageFile += Long.BYTES; - offsetInStorageFile += keySize; - - long valueSize = storageMapped.get(ValueLayout.JAVA_LONG_UNALIGNED, offsetInStorageFile); - offsetInStorageFile += Long.BYTES; - MemorySegment key = storageMapped.asSlice(offsetInStorageFile - keySize - Long.BYTES, keySize); - MemorySegment value; - if (valueSize == -1) { - value = null; - } else { - value = storageMapped.asSlice(offsetInStorageFile, valueSize); + @Override + public void compact() throws IOException { + var iterator = get(null, null); + if (!iterator.hasNext()) { + return; } - return new BaseEntry<>(key, value); + storage.compact(iterator, get(null, null)); + map.clear(); } @Override public void flush() throws IOException { - writeMapIntoFile(); - if (!map.isEmpty()) incTotalSStablesAmount(); - } - - private void incTotalSStablesAmount() throws IOException { - int totalSStables = getTotalSStables(); - Files.writeString(metaFilePath, String.valueOf(totalSStables + 1)); - } - - @Override - public void close() throws IOException { - if (arena.scope().isAlive()) { - arena.close(); - } - flush(); - for (FileChannel fc : sstableFileChannels) { - if (fc.isOpen()) fc.close(); - } - for (FileChannel fc : indexFileChannels) { - if (fc.isOpen()) fc.close(); + if (!map.isEmpty()) { + writeMapIntoFile(); + storage.incTotalSStablesAmount(); } } @@ -197,30 +66,17 @@ private void writeMapIntoFile() throws IOException { if (map.isEmpty()) { return; } - - int currSStableNum = getTotalSStables(); - Path sstablePath = storagePath.resolve(SSTABLE_BASE_NAME + currSStableNum); - Path indexPath = storagePath.resolve(INDEX_BASE_NAME + currSStableNum); - - StorageWriter.writeSStableAndIndex(sstablePath, - calcMapByteSizeInFile(), - indexPath, - calcIndexByteSizeInFile(), - map); - } - - private int getTotalSStables() { - return sstableFileChannels.size(); - } - - private long calcIndexByteSizeInFile() { - return (long) map.size() * (Integer.BYTES + Long.BYTES); + storage.writeMapIntoFile( + mapByteSizeInFile(), + indexByteSizeInFile(), + map + ); } - private long calcMapByteSizeInFile() { + private long mapByteSizeInFile() { long size = 0; for (var entry : map.values()) { - size += 2 * Long.BYTES; + size += Storage.BYTES_TO_STORE_ENTRY_SIZE; size += entry.key().byteSize(); if (entry.value() != null) { size += entry.value().byteSize(); @@ -229,17 +85,29 @@ private long calcMapByteSizeInFile() { return size; } + private long indexByteSizeInFile() { + return (long) map.size() * Storage.INDEX_ENTRY_SIZE; + } + + @Override + public void close() throws IOException { + if (arena.scope().isAlive()) { + arena.close(); + } + flush(); + storage.close(); + } + public static int compareMemorySegments(MemorySegment segment1, MemorySegment segment2) { - long mismatch = segment1.mismatch(segment2); - if (mismatch == -1) { + long offset = segment1.mismatch(segment2); + if (offset == -1) { return 0; - } else if (mismatch == segment1.byteSize()) { + } else if (offset == segment1.byteSize()) { return -1; - } else if (mismatch == segment2.byteSize()) { + } else if (offset == segment2.byteSize()) { return 1; } - return Byte.compare(segment1.get(ValueLayout.JAVA_BYTE, mismatch), - segment2.get(ValueLayout.JAVA_BYTE, mismatch)); + return Byte.compare(segment1.get(ValueLayout.JAVA_BYTE, offset), segment2.get(ValueLayout.JAVA_BYTE, offset)); } public static int compareMemorySegmentsUsingOffset(MemorySegment segment1, @@ -261,6 +129,5 @@ public static int compareMemorySegmentsUsingOffset(MemorySegment segment1, } return Byte.compare(segment1.get(ValueLayout.JAVA_BYTE, mismatch), segment2.get(ValueLayout.JAVA_BYTE, segment2Offset + mismatch)); - } } diff --git a/src/main/java/ru/vk/itmo/abramovilya/DaoIterator.java b/src/main/java/ru/vk/itmo/abramovilya/DaoIterator.java index d62fbd2b7..fdc8de6a2 100644 --- a/src/main/java/ru/vk/itmo/abramovilya/DaoIterator.java +++ b/src/main/java/ru/vk/itmo/abramovilya/DaoIterator.java @@ -7,9 +7,7 @@ import ru.vk.itmo.abramovilya.table.TableEntry; import java.lang.foreign.MemorySegment; -import java.lang.foreign.ValueLayout; import java.util.Iterator; -import java.util.List; import java.util.NavigableMap; import java.util.NoSuchElementException; import java.util.PriorityQueue; @@ -18,19 +16,17 @@ class DaoIterator implements Iterator> { private final PriorityQueue priorityQueue = new PriorityQueue<>(); private final MemorySegment from; private final MemorySegment to; - private final List sstableMappedList; - private final List indexMappedList; + private final Storage storage; DaoIterator(int totalSStables, MemorySegment from, MemorySegment to, - List sstableMappedList, - List indexMappedList, + Storage storage, NavigableMap> memTable) { + this.from = from; this.to = to; - this.sstableMappedList = sstableMappedList; - this.indexMappedList = indexMappedList; + this.storage = storage; NavigableMap> subMap = getSubMap(memTable); for (int i = 0; i < totalSStables; i++) { @@ -39,8 +35,8 @@ class DaoIterator implements Iterator> { priorityQueue.add(new SSTable( i, offset, - sstableMappedList.get(i), - indexMappedList.get(i) + storage.mappedSStable(i), + storage.mappedIndex(i) ).currentEntry()); } } @@ -115,33 +111,7 @@ private void cleanUpSStableQueue() { } } - private long findOffsetInIndex(MemorySegment from, MemorySegment to, int i) { - long readOffset = 0; - MemorySegment storageMapped = sstableMappedList.get(i); - MemorySegment indexMapped = indexMappedList.get(i); - - if (from == null && to == null) { - return Integer.BYTES; - } else if (from == null) { - long firstKeySize = storageMapped.get(ValueLayout.JAVA_LONG_UNALIGNED, readOffset); - readOffset += Long.BYTES; - MemorySegment firstKey = storageMapped.asSlice(readOffset, firstKeySize); - if (DaoImpl.compareMemorySegments(firstKey, to) >= 0) { - return -1; - } - return Integer.BYTES; - } else { - int foundIndex = DaoImpl.upperBound(from, storageMapped, indexMapped, indexMapped.byteSize()); - long keyStorageOffset = DaoImpl.getKeyStorageOffset(indexMapped, foundIndex); - long keySize = storageMapped.get(ValueLayout.JAVA_LONG_UNALIGNED, keyStorageOffset); - keyStorageOffset += Long.BYTES; - - if (DaoImpl.compareMemorySegmentsUsingOffset(from, storageMapped, keyStorageOffset, keySize) > 0 - || (to != null && DaoImpl.compareMemorySegmentsUsingOffset( - to, storageMapped, keyStorageOffset, keySize) <= 0)) { - return -1; - } - return (long) foundIndex * (Integer.BYTES + Long.BYTES) + Integer.BYTES; - } + private long findOffsetInIndex(MemorySegment from, MemorySegment to, int fileNum) { + return storage.findOffsetInIndex(from, to, fileNum); } } diff --git a/src/main/java/ru/vk/itmo/abramovilya/Storage.java b/src/main/java/ru/vk/itmo/abramovilya/Storage.java new file mode 100644 index 000000000..abcb788dd --- /dev/null +++ b/src/main/java/ru/vk/itmo/abramovilya/Storage.java @@ -0,0 +1,293 @@ +package ru.vk.itmo.abramovilya; + +import ru.vk.itmo.BaseEntry; +import ru.vk.itmo.Config; +import ru.vk.itmo.Entry; + +import java.io.Closeable; +import java.io.IOException; +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.nio.channels.FileChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.NavigableMap; + +class Storage implements Closeable { + private static final String COMPACTED_SUFFIX = "_compacted"; + private static final String COMPACTING_SUFFIX = "_compacting"; + private static final String SSTABLE_BASE_NAME = "storage"; + private static final String INDEX_BASE_NAME = "index"; + public static final String META_FILE_BASE_NAME = "meta"; + public static final int BYTES_TO_STORE_ENTRY_ELEMENT_SIZE = Long.BYTES; + public static final int BYTES_TO_STORE_ENTRY_SIZE = 2 * BYTES_TO_STORE_ENTRY_ELEMENT_SIZE; + public static final int BYTES_TO_STORE_INDEX_KEY = Integer.BYTES; + public static final long INDEX_ENTRY_SIZE = BYTES_TO_STORE_INDEX_KEY + BYTES_TO_STORE_ENTRY_ELEMENT_SIZE; + private final Path storagePath; + private final Path metaFilePath; + private final List sstableFileChannels = new ArrayList<>(); + private final List sstableMappedList = new ArrayList<>(); + private final List indexFileChannels = new ArrayList<>(); + private final List indexMappedList = new ArrayList<>(); + + Storage(Config config, Arena arena) throws IOException { + storagePath = config.basePath(); + + Files.createDirectories(storagePath); + metaFilePath = storagePath.resolve(META_FILE_BASE_NAME); + if (!Files.exists(metaFilePath)) { + Files.createFile(metaFilePath); + + int totalSStables = 0; + Files.writeString(metaFilePath, String.valueOf(totalSStables), StandardOpenOption.WRITE); + } + + // Restore consistent state if db was dropped during compaction + if (Files.exists(storagePath.resolve(SSTABLE_BASE_NAME + COMPACTED_SUFFIX)) + || Files.exists(storagePath.resolve(INDEX_BASE_NAME + COMPACTED_SUFFIX))) { + finishCompact(); + } + + // Delete artifacts from unsuccessful compaction + Files.deleteIfExists(storagePath.resolve(SSTABLE_BASE_NAME + COMPACTING_SUFFIX)); + Files.deleteIfExists(storagePath.resolve(INDEX_BASE_NAME + COMPACTING_SUFFIX)); + + int totalSSTables = Integer.parseInt(Files.readString(metaFilePath)); + for (int sstableNum = 0; sstableNum < totalSSTables; sstableNum++) { + Path sstablePath = storagePath.resolve(SSTABLE_BASE_NAME + sstableNum); + Path indexPath = storagePath.resolve(INDEX_BASE_NAME + sstableNum); + + FileChannel sstableFileChannel = FileChannel.open(sstablePath, StandardOpenOption.READ); + sstableFileChannels.add(sstableFileChannel); + MemorySegment sstableMapped = + sstableFileChannel.map(FileChannel.MapMode.READ_ONLY, 0, Files.size(sstablePath), arena); + sstableMappedList.add(sstableMapped); + + FileChannel indexFileChannel = FileChannel.open(indexPath, StandardOpenOption.READ); + indexFileChannels.add(indexFileChannel); + MemorySegment indexMapped = + indexFileChannel.map(FileChannel.MapMode.READ_ONLY, 0, Files.size(indexPath), arena); + indexMappedList.add(indexMapped); + } + } + + Entry get(MemorySegment key) { + int totalSStables = getTotalSStables(); + for (int sstableNum = totalSStables; sstableNum >= 0; sstableNum--) { + var foundEntry = seekForValueInFile(key, sstableNum); + if (foundEntry != null) { + if (foundEntry.value() != null) { + return foundEntry; + } + return null; + } + } + return null; + } + + final int getTotalSStables() { + return sstableFileChannels.size(); + } + + private Entry seekForValueInFile(MemorySegment key, int sstableNum) { + if (sstableNum >= sstableFileChannels.size()) { + return null; + } + + MemorySegment storageMapped = sstableMappedList.get(sstableNum); + MemorySegment indexMapped = indexMappedList.get(sstableNum); + + int foundIndex = upperBound(key, storageMapped, indexMapped, indexMapped.byteSize()); + long keyStorageOffset = getKeyStorageOffset(indexMapped, foundIndex); + long foundKeySize = storageMapped.get(ValueLayout.JAVA_LONG_UNALIGNED, keyStorageOffset); + keyStorageOffset += BYTES_TO_STORE_ENTRY_ELEMENT_SIZE; + + if (MemorySegment.mismatch(key, + 0, + key.byteSize(), + storageMapped, + keyStorageOffset, + keyStorageOffset + foundKeySize) == -1) { + return getEntryFromIndexFile(storageMapped, indexMapped, foundIndex); + } + return null; + } + + static long getKeyStorageOffset(MemorySegment indexMapped, int entryNum) { + return indexMapped.get( + ValueLayout.JAVA_LONG_UNALIGNED, + INDEX_ENTRY_SIZE * entryNum + BYTES_TO_STORE_INDEX_KEY + ); + } + + private Entry getEntryFromIndexFile(MemorySegment sstableMapped, + MemorySegment indexMapped, + int entryNum) { + long offsetInStorageFile = indexMapped.get( + ValueLayout.JAVA_LONG_UNALIGNED, + INDEX_ENTRY_SIZE * entryNum + BYTES_TO_STORE_INDEX_KEY + ); + + long keySize = sstableMapped.get(ValueLayout.JAVA_LONG_UNALIGNED, offsetInStorageFile); + offsetInStorageFile += BYTES_TO_STORE_ENTRY_ELEMENT_SIZE; + offsetInStorageFile += keySize; + + long valueSize = sstableMapped.get(ValueLayout.JAVA_LONG_UNALIGNED, offsetInStorageFile); + offsetInStorageFile += BYTES_TO_STORE_ENTRY_ELEMENT_SIZE; + MemorySegment key = sstableMapped.asSlice( + offsetInStorageFile - keySize - BYTES_TO_STORE_ENTRY_ELEMENT_SIZE, keySize); + MemorySegment value; + if (valueSize == -1) { + value = null; + } else { + value = sstableMapped.asSlice(offsetInStorageFile, valueSize); + } + return new BaseEntry<>(key, value); + } + + void writeMapIntoFile(long sstableSize, long indexSize, NavigableMap> map) + throws IOException { + + int totalSStables = getTotalSStables(); + Path sstablePath = storagePath.resolve(Storage.SSTABLE_BASE_NAME + totalSStables); + Path indexPath = storagePath.resolve(Storage.INDEX_BASE_NAME + totalSStables); + StorageFileWriter.writeMapIntoFile(sstableSize, indexSize, map, sstablePath, indexPath); + } + + private Entry calcCompactedSStableIndexSize(Iterator> iterator) { + long storageSize = 0; + long indexSize = 0; + while (iterator.hasNext()) { + Entry entry = iterator.next(); + storageSize += entry.key().byteSize() + entry.value().byteSize() + BYTES_TO_STORE_ENTRY_SIZE; + indexSize += INDEX_ENTRY_SIZE; + } + return new BaseEntry<>(storageSize, indexSize); + } + + void incTotalSStablesAmount() throws IOException { + int totalSStables = getTotalSStables(); + Files.writeString(metaFilePath, String.valueOf(totalSStables + 1)); + } + + @Override + public void close() throws IOException { + for (FileChannel fc : sstableFileChannels) { + if (fc.isOpen()) fc.close(); + } + for (FileChannel fc : indexFileChannels) { + if (fc.isOpen()) fc.close(); + } + } + + public MemorySegment mappedSStable(int i) { + return sstableMappedList.get(i); + } + + public MemorySegment mappedIndex(int i) { + return indexMappedList.get(i); + } + + void compact(Iterator> iterator1, Iterator> iterator2) + throws IOException { + Entry storageIndexSize = calcCompactedSStableIndexSize(iterator1); + Path compactingSStablePath = storagePath.resolve(SSTABLE_BASE_NAME + COMPACTING_SUFFIX); + Path compactingIndexPath = storagePath.resolve(INDEX_BASE_NAME + COMPACTING_SUFFIX); + StorageFileWriter.writeIteratorIntoFile(storageIndexSize.key(), + storageIndexSize.value(), + iterator2, + compactingSStablePath, + compactingIndexPath); + + // Move to ensure that compacting completed successfully + Path compactedSStablePath = storagePath.resolve(SSTABLE_BASE_NAME + COMPACTED_SUFFIX); + Path compactedIndexPath = storagePath.resolve(INDEX_BASE_NAME + COMPACTED_SUFFIX); + Files.move(compactingSStablePath, compactedSStablePath, StandardCopyOption.ATOMIC_MOVE); + Files.move(compactingIndexPath, compactedIndexPath, StandardCopyOption.ATOMIC_MOVE); + + finishCompact(); + } + + private void finishCompact() throws IOException { + int totalSStables = getTotalSStables(); + for (int i = 0; i < totalSStables; i++) { + Files.deleteIfExists(storagePath.resolve(SSTABLE_BASE_NAME + i)); + Files.deleteIfExists(storagePath.resolve(INDEX_BASE_NAME + i)); + } + + Files.writeString(metaFilePath, String.valueOf(1)); + Path compactedSStablePath = storagePath.resolve(SSTABLE_BASE_NAME + COMPACTED_SUFFIX); + if (Files.exists(compactedSStablePath)) { + Files.move(compactedSStablePath, + storagePath.resolve(SSTABLE_BASE_NAME + 0), + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING); + } + Path compactedIndexPath = storagePath.resolve(INDEX_BASE_NAME + COMPACTED_SUFFIX); + + if (Files.exists(compactedIndexPath)) { + Files.move(compactedIndexPath, + storagePath.resolve(INDEX_BASE_NAME + 0), + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING); + } + } + + long findOffsetInIndex(MemorySegment from, MemorySegment to, int fileNum) { + long readOffset = 0; + MemorySegment storageMapped = sstableMappedList.get(fileNum); + MemorySegment indexMapped = indexMappedList.get(fileNum); + + if (from == null && to == null) { + return BYTES_TO_STORE_INDEX_KEY; + } else if (from == null) { + long firstKeySize = storageMapped.get(ValueLayout.JAVA_LONG_UNALIGNED, readOffset); + readOffset += BYTES_TO_STORE_ENTRY_ELEMENT_SIZE; + MemorySegment firstKey = storageMapped.asSlice(readOffset, firstKeySize); + if (DaoImpl.compareMemorySegments(firstKey, to) >= 0) { + return -1; + } + return BYTES_TO_STORE_INDEX_KEY; + } else { + int foundIndex = upperBound(from, storageMapped, indexMapped, indexMapped.byteSize()); + long keyStorageOffset = getKeyStorageOffset(indexMapped, foundIndex); + long keySize = storageMapped.get(ValueLayout.JAVA_LONG_UNALIGNED, keyStorageOffset); + keyStorageOffset += BYTES_TO_STORE_ENTRY_ELEMENT_SIZE; + + if (DaoImpl.compareMemorySegmentsUsingOffset(from, storageMapped, keyStorageOffset, keySize) > 0 + || (to != null && DaoImpl.compareMemorySegmentsUsingOffset( + to, storageMapped, keyStorageOffset, keySize) <= 0)) { + return -1; + } + return (long) foundIndex * INDEX_ENTRY_SIZE + BYTES_TO_STORE_INDEX_KEY; + } + } + + private static int upperBound(MemorySegment key, + MemorySegment storageMapped, + MemorySegment indexMapped, + long indexSize) { + int l = -1; + int r = indexMapped.get(ValueLayout.JAVA_INT_UNALIGNED, indexSize - INDEX_ENTRY_SIZE); + + while (r - l > 1) { + int m = (r + l) / 2; + long keyStorageOffset = getKeyStorageOffset(indexMapped, m); + long keySize = storageMapped.get(ValueLayout.JAVA_LONG_UNALIGNED, keyStorageOffset); + keyStorageOffset += BYTES_TO_STORE_ENTRY_ELEMENT_SIZE; + + if (DaoImpl.compareMemorySegmentsUsingOffset(key, storageMapped, keyStorageOffset, keySize) > 0) { + l = m; + } else { + r = m; + } + } + return r; + } +} diff --git a/src/main/java/ru/vk/itmo/abramovilya/StorageFileWriter.java b/src/main/java/ru/vk/itmo/abramovilya/StorageFileWriter.java new file mode 100644 index 000000000..67dceed60 --- /dev/null +++ b/src/main/java/ru/vk/itmo/abramovilya/StorageFileWriter.java @@ -0,0 +1,149 @@ +package ru.vk.itmo.abramovilya; + +import ru.vk.itmo.Entry; + +import java.io.IOException; +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.nio.channels.FileChannel; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.Iterator; +import java.util.NavigableMap; + +final class StorageFileWriter { + + public static final ValueLayout.OfInt ENTRY_NUMBER_LAYOUT = ValueLayout.JAVA_INT_UNALIGNED; + public static final ValueLayout.OfLong MEMORY_SEGMENT_SIZE_LAYOUT = ValueLayout.JAVA_LONG_UNALIGNED; + + private StorageFileWriter() { + } + + static void writeIteratorIntoFile(long storageSize, + long indexSize, + Iterator> iterator, + Path sstablePath, + Path indexPath) throws IOException { + long storageWriteOffset = 0; + long indexWriteOffset = 0; + try (var storageChannel = FileChannel.open(sstablePath, + StandardOpenOption.READ, + StandardOpenOption.WRITE, + StandardOpenOption.CREATE); + + var indexChannel = FileChannel.open(indexPath, + StandardOpenOption.READ, + StandardOpenOption.WRITE, + StandardOpenOption.CREATE); + + var writeArena = Arena.ofConfined()) { + + MemorySegment mappedStorage = + storageChannel.map(FileChannel.MapMode.READ_WRITE, 0, storageSize, writeArena); + MemorySegment mappedIndex = + indexChannel.map(FileChannel.MapMode.READ_WRITE, 0, indexSize, writeArena); + + int entryNum = 0; + while (iterator.hasNext()) { + var entry = iterator.next(); + indexWriteOffset = + writeEntryNumAndStorageOffset(mappedIndex, indexWriteOffset, entryNum, storageWriteOffset); + entryNum++; + + storageWriteOffset = writeMemorySegment(entry.key(), mappedStorage, storageWriteOffset); + storageWriteOffset = writeMemorySegment(entry.value(), mappedStorage, storageWriteOffset); + } + mappedStorage.force(); + mappedIndex.force(); + } + } + + // writeMapIntoFile and writeIteratorInto file are pretty much the same, + // but I can't use writeIteratorIntoFile here due to optimization purposes: + // I have to write sstable and index separately + // I can't use writeMapIntoFile's code in the method above either, + // because it will slow down the execution due to the need of creating iterator twice + // And it also won't give any speed boost, + // because I would still be in need to find iterator.next() entry in another file + static void writeMapIntoFile(long sstableSize, + long indexSize, + NavigableMap> map, + Path sstablePath, + Path indexPath) throws IOException { + long storageWriteOffset = 0; + long indexWriteOffset = 0; + try (var storageChannel = FileChannel.open(sstablePath, + StandardOpenOption.READ, + StandardOpenOption.WRITE, + StandardOpenOption.CREATE); + + var indexChannel = FileChannel.open(indexPath, + StandardOpenOption.READ, + StandardOpenOption.WRITE, + StandardOpenOption.CREATE); + + var writeArena = Arena.ofConfined()) { + MemorySegment mappedIndex = + indexChannel.map(FileChannel.MapMode.READ_WRITE, 0, indexSize, writeArena); + + int entryNum = 0; + for (var entry : map.values()) { + indexWriteOffset = writeEntryNumAndStorageOffset( + mappedIndex, + indexWriteOffset, + entryNum, + storageWriteOffset + ); + entryNum++; + + storageWriteOffset += Storage.BYTES_TO_STORE_ENTRY_SIZE; + storageWriteOffset += entry.key().byteSize(); + if (entry.value() != null) { + storageWriteOffset += entry.value().byteSize(); + } + } + mappedIndex.force(); + + MemorySegment mappedStorage = + storageChannel.map(FileChannel.MapMode.READ_WRITE, 0, sstableSize, writeArena); + storageWriteOffset = 0; + for (var entry : map.values()) { + storageWriteOffset = writeMemorySegment(entry.key(), mappedStorage, storageWriteOffset); + storageWriteOffset = writeMemorySegment(entry.value(), mappedStorage, storageWriteOffset); + } + mappedStorage.force(); + } + } + + static long writeEntryNumAndStorageOffset(MemorySegment mappedIndex, + long indexWriteOffset, + int entryNum, + long storageWriteOffset) { + long offset = indexWriteOffset; + mappedIndex.set(ENTRY_NUMBER_LAYOUT, offset, entryNum); + offset += Storage.BYTES_TO_STORE_INDEX_KEY; + mappedIndex.set(MEMORY_SEGMENT_SIZE_LAYOUT, offset, storageWriteOffset); + offset += Storage.BYTES_TO_STORE_ENTRY_ELEMENT_SIZE; + return offset; + } + + // Every memorySegment in file has the following structure: + // 8 bytes - size, bytes - value + // If memorySegment has the size of -1 byte, then it means its value is DELETED + static long writeMemorySegment(MemorySegment memorySegment, MemorySegment mapped, long writeOffset) { + long offset = writeOffset; + if (memorySegment == null) { + mapped.set(MEMORY_SEGMENT_SIZE_LAYOUT, offset, -1); + offset += Storage.BYTES_TO_STORE_ENTRY_ELEMENT_SIZE; + } else { + long msSize = memorySegment.byteSize(); + mapped.set(MEMORY_SEGMENT_SIZE_LAYOUT, offset, msSize); + offset += Storage.BYTES_TO_STORE_ENTRY_ELEMENT_SIZE; + MemorySegment.copy(memorySegment, 0, mapped, offset, msSize); + offset += msSize; + } + return offset; + } + +} diff --git a/src/main/java/ru/vk/itmo/abramovilya/StorageWriter.java b/src/main/java/ru/vk/itmo/abramovilya/StorageWriter.java deleted file mode 100644 index 9f34bbdfa..000000000 --- a/src/main/java/ru/vk/itmo/abramovilya/StorageWriter.java +++ /dev/null @@ -1,96 +0,0 @@ -package ru.vk.itmo.abramovilya; - -import ru.vk.itmo.Entry; - -import java.io.IOException; -import java.lang.foreign.Arena; -import java.lang.foreign.MemorySegment; -import java.lang.foreign.ValueLayout; -import java.nio.channels.FileChannel; -import java.nio.file.Path; -import java.nio.file.StandardOpenOption; -import java.util.NavigableMap; - -final class StorageWriter { - private StorageWriter() { - } - - static long writeEntryNumAndStorageOffset(MemorySegment mappedIndex, - long indexWriteOffset, - int entryNum, - long storageWriteOffset) { - long offset = indexWriteOffset; - mappedIndex.set(ValueLayout.JAVA_INT_UNALIGNED, offset, entryNum); - offset += Integer.BYTES; - mappedIndex.set(ValueLayout.JAVA_LONG_UNALIGNED, offset, storageWriteOffset); - offset += Long.BYTES; - return offset; - } - - static long writeMemorySegment(MemorySegment memorySegment, MemorySegment mapped, long writeOffset) { - long offset = writeOffset; - if (memorySegment == null) { - mapped.set(ValueLayout.JAVA_LONG_UNALIGNED, offset, -1); - offset += Long.BYTES; - } else { - long msSize = memorySegment.byteSize(); - mapped.set(ValueLayout.JAVA_LONG_UNALIGNED, offset, msSize); - offset += Long.BYTES; - MemorySegment.copy(memorySegment, 0, mapped, offset, msSize); - offset += msSize; - } - return offset; - } - - // SSTable: |keySize: 8 bytes|key|valueSize: 8 bytes (size == -1 means value is deleted)|value| - // Index: |entryNum: 4 bytes|storageKeyOffset: 8 bytes| - static void writeSStableAndIndex(Path sstablePath, - long sstableSize, - Path indexPath, - long indexSize, - NavigableMap> map) throws IOException { - long storageWriteOffset = 0; - long indexWriteOffset = 0; - try (var storageChannel = FileChannel.open(sstablePath, - StandardOpenOption.READ, - StandardOpenOption.WRITE, - StandardOpenOption.CREATE); - - var indexChannel = FileChannel.open(indexPath, - StandardOpenOption.READ, - StandardOpenOption.WRITE, - StandardOpenOption.CREATE); - - var writeArena = Arena.ofConfined()) { - MemorySegment mappedIndex = - indexChannel.map(FileChannel.MapMode.READ_WRITE, 0, indexSize, writeArena); - - int entryNum = 0; - for (var entry : map.values()) { - indexWriteOffset = StorageWriter.writeEntryNumAndStorageOffset( - mappedIndex, - indexWriteOffset, - entryNum, - storageWriteOffset - ); - entryNum++; - - storageWriteOffset += 2 * Long.BYTES; - storageWriteOffset += entry.key().byteSize(); - if (entry.value() != null) { - storageWriteOffset += entry.value().byteSize(); - } - } - mappedIndex.force(); - - MemorySegment mappedStorage = - storageChannel.map(FileChannel.MapMode.READ_WRITE, 0, sstableSize, writeArena); - storageWriteOffset = 0; - for (var entry : map.values()) { - storageWriteOffset = StorageWriter.writeMemorySegment(entry.key(), mappedStorage, storageWriteOffset); - storageWriteOffset = StorageWriter.writeMemorySegment(entry.value(), mappedStorage, storageWriteOffset); - } - mappedStorage.force(); - } - } -} diff --git a/src/main/java/ru/vk/itmo/test/abramovilya/DaoFactoryImpl.java b/src/main/java/ru/vk/itmo/test/abramovilya/DaoFactoryImpl.java index 17e21aa3d..08025974b 100644 --- a/src/main/java/ru/vk/itmo/test/abramovilya/DaoFactoryImpl.java +++ b/src/main/java/ru/vk/itmo/test/abramovilya/DaoFactoryImpl.java @@ -10,7 +10,7 @@ import java.lang.foreign.ValueLayout; import java.nio.charset.StandardCharsets; -@DaoFactory(stage = 3) +@DaoFactory(stage = 4) public class DaoFactoryImpl implements DaoFactory.Factory> { @Override public ru.vk.itmo.Dao> createDao(Config config) throws IOException { From 1b28541ae94e650516ec6204428b15a6d8f30706 Mon Sep 17 00:00:00 2001 From: Vadim Tsesko Date: Mon, 4 Dec 2023 23:34:07 +0300 Subject: [PATCH 12/20] Switch to Gradle 8.5 --- gradle/wrapper/gradle-wrapper.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index ac72c34e8..1af9e0930 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.3-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.5-bin.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME From 8ce4ab8dba237f813451d85e2e4327c426cd632b Mon Sep 17 00:00:00 2001 From: Vadim Tsesko Date: Tue, 5 Dec 2023 01:21:15 +0300 Subject: [PATCH 13/20] Relaxed delays to be able to autoflush --- src/test/java/ru/vk/itmo/BasicTest.java | 2 +- src/test/java/ru/vk/itmo/PersistentTest.java | 12 +++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/test/java/ru/vk/itmo/BasicTest.java b/src/test/java/ru/vk/itmo/BasicTest.java index 6fdcdab82..0231f8374 100644 --- a/src/test/java/ru/vk/itmo/BasicTest.java +++ b/src/test/java/ru/vk/itmo/BasicTest.java @@ -146,7 +146,7 @@ void testHugeData(Dao> dao) throws Exception { // Back off after 1K upserts to be able to flush if (entry % 1000 == 0) { - Thread.sleep(1); + Thread.sleep(10); } } diff --git a/src/test/java/ru/vk/itmo/PersistentTest.java b/src/test/java/ru/vk/itmo/PersistentTest.java index 9b5f2b349..9746bfa51 100644 --- a/src/test/java/ru/vk/itmo/PersistentTest.java +++ b/src/test/java/ru/vk/itmo/PersistentTest.java @@ -2,7 +2,6 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Timeout; - import ru.vk.itmo.test.DaoFactory; import java.io.IOException; @@ -60,13 +59,20 @@ void cleanup(Dao> dao) throws IOException { } @DaoTest(stage = 2) - void persistentPreventInMemoryStorage(Dao> dao) throws IOException { + void persistentPreventInMemoryStorage(Dao> dao) throws Exception { int keys = 175_000; int entityIndex = keys / 2 - 7; // Fill List> entries = entries(keys); - entries.forEach(dao::upsert); + for (int entry = 0; entry < keys; entry++) { + dao.upsert(entries.get(entry)); + + // Back off after 1K upserts to be able to flush + if (entry % 1000 == 0) { + Thread.sleep(10); + } + } dao.close(); // Materialize to consume heap From 9847aefcdb438c2c9afecdc413ca952bbe093991 Mon Sep 17 00:00:00 2001 From: Vadim Tsesko Date: Tue, 5 Dec 2023 01:45:37 +0300 Subject: [PATCH 14/20] Relax even more --- src/test/java/ru/vk/itmo/BasicTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/ru/vk/itmo/BasicTest.java b/src/test/java/ru/vk/itmo/BasicTest.java index 0231f8374..a66d202a9 100644 --- a/src/test/java/ru/vk/itmo/BasicTest.java +++ b/src/test/java/ru/vk/itmo/BasicTest.java @@ -146,7 +146,7 @@ void testHugeData(Dao> dao) throws Exception { // Back off after 1K upserts to be able to flush if (entry % 1000 == 0) { - Thread.sleep(10); + Thread.sleep(100); } } From 51d62b8636a25d3f09c3f0805da39552784c3edb Mon Sep 17 00:00:00 2001 From: Vadim Tsesko Date: Tue, 5 Dec 2023 10:36:10 +0300 Subject: [PATCH 15/20] Revert "Relax even more" This reverts commit 9847aefcdb438c2c9afecdc413ca952bbe093991. --- src/test/java/ru/vk/itmo/BasicTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/ru/vk/itmo/BasicTest.java b/src/test/java/ru/vk/itmo/BasicTest.java index a66d202a9..0231f8374 100644 --- a/src/test/java/ru/vk/itmo/BasicTest.java +++ b/src/test/java/ru/vk/itmo/BasicTest.java @@ -146,7 +146,7 @@ void testHugeData(Dao> dao) throws Exception { // Back off after 1K upserts to be able to flush if (entry % 1000 == 0) { - Thread.sleep(100); + Thread.sleep(10); } } From 3915c7625893b6ba4fe9e955b29b4cf338bade9f Mon Sep 17 00:00:00 2001 From: Belonogov Nikolay <93780765+nickkkcc@users.noreply.github.com> Date: Tue, 5 Dec 2023 15:53:21 +0300 Subject: [PATCH 16/20] =?UTF-8?q?HW-4.=20=D0=91=D0=B5=D0=BB=D0=BE=D0=BD?= =?UTF-8?q?=D0=BE=D0=B3=D0=BE=D0=B2=20=D0=9D=D0=B8=D0=BA=D0=BE=D0=BB=D0=B0?= =?UTF-8?q?=D0=B9,=20=D0=A1=D0=9F=D0=91=D0=9F=D0=A3=20(#221)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Igor Lopatinskii <9440533@gmail.com> --- .../vk/itmo/belonogovnikolay/DiskStorage.java | 250 ++++++++++++++++++ .../belonogovnikolay/DiskStorageHelper.java | 92 +++++++ .../ru/vk/itmo/belonogovnikolay/FileType.java | 25 -- .../belonogovnikolay/InMemoryTreeDao.java | 123 +++++---- .../MemorySegmentComparator.java | 25 -- .../itmo/belonogovnikolay/MergeIterator.java | 142 ++++++++++ .../belonogovnikolay/PersistenceHelper.java | 214 --------------- .../InMemoryDaoFactoryImpl.java | 12 +- 8 files changed, 566 insertions(+), 317 deletions(-) create mode 100644 src/main/java/ru/vk/itmo/belonogovnikolay/DiskStorage.java create mode 100644 src/main/java/ru/vk/itmo/belonogovnikolay/DiskStorageHelper.java delete mode 100644 src/main/java/ru/vk/itmo/belonogovnikolay/FileType.java delete mode 100644 src/main/java/ru/vk/itmo/belonogovnikolay/MemorySegmentComparator.java create mode 100644 src/main/java/ru/vk/itmo/belonogovnikolay/MergeIterator.java delete mode 100644 src/main/java/ru/vk/itmo/belonogovnikolay/PersistenceHelper.java diff --git a/src/main/java/ru/vk/itmo/belonogovnikolay/DiskStorage.java b/src/main/java/ru/vk/itmo/belonogovnikolay/DiskStorage.java new file mode 100644 index 000000000..e03c8fcfc --- /dev/null +++ b/src/main/java/ru/vk/itmo/belonogovnikolay/DiskStorage.java @@ -0,0 +1,250 @@ +package ru.vk.itmo.belonogovnikolay; + +import ru.vk.itmo.BaseEntry; +import ru.vk.itmo.Entry; + +import java.io.IOException; +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.nio.channels.FileChannel; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.Iterator; +import java.util.List; +import java.util.NoSuchElementException; + +public class DiskStorage { + private final List segmentList; + private static final String INDEX_FILE_NAME = "index.idx"; + private static final String INDEX_TEMP_FILE_NAME = "index.tmp"; + + public DiskStorage(List segmentList) { + this.segmentList = segmentList; + } + + public Iterator> range( + Iterator> firstIterator, + MemorySegment from, + MemorySegment to) { + List>> iterators = new ArrayList<>(segmentList.size() + 1); + for (MemorySegment memorySegment : segmentList) { + iterators.add(iterator(memorySegment, from, to)); + } + iterators.add(firstIterator); + + return new MergeIterator<>(iterators, Comparator.comparing(Entry::key, InMemoryTreeDao::compare)) { + @Override + protected boolean skip(Entry memorySegmentEntry) { + return memorySegmentEntry.value() == null; + } + }; + } + + public static boolean save(Path storagePath, Iterable> iterable) + throws IOException { + if (!iterable.iterator().hasNext()) { + return false; + } + final Path indexTmp = storagePath.resolve(INDEX_TEMP_FILE_NAME); + final Path indexFile = storagePath.resolve(INDEX_FILE_NAME); + + try { + Files.createFile(indexFile); + } catch (FileAlreadyExistsException ignored) { + // it is ok, actually it is normal state + } + List existedFiles = Files.readAllLines(indexFile, StandardCharsets.UTF_8); + + String newFileName = String.valueOf(existedFiles.size()); + + long dataSize = 0; + long count = 0; + for (Entry entry : iterable) { + dataSize += entry.key().byteSize(); + MemorySegment value = entry.value(); + if (value != null) { + dataSize += value.byteSize(); + } + count++; + } + long indexSize = count * 2 * Long.BYTES; + + try ( + FileChannel fileChannel = FileChannel.open( + storagePath.resolve(newFileName), + StandardOpenOption.WRITE, + StandardOpenOption.READ, + StandardOpenOption.CREATE + ); + Arena writeArena = Arena.ofConfined() + ) { + MemorySegment fileSegment = fileChannel.map( + FileChannel.MapMode.READ_WRITE, + 0, + indexSize + dataSize, + writeArena + ); + + long dataOffset = indexSize; + int indexOffset = 0; + for (Entry entry : iterable) { + fileSegment.set(ValueLayout.JAVA_LONG_UNALIGNED, indexOffset, dataOffset); + dataOffset += entry.key().byteSize(); + indexOffset += Long.BYTES; + + MemorySegment value = entry.value(); + if (value == null) { + fileSegment.set(ValueLayout.JAVA_LONG_UNALIGNED, indexOffset, + DiskStorageHelper.tombstone(dataOffset)); + } else { + fileSegment.set(ValueLayout.JAVA_LONG_UNALIGNED, indexOffset, dataOffset); + dataOffset += value.byteSize(); + } + indexOffset += Long.BYTES; + } + + dataOffset = indexSize; + for (Entry entry : iterable) { + MemorySegment key = entry.key(); + MemorySegment.copy(key, 0, fileSegment, dataOffset, key.byteSize()); + dataOffset += key.byteSize(); + + MemorySegment value = entry.value(); + if (value != null) { + MemorySegment.copy(value, 0, fileSegment, dataOffset, value.byteSize()); + dataOffset += value.byteSize(); + } + } + } + + Files.move(indexFile, indexTmp, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + + List list = new ArrayList<>(existedFiles.size() + 1); + list.addAll(existedFiles); + list.add(newFileName); + Files.write( + indexFile, + list, + StandardOpenOption.WRITE, + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING + ); + + Files.delete(indexTmp); + return true; + } + + public static List loadOrRecover(Path storagePath, Arena arena) throws IOException { + Path indexTmp = storagePath.resolve(INDEX_TEMP_FILE_NAME); + Path indexFile = storagePath.resolve(INDEX_FILE_NAME); + + if (Files.exists(indexTmp)) { + Files.move(indexTmp, indexFile, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + } else { + try { + Files.createFile(indexFile); + } catch (FileAlreadyExistsException ignored) { + // it is ok, actually it is normal state + } + } + + List existedFiles = Files.readAllLines(indexFile, StandardCharsets.UTF_8); + List result = new ArrayList<>(existedFiles.size()); + for (String fileName : existedFiles) { + Path file = storagePath.resolve(fileName); + try (FileChannel fileChannel = FileChannel.open(file, StandardOpenOption.READ, StandardOpenOption.WRITE)) { + MemorySegment fileSegment = fileChannel.map( + FileChannel.MapMode.READ_WRITE, + 0, + Files.size(file), + arena + ); + result.add(fileSegment); + } + } + + return result; + } + + private static Iterator> iterator(MemorySegment page, MemorySegment from, MemorySegment to) { + long recordIndexFrom = from == null ? 0 : DiskStorageHelper.normalize(DiskStorageHelper.indexOf(page, from)); + long recordIndexTo = to == null ? DiskStorageHelper.recordsCount(page) + : DiskStorageHelper.normalize(DiskStorageHelper.indexOf(page, to)); + long recordsCount = DiskStorageHelper.recordsCount(page); + + return new Iterator<>() { + private long index = recordIndexFrom; + + @Override + public boolean hasNext() { + return index < recordIndexTo; + } + + @Override + public Entry next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + MemorySegment key = DiskStorageHelper.slice(page, DiskStorageHelper.startOfKey(page, index), + DiskStorageHelper.endOfKey(page, index)); + long startOfValue = DiskStorageHelper.startOfValue(page, index); + MemorySegment value = + startOfValue < 0 + ? null + : DiskStorageHelper.slice(page, startOfValue, + DiskStorageHelper.endOfValue(page, index, recordsCount)); + index++; + return new BaseEntry<>(key, value); + } + }; + } + + public static void compact(Path storagePath, Iterable> iterable) throws IOException { + if (!save(storagePath, iterable)) { + return; + } + + Path indexTmp = storagePath.resolve(INDEX_TEMP_FILE_NAME); + Path indexFile = storagePath.resolve(INDEX_FILE_NAME); + + if (Files.exists(indexTmp)) { + Files.move(indexTmp, indexFile, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + } else { + try { + Files.createFile(indexFile); + } catch (FileAlreadyExistsException ignored) { + // it is ok, actually it is normal state + } + } + + List existedFiles = Files.readAllLines(indexFile, StandardCharsets.UTF_8); + int filesCount = existedFiles.size(); + + if (filesCount >= 2) { + for (int i = 0; i < filesCount - 1; i++) { + Path file = storagePath.resolve(existedFiles.get(i)); + if (!Files.deleteIfExists(file)) { + break; + } + } + + Files.write( + indexFile, + Collections.singletonList("0"), + StandardOpenOption.WRITE, + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING + ); + Files.move(storagePath.resolve(existedFiles.get(filesCount - 1)), storagePath.resolve("0"), + StandardCopyOption.ATOMIC_MOVE); + } + } +} diff --git a/src/main/java/ru/vk/itmo/belonogovnikolay/DiskStorageHelper.java b/src/main/java/ru/vk/itmo/belonogovnikolay/DiskStorageHelper.java new file mode 100644 index 000000000..f3abf747c --- /dev/null +++ b/src/main/java/ru/vk/itmo/belonogovnikolay/DiskStorageHelper.java @@ -0,0 +1,92 @@ +package ru.vk.itmo.belonogovnikolay; + +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; + +public final class DiskStorageHelper { + + private DiskStorageHelper() { + + } + + static long endOfKey(MemorySegment segment, long recordIndex) { + return normalizedStartOfValue(segment, recordIndex); + } + + static long indexSize(MemorySegment segment) { + return segment.get(ValueLayout.JAVA_LONG_UNALIGNED, 0); + } + + static long recordsCount(MemorySegment segment) { + long indexSize = indexSize(segment); + return indexSize / Long.BYTES / 2; + } + + static long indexOf(MemorySegment segment, MemorySegment key) { + long recordsCount = recordsCount(segment); + + long left = 0; + long right = recordsCount - 1; + while (left <= right) { + long mid = (left + right) >>> 1; + + long startOfKey = DiskStorageHelper.startOfKey(segment, mid); + long endOfKey = DiskStorageHelper.endOfKey(segment, mid); + long mismatch = MemorySegment.mismatch(segment, startOfKey, endOfKey, key, 0, key.byteSize()); + if (mismatch == -1) { + return mid; + } + + if (mismatch == key.byteSize()) { + right = mid - 1; + continue; + } + + if (mismatch == endOfKey - startOfKey) { + left = mid + 1; + continue; + } + + int b1 = Byte.toUnsignedInt(segment.get(ValueLayout.JAVA_BYTE, startOfKey + mismatch)); + int b2 = Byte.toUnsignedInt(key.get(ValueLayout.JAVA_BYTE, mismatch)); + if (b1 > b2) { + right = mid - 1; + } else { + left = mid + 1; + } + } + + return DiskStorageHelper.tombstone(left); + } + + static long normalizedStartOfValue(MemorySegment segment, long recordIndex) { + return normalize(startOfValue(segment, recordIndex)); + } + + static long startOfValue(MemorySegment segment, long recordIndex) { + return segment.get(ValueLayout.JAVA_LONG_UNALIGNED, recordIndex * 2 * Long.BYTES + Long.BYTES); + } + + static long startOfKey(MemorySegment segment, long recordIndex) { + return segment.get(ValueLayout.JAVA_LONG_UNALIGNED, recordIndex * 2 * Long.BYTES); + } + + static long endOfValue(MemorySegment segment, long recordIndex, long recordsCount) { + if (recordIndex < recordsCount - 1) { + return startOfKey(segment, recordIndex + 1); + } + return segment.byteSize(); + } + + static long tombstone(long offset) { + return 1L << 63 | offset; + } + + static long normalize(long value) { + return value & ~(1L << 63); + } + + static MemorySegment slice(MemorySegment page, long start, long end) { + return page.asSlice(start, end - start); + } +} diff --git a/src/main/java/ru/vk/itmo/belonogovnikolay/FileType.java b/src/main/java/ru/vk/itmo/belonogovnikolay/FileType.java deleted file mode 100644 index 7709ee335..000000000 --- a/src/main/java/ru/vk/itmo/belonogovnikolay/FileType.java +++ /dev/null @@ -1,25 +0,0 @@ -package ru.vk.itmo.belonogovnikolay; - -/** - * The class represents an enumeration of two file types. `data-file` stores data that is written when reconnecting, - * disconnecting, etc. `offset-file` contains data about data offsets in `data-file`. - * @author Belonogov Nikolay - */ - -public enum FileType { - - DATA("data-file"), - - OFFSET("offset-file"); - - private final String fileName; - - FileType(String name) { - this.fileName = name; - } - - @Override - public String toString() { - return this.fileName; - } -} diff --git a/src/main/java/ru/vk/itmo/belonogovnikolay/InMemoryTreeDao.java b/src/main/java/ru/vk/itmo/belonogovnikolay/InMemoryTreeDao.java index 6985eb368..c842aa40f 100644 --- a/src/main/java/ru/vk/itmo/belonogovnikolay/InMemoryTreeDao.java +++ b/src/main/java/ru/vk/itmo/belonogovnikolay/InMemoryTreeDao.java @@ -5,84 +5,117 @@ import ru.vk.itmo.Entry; import java.io.IOException; +import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collections; +import java.util.Comparator; import java.util.Iterator; import java.util.NavigableMap; import java.util.concurrent.ConcurrentSkipListMap; -/** - * The class is an implementation of in memory persistent DAO. - * - * @author Belonogov Nikolay - */ -public final class InMemoryTreeDao implements Dao> { +public class InMemoryTreeDao implements Dao> { - private final NavigableMap> memTable; - private PersistenceHelper persistenceHelper; + private final Comparator comparator = InMemoryTreeDao::compare; + private NavigableMap> storage = new ConcurrentSkipListMap<>(comparator); + private final Arena arena; + private final DiskStorage diskStorage; + private final Path path; - private InMemoryTreeDao() { - this.memTable = new ConcurrentSkipListMap<>(new MemorySegmentComparator()); - } - - private InMemoryTreeDao(Config config) { - this(); - this.persistenceHelper = PersistenceHelper.newInstance(config.basePath()); - } + public InMemoryTreeDao(Config config) throws IOException { + this.path = config.basePath().resolve("data"); + Files.createDirectories(path); - public static Dao> newInstance() { - return new InMemoryTreeDao(); - } + arena = Arena.ofShared(); - public static Dao> newInstance(Config config) { - return new InMemoryTreeDao(config); + this.diskStorage = new DiskStorage(DiskStorage.loadOrRecover(path, arena)); } @Override - public Iterator> allFrom(MemorySegment from) { - return this.memTable.tailMap(from).values().iterator(); + public void compact() throws IOException { + DiskStorage.compact(path, () -> diskStorage.range(getInMemory(null, null), null, null)); + + if (this.arena.scope().isAlive()) { + arena.close(); + } + storage = new ConcurrentSkipListMap<>(comparator); } - @Override - public Iterator> allTo(MemorySegment to) { - return this.memTable.headMap(to).values().iterator(); + static int compare(MemorySegment memorySegment1, MemorySegment memorySegment2) { + long mismatch = memorySegment1.mismatch(memorySegment2); + if (mismatch == -1) { + return 0; + } + + if (mismatch == memorySegment1.byteSize()) { + return -1; + } + + if (mismatch == memorySegment2.byteSize()) { + return 1; + } + byte b1 = memorySegment1.get(ValueLayout.JAVA_BYTE, mismatch); + byte b2 = memorySegment2.get(ValueLayout.JAVA_BYTE, mismatch); + return Byte.compare(b1, b2); } @Override public Iterator> get(MemorySegment from, MemorySegment to) { + return diskStorage.range(getInMemory(from, to), from, to); + } + + private Iterator> getInMemory(MemorySegment from, MemorySegment to) { if (from == null && to == null) { - return this.memTable.values().iterator(); - } else if (from == null) { - return allTo(to); - } else if (to == null) { - return allFrom(from); + return storage.values().iterator(); } + if (from == null) { + return storage.headMap(to).values().iterator(); + } + if (to == null) { + return storage.tailMap(from).values().iterator(); + } + return storage.subMap(from, to).values().iterator(); + } - return this.memTable.subMap(from, to).values().iterator(); + @Override + public void upsert(Entry entry) { + storage.put(entry.key(), entry); } @Override public Entry get(MemorySegment key) { - Entry entry = this.memTable.get(key); - if (entry == null) { - try { - entry = persistenceHelper.readEntry(key); - } catch (IOException e) { + Entry entry = storage.get(key); + if (entry != null) { + if (entry.value() == null) { return null; } + return entry; + } + + Iterator> iterator = diskStorage.range(Collections.emptyIterator(), key, null); + + if (!iterator.hasNext()) { + return null; + } + Entry next = iterator.next(); + if (compare(next.key(), key) == 0) { + return next; } - return entry; + return null; } @Override - public void upsert(Entry entry) { - if (entry == null) { + public void close() throws IOException { + if (!arena.scope().isAlive()) { return; } - this.memTable.put(entry.key(), entry); - } - @Override - public void flush() throws IOException { - persistenceHelper.writeEntries(this.memTable); + arena.close(); + + if (!storage.isEmpty()) { + DiskStorage.save(path, storage.values()); + } } } diff --git a/src/main/java/ru/vk/itmo/belonogovnikolay/MemorySegmentComparator.java b/src/main/java/ru/vk/itmo/belonogovnikolay/MemorySegmentComparator.java deleted file mode 100644 index 7b6b2c672..000000000 --- a/src/main/java/ru/vk/itmo/belonogovnikolay/MemorySegmentComparator.java +++ /dev/null @@ -1,25 +0,0 @@ -package ru.vk.itmo.belonogovnikolay; - -import java.lang.foreign.MemorySegment; -import java.lang.foreign.ValueLayout; -import java.util.Comparator; - -public class MemorySegmentComparator implements Comparator { - - @Override - public int compare(MemorySegment prevSegment, MemorySegment nextSegment) { - - long offset = prevSegment.mismatch(nextSegment); - - if (offset == nextSegment.byteSize()) { - return 1; - } else if (offset == prevSegment.byteSize()) { - return -1; - } else if (offset == -1) { - return 0; - } - - return Byte.compare(prevSegment.get(ValueLayout.JAVA_BYTE, offset), - nextSegment.get(ValueLayout.JAVA_BYTE, offset)); - } -} diff --git a/src/main/java/ru/vk/itmo/belonogovnikolay/MergeIterator.java b/src/main/java/ru/vk/itmo/belonogovnikolay/MergeIterator.java new file mode 100644 index 000000000..749f8de4a --- /dev/null +++ b/src/main/java/ru/vk/itmo/belonogovnikolay/MergeIterator.java @@ -0,0 +1,142 @@ +package ru.vk.itmo.belonogovnikolay; + +import java.util.Collection; +import java.util.Comparator; +import java.util.Iterator; +import java.util.NoSuchElementException; +import java.util.PriorityQueue; + +public class MergeIterator implements Iterator { + + private final PriorityQueue> priorityQueue; + private final Comparator comparator; + private PeekIterator peekIterator; + + public MergeIterator(Collection> iterators, Comparator comparator) { + this.comparator = comparator; + Comparator> peekComp = (o1, o2) -> comparator.compare(o1.peek(), o2.peek()); + priorityQueue = new PriorityQueue<>( + iterators.size(), + peekComp.thenComparing(o -> -o.id) + ); + + int id = 0; + for (Iterator iterator : iterators) { + if (iterator.hasNext()) { + priorityQueue.add(new PeekIterator<>(id++, iterator)); + } + } + } + + private PeekIterator peek() { + while (peekIterator == null) { + peekIterator = priorityQueue.poll(); + if (peekIterator == null) { + return null; + } + + whileProcess(); + + if (peekIterator.peek() == null) { + peekIterator = null; + continue; + } + + if (skip(peekIterator.peek())) { + peekIterator.next(); + if (peekIterator.hasNext()) { + priorityQueue.add(peekIterator); + } + peekIterator = null; + } + } + + return peekIterator; + } + + private void whileProcess() { + while (true) { + PeekIterator next = priorityQueue.peek(); + if (next == null) { + break; + } + + int compare = comparator.compare(peekIterator.peek(), next.peek()); + if (compare == 0) { + PeekIterator poll = priorityQueue.poll(); + if (poll != null) { + poll.next(); + if (poll.hasNext()) { + priorityQueue.add(poll); + } + } + } else { + break; + } + } + } + + protected boolean skip(T t) { + return t == null; + } + + @Override + public boolean hasNext() { + return peek() != null; + } + + @Override + public T next() { + PeekIterator peek = peek(); + if (peek == null) { + throw new NoSuchElementException(); + } + T next = peek.next(); + this.peekIterator = null; + if (peek.hasNext()) { + priorityQueue.add(peek); + } + return next; + } + + private static class PeekIterator implements Iterator { + + public final int id; + private final Iterator delegate; + private T peekEntry; + + private PeekIterator(int id, Iterator delegate) { + this.id = id; + this.delegate = delegate; + } + + @Override + public boolean hasNext() { + if (peekEntry == null) { + return delegate.hasNext(); + } + return true; + } + + @Override + public T next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + T peek = peek(); + this.peekEntry = null; + return peek; + } + + private T peek() { + if (peekEntry == null) { + if (!delegate.hasNext()) { + return null; + } + peekEntry = delegate.next(); + } + return peekEntry; + } + + } +} diff --git a/src/main/java/ru/vk/itmo/belonogovnikolay/PersistenceHelper.java b/src/main/java/ru/vk/itmo/belonogovnikolay/PersistenceHelper.java deleted file mode 100644 index 42dd1d0f0..000000000 --- a/src/main/java/ru/vk/itmo/belonogovnikolay/PersistenceHelper.java +++ /dev/null @@ -1,214 +0,0 @@ -package ru.vk.itmo.belonogovnikolay; - -import ru.vk.itmo.BaseEntry; -import ru.vk.itmo.Entry; - -import java.io.IOException; -import java.lang.foreign.Arena; -import java.lang.foreign.MemorySegment; -import java.lang.foreign.ValueLayout; -import java.nio.channels.FileChannel; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardOpenOption; -import java.util.Map; -import java.util.NavigableMap; - -/** - * Util class for write, read, and persistence recovery operations when the {@link InMemoryTreeDao DAO} is restarted. - * - * @author Belonogov Nikolay - */ -public final class PersistenceHelper { - - private final Path basePath; - private final MemorySegmentComparator segmentComparator; - - private MemorySegment dataMappedSegment; - private MemorySegment offsetMappedSegment; - private long[] positionOffsets; - private int position; - private Path pathToDataFile; - private Path pathToOffsetFile; - private long offsetFileSize; - private boolean isReadingPrepared; - private Arena readingDataArena; - private Arena readingOffsetArena; - - private PersistenceHelper(Path basePath) { - this.basePath = basePath; - this.segmentComparator = new MemorySegmentComparator(); - resolvePaths(); - } - - /** - * Returns instance of PersistenceHelper class. - * - * @param basePath directory for storing snapshots. - * @return PersistentHelper instance. - * @throws NullPointerException is thrown when the path to the directory with snapshot files is not specified. - */ - public static PersistenceHelper newInstance(Path basePath) { - return new PersistenceHelper(basePath); - } - - /** - * The function writes to the file specified in the config. If the config is not specified, an exception is thrown. - * - * @param entries data to be written to disk. - * @throws IOException is thrown when exceptions occur while working with a file. - */ - public void writeEntries(NavigableMap> entries) throws IOException { - - int size = entries.size(); - - positionOffsets = new long[size * 2 + 1]; - - long fileSize = 0; - for (Map.Entry> entry : entries.entrySet()) { - fileSize += entry.getKey().byteSize() + entry.getValue().value().byteSize(); - } - - Files.deleteIfExists(pathToDataFile); - Files.deleteIfExists(pathToOffsetFile); - - Files.createFile(pathToDataFile); - Files.createFile(pathToOffsetFile); - - try (Arena dataArena = Arena.ofConfined()) { - try (Arena offsetArena = Arena.ofConfined()) { - - this.dataMappedSegment = mapFilesWriteRead(pathToDataFile, fileSize, dataArena); - this.offsetMappedSegment = mapFilesWriteRead(pathToOffsetFile, - (long) Long.BYTES * positionOffsets.length, offsetArena); - - entries.values().forEach(entry -> { - long keySize = entry.key().byteSize(); - long valueSize = entry.value().byteSize(); - positionOffsets[position + 1] = positionOffsets[position] + keySize; - positionOffsets[position + 2] = positionOffsets[position + 1] + valueSize; - this.dataMappedSegment.asSlice(positionOffsets[position], keySize).copyFrom(entry.key()); - this.dataMappedSegment.asSlice(positionOffsets[position + 1], valueSize).copyFrom(entry.value()); - position = position + 2; - }); - - offsetMappedSegment - .asSlice(0L, (long) Long.BYTES * positionOffsets.length) - .copyFrom(MemorySegment.ofArray(positionOffsets)); - } - } finally { - if (this.readingDataArena != null) { - this.readingDataArena.close(); - this.readingDataArena = null; - } - - if (this.readingOffsetArena != null) { - this.readingOffsetArena.close(); - this.readingOffsetArena = null; - } - } - } - - /** - * Returns entry of data which is read from file. - * - * @param key is search key of data entry which is read from file. - * @return entry of data. - * @throws IOException is thrown when exceptions occur while working with a file. - */ - public Entry readEntry(MemorySegment key) throws IOException { - - if (Files.notExists(pathToDataFile) || Files.notExists(pathToOffsetFile)) { - return null; - } - - if (!isReadingPrepared) { - readingPreparation(); - this.isReadingPrepared = true; - } - - long index = 0; - long beginLong; - long endLong; - long keyValueSize; - long offsetFileOffsetCount = (this.offsetFileSize - Long.BYTES) / 8 - 1; - - while (index < offsetFileOffsetCount) { - beginLong = offsetMappedSegment.getAtIndex(ValueLayout.JAVA_LONG, index); - endLong = offsetMappedSegment.getAtIndex(ValueLayout.JAVA_LONG, index + 1); - keyValueSize = endLong - beginLong; - MemorySegment keySegment = dataMappedSegment.asSlice(beginLong, keyValueSize); - if (this.segmentComparator.compare(keySegment, key) == 0) { - keyValueSize = offsetMappedSegment.getAtIndex(ValueLayout.JAVA_LONG, index + 2) - endLong; - return new BaseEntry<>(keySegment, dataMappedSegment.asSlice(endLong, keyValueSize)); - } - index++; - } - return null; - } - - /** - * Function of mapping MemorySegment and data file in READ-ONLY mode. - * - * @param filePath file path. - * @param byteSize file size (offset). - * @return {@link MemorySegment} which map with file. - * @throws IOException is thrown when exceptions occur while working with a file. - */ - private MemorySegment mapDataFileReadOnly(Path filePath, long byteSize) throws IOException { - try (FileChannel channel = FileChannel.open(filePath, StandardOpenOption.READ)) { - this.readingDataArena = Arena.ofConfined(); - return channel.map(FileChannel.MapMode.READ_ONLY, 0, byteSize, this.readingDataArena); - } - } - - /** - * Function of mapping MemorySegment and offset file in READ-ONLY mode. - * - * @param filePath file path. - * @param byteSize file size (offset). - * @return {@link MemorySegment} which map with file. - * @throws IOException is thrown when exceptions occur while working with a file. - */ - private MemorySegment mapOffsetFileReadOnly(Path filePath, long byteSize) throws IOException { - try (FileChannel channel = FileChannel.open(filePath, StandardOpenOption.READ)) { - this.readingOffsetArena = Arena.ofConfined(); - return channel.map(FileChannel.MapMode.READ_ONLY, 0, byteSize, this.readingOffsetArena); - } - } - - /** - * Function of mapping MemorySegment and file in READ-WRITE mode. - * - * @param filePath file path. - * @param byteSize file size (offset). - * @return {@link MemorySegment} which map with file. - * @throws IOException is thrown when exceptions occur while working with a file. - */ - private MemorySegment mapFilesWriteRead(Path filePath, long byteSize, Arena arena) throws IOException { - try (FileChannel channel = FileChannel.open(filePath, StandardOpenOption.READ, StandardOpenOption.WRITE)) { - return channel.map(FileChannel.MapMode.READ_WRITE, 0, byteSize, arena); - } - } - - /** - * Function resolves {@link #basePath} to file {@link #pathToDataFile}, {@link #pathToOffsetFile} paths. - */ - private void resolvePaths() { - this.pathToDataFile = this.basePath.resolve(FileType.DATA.toString()); - this.pathToOffsetFile = this.basePath.resolve(FileType.OFFSET.toString()); - } - - /** - * Until function to prepare for reading operations. - * - * @throws IOException is thrown when exceptions occur while working with a file. - */ - private void readingPreparation() throws IOException { - this.offsetFileSize = Files.size(pathToOffsetFile); - long dataFileSize = Files.size(pathToDataFile); - - this.dataMappedSegment = mapDataFileReadOnly(pathToDataFile, dataFileSize); - this.offsetMappedSegment = mapOffsetFileReadOnly(pathToOffsetFile, this.offsetFileSize); - } -} diff --git a/src/main/java/ru/vk/itmo/test/belonogovnikolay/InMemoryDaoFactoryImpl.java b/src/main/java/ru/vk/itmo/test/belonogovnikolay/InMemoryDaoFactoryImpl.java index 1daaef62d..7f892292e 100644 --- a/src/main/java/ru/vk/itmo/test/belonogovnikolay/InMemoryDaoFactoryImpl.java +++ b/src/main/java/ru/vk/itmo/test/belonogovnikolay/InMemoryDaoFactoryImpl.java @@ -6,12 +6,13 @@ import ru.vk.itmo.belonogovnikolay.InMemoryTreeDao; import ru.vk.itmo.test.DaoFactory; +import java.io.IOException; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; import java.nio.charset.StandardCharsets; import java.util.Objects; -@DaoFactory(stage = 2) +@DaoFactory(stage = 4) public class InMemoryDaoFactoryImpl implements DaoFactory.Factory> { /** * Creates new instance of Dao. @@ -19,13 +20,8 @@ public class InMemoryDaoFactoryImpl implements DaoFactory.Factory> createDao() { - return InMemoryTreeDao.newInstance(); - } - - @Override - public Dao> createDao(Config config) { - return InMemoryTreeDao.newInstance(config); + public Dao> createDao(Config config) throws IOException { + return new InMemoryTreeDao(config); } From 33891687d422079a924b849cb108e86e6507dda7 Mon Sep 17 00:00:00 2001 From: Dmitry Osokin <114069284+osokindm@users.noreply.github.com> Date: Tue, 5 Dec 2023 23:11:07 +0300 Subject: [PATCH 17/20] =?UTF-8?q?=D0=9E=D1=81=D0=BE=D0=BA=D0=B8=D0=BD=20?= =?UTF-8?q?=D0=94=D0=BC=D0=B8=D1=82=D1=80=D0=B8=D0=B9,=20HW4,=20=D0=9F?= =?UTF-8?q?=D0=BE=D0=BB=D0=B8=D1=82=D0=B5=D1=85=20(#220)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Vadim Tsesko Co-authored-by: Anton Lamtev --- .../ru/vk/itmo/osokindmitry/DiskStorage.java | 250 ++++++++++++++++++ .../ru/vk/itmo/osokindmitry/InMemoryDao.java | 166 ------------ .../vk/itmo/osokindmitry/MergeIterator.java | 141 ++++++++++ .../vk/itmo/osokindmitry/PersistentDao.java | 119 +++++++++ .../java/ru/vk/itmo/osokindmitry/SsTable.java | 146 ++++++++++ .../java/ru/vk/itmo/osokindmitry/Utils.java | 44 +++ .../{MyFactory.java => DmitFactory.java} | 11 +- 7 files changed, 706 insertions(+), 171 deletions(-) create mode 100644 src/main/java/ru/vk/itmo/osokindmitry/DiskStorage.java delete mode 100644 src/main/java/ru/vk/itmo/osokindmitry/InMemoryDao.java create mode 100644 src/main/java/ru/vk/itmo/osokindmitry/MergeIterator.java create mode 100644 src/main/java/ru/vk/itmo/osokindmitry/PersistentDao.java create mode 100644 src/main/java/ru/vk/itmo/osokindmitry/SsTable.java create mode 100644 src/main/java/ru/vk/itmo/osokindmitry/Utils.java rename src/main/java/ru/vk/itmo/test/osokindmitry/{MyFactory.java => DmitFactory.java} (77%) diff --git a/src/main/java/ru/vk/itmo/osokindmitry/DiskStorage.java b/src/main/java/ru/vk/itmo/osokindmitry/DiskStorage.java new file mode 100644 index 000000000..ec803501e --- /dev/null +++ b/src/main/java/ru/vk/itmo/osokindmitry/DiskStorage.java @@ -0,0 +1,250 @@ +package ru.vk.itmo.osokindmitry; + +import ru.vk.itmo.Entry; + +import java.io.IOException; +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.nio.channels.FileChannel; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.Iterator; +import java.util.List; + +public class DiskStorage { + + private final List tableList; + private static final String INDEX_FILE_NAME = "index"; + private static final String SSTABLE_EXT = ".sstable"; + private static final String TMP_EXT = ".tmp"; + + public DiskStorage(List ssTablePaths, Arena arena) throws IOException { + tableList = new ArrayList<>(); + for (Path path : ssTablePaths) { + tableList.add(new SsTable(path, arena)); + } + } + + public Iterator> range( + Iterator> firstIterator, + MemorySegment from, + MemorySegment to) { + + List>> iterators = new ArrayList<>(tableList.size() + 1); + for (SsTable ssTable : tableList) { + iterators.add(ssTable.iterator(from, to)); + } + iterators.add(firstIterator); + + return new MergeIterator<>(iterators, Comparator.comparing(Entry::key, PersistentDao::compare)) { + @Override + protected boolean skip(Entry memorySegmentEntry) { + return memorySegmentEntry.value() == null; + } + }; + } + + /** + * Stores memTable as follows: + * index: + * |key0_Start|value0_Start|key1_Start|value1_Start|key2_Start|value2_Start|... + * key0_Start = data start = end of index + * data: + * |key0|value0|key1|value1|... + */ + public static void save(Path storagePath, Iterable> iterable) + throws IOException { + final Path indexTmp = storagePath.resolve(INDEX_FILE_NAME + TMP_EXT); + final Path indexFile = storagePath.resolve(INDEX_FILE_NAME + SSTABLE_EXT); + + try { + Files.createFile(indexFile); + } catch (FileAlreadyExistsException ignored) { + // it is ok, actually it is normal state + } + List existedFiles = Files.readAllLines(indexFile, StandardCharsets.UTF_8); + + String newFileName = existedFiles.size() + SSTABLE_EXT; + + long dataSize = 0; + long count = 0; + + for (Entry entry : iterable) { + dataSize += entry.key().byteSize(); + + if (entry.value() != null) { + dataSize += entry.value().byteSize(); + } + count++; + } + long indexSize = count * 2 * Long.BYTES; + + try ( + FileChannel fileChannel = FileChannel.open( + storagePath.resolve(newFileName), + StandardOpenOption.WRITE, + StandardOpenOption.READ, + StandardOpenOption.CREATE + ); + Arena writeArena = Arena.ofConfined() + ) { + MemorySegment fileSegment = fileChannel.map( + FileChannel.MapMode.READ_WRITE, + 0, + indexSize + dataSize, + writeArena + ); + + long dataOffset = indexSize; + int indexOffset = 0; + for (Entry entry : iterable) { + fileSegment.set(ValueLayout.JAVA_LONG_UNALIGNED, indexOffset, dataOffset); + MemorySegment.copy(entry.key(), 0, fileSegment, dataOffset, entry.key().byteSize()); + dataOffset += entry.key().byteSize(); + indexOffset += Long.BYTES; + + if (entry.value() == null) { + fileSegment.set(ValueLayout.JAVA_LONG_UNALIGNED, indexOffset, SsTable.tombstone(dataOffset)); + } else { + fileSegment.set(ValueLayout.JAVA_LONG_UNALIGNED, indexOffset, dataOffset); + MemorySegment.copy(entry.value(), 0, fileSegment, dataOffset, entry.value().byteSize()); + dataOffset += entry.value().byteSize(); + } + indexOffset += Long.BYTES; + } + } + + updateIndex(indexFile, indexTmp, existedFiles, newFileName); + } + + private static void updateIndex(Path indexFile, Path indexTmp, List existedFiles, String newFileName) + throws IOException { + Files.move(indexFile, indexTmp, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + + List list = new ArrayList<>(existedFiles.size() + 1); + list.addAll(existedFiles); + list.add(newFileName); + Files.write( + indexFile, + list, + StandardOpenOption.WRITE, + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING + ); + Files.delete(indexTmp); + } + + public void compact(Path storagePath) throws IOException { + if (tableList.isEmpty()) { + return; + } + final Path indexFile = storagePath.resolve(INDEX_FILE_NAME + SSTABLE_EXT); + + try { + Files.createFile(indexFile); + } catch (FileAlreadyExistsException ignored) { + // it is ok, actually it is normal state + } + + MergeIterator> mergeIterator = getMergeIterator(); + long dataSize = 0; + long count = 0; + while (mergeIterator.hasNext()) { + count++; + Entry next = mergeIterator.next(); + dataSize += next.key().byteSize() + next.value().byteSize(); + } + dataSize += count * Long.BYTES * 2; + + MemorySegment fileSegment; + try ( + FileChannel fileChannel = FileChannel.open( + storagePath.resolve("0" + TMP_EXT), + StandardOpenOption.WRITE, + StandardOpenOption.READ, + StandardOpenOption.CREATE + ); + Arena writeArena = Arena.ofConfined() + ) { + fileSegment = fileChannel.map( + FileChannel.MapMode.READ_WRITE, + 0, + dataSize, + writeArena + ); + + mergeIterator = getMergeIterator(); + long dataOffset = count * 2 * Long.BYTES; + int indexOffset = 0; + while (mergeIterator.hasNext()) { + Entry entry = mergeIterator.next(); + + fileSegment.set(ValueLayout.JAVA_LONG_UNALIGNED, indexOffset, dataOffset); + MemorySegment.copy(entry.key(), 0, fileSegment, dataOffset, entry.key().byteSize()); + dataOffset += entry.key().byteSize(); + indexOffset += Long.BYTES; + + fileSegment.set(ValueLayout.JAVA_LONG_UNALIGNED, indexOffset, dataOffset); + MemorySegment.copy(entry.value(), 0, fileSegment, dataOffset, entry.value().byteSize()); + dataOffset += entry.value().byteSize(); + indexOffset += Long.BYTES; + } + } + + updateIndexAndCleanUp(storagePath, indexFile); + tableList.clear(); + tableList.add(new SsTable(fileSegment)); + } + + private void updateIndexAndCleanUp(Path storagePath, Path indexFile) throws IOException { + + final List existedFiles = Files.readAllLines(indexFile, StandardCharsets.UTF_8); + final Path indexTmp = storagePath.resolve(INDEX_FILE_NAME + TMP_EXT); + + Files.move(indexFile, indexTmp, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + Files.move(storagePath.resolve("0" + TMP_EXT), storagePath.resolve("0" + SSTABLE_EXT), + StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + + Files.writeString( + indexFile, + "0" + SSTABLE_EXT, + StandardOpenOption.WRITE, + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING + ); + + try { + for (int i = 1; i < existedFiles.size(); i++) { + Files.delete(storagePath.resolve(existedFiles.get(i))); + } + } catch (IOException e) { + // If we fail during delete, db will recover with index.tmp that points to deleted files + Files.delete(indexTmp); + throw e; + } + + Files.delete(indexTmp); + } + + private MergeIterator> getMergeIterator() { + List>> iterators = new ArrayList<>(tableList.size() + 1); + for (SsTable ssTable : tableList) { + iterators.add(ssTable.iterator(null, null)); + } + + return new MergeIterator<>(iterators, Comparator.comparing(Entry::key, PersistentDao::compare)) { + @Override + protected boolean skip(Entry memorySegmentEntry) { + return memorySegmentEntry.value() == null; + } + }; + } + +} diff --git a/src/main/java/ru/vk/itmo/osokindmitry/InMemoryDao.java b/src/main/java/ru/vk/itmo/osokindmitry/InMemoryDao.java deleted file mode 100644 index 948195a75..000000000 --- a/src/main/java/ru/vk/itmo/osokindmitry/InMemoryDao.java +++ /dev/null @@ -1,166 +0,0 @@ -package ru.vk.itmo.osokindmitry; - -import ru.vk.itmo.BaseEntry; -import ru.vk.itmo.Config; -import ru.vk.itmo.Dao; -import ru.vk.itmo.Entry; - -import java.io.IOException; -import java.lang.foreign.Arena; -import java.lang.foreign.MemorySegment; -import java.lang.foreign.ValueLayout; -import java.nio.channels.FileChannel; -import java.nio.file.Path; -import java.nio.file.StandardOpenOption; -import java.util.Collections; -import java.util.Iterator; -import java.util.Set; -import java.util.concurrent.ConcurrentNavigableMap; -import java.util.concurrent.ConcurrentSkipListMap; - -public class InMemoryDao implements Dao> { - - private final ConcurrentNavigableMap> storage; - private final ConcurrentNavigableMap> cachedValues; - - private final Arena arena; - private final Path path; - private final MemorySegment mappedFile; - - private static final String FILE_NAME = "sstable.txt"; - - public InMemoryDao(Config config) { - path = config.basePath().resolve(FILE_NAME); - arena = Arena.ofConfined(); - storage = new ConcurrentSkipListMap<>(InMemoryDao::compare); - cachedValues = new ConcurrentSkipListMap<>(InMemoryDao::compare); - mappedFile = mapFile(path); - } - - @Override - public Entry get(MemorySegment key) { - Entry entry = storage.get(key); - // avoiding extra file operations by checking cached values - if (entry == null) { - entry = cachedValues.get(key); - } - // if value is still null then searching in file - if (entry == null && mappedFile != null) { - entry = searchInSlice(mappedFile, key); - } - - return entry; - } - - @Override - public Iterator> get(MemorySegment from, MemorySegment to) { - if (storage.isEmpty()) { - return Collections.emptyIterator(); - } - boolean empty = to == null; - MemorySegment first = from == null ? storage.firstKey() : from; - MemorySegment last = to == null ? storage.lastKey() : to; - return storage.subMap(first, true, last, empty).values().iterator(); - } - - @Override - public void upsert(Entry entry) { - storage.put(entry.key(), entry); - } - - @Override - public void flush() throws IOException { - try ( - FileChannel fc = FileChannel.open( - path, - Set.of(StandardOpenOption.CREATE, StandardOpenOption.READ, StandardOpenOption.WRITE)) - ) { - - long ssTableSize = Long.BYTES * 2L * storage.size(); - for (Entry value : storage.values()) { - ssTableSize += value.key().byteSize() + value.value().byteSize(); - } - - MemorySegment ssTable = fc.map(FileChannel.MapMode.READ_WRITE, 0, ssTableSize, arena); - long offset = 0; - - for (Entry value : storage.values()) { - offset = writeEntry(value.key(), ssTable, offset); - offset = writeEntry(value.value(), ssTable, offset); - } - } - } - - @Override - public void close() throws IOException { - flush(); - if (arena.scope().isAlive()) { - arena.close(); - } - } - - private long writeEntry(MemorySegment entry, MemorySegment ssTable, long offset) { - long curOffset = offset; - ssTable.set(ValueLayout.JAVA_LONG_UNALIGNED, curOffset, entry.byteSize()); - curOffset += Long.BYTES; - MemorySegment.copy(entry, 0, ssTable, curOffset, entry.byteSize()); - curOffset += entry.byteSize(); - return curOffset; - } - - private MemorySegment mapFile(Path path) { - if (path.toFile().exists()) { - try ( - FileChannel fc = FileChannel.open( - path, - Set.of(StandardOpenOption.CREATE, StandardOpenOption.READ)) - ) { - if (fc.size() != 0) { - return fc.map(FileChannel.MapMode.READ_ONLY, 0, fc.size(), arena); - } - } catch (IOException e) { - return null; - } - } - return null; - } - - private Entry searchInSlice(MemorySegment mappedSegment, MemorySegment key) { - long offset = 0; - while (offset < mappedSegment.byteSize() - Long.BYTES) { - - long size = mappedSegment.get(ValueLayout.JAVA_LONG_UNALIGNED, offset); - offset += Long.BYTES; - MemorySegment slicedKey = mappedSegment.asSlice(offset, size); - offset += size; - - long mismatch = key.mismatch(slicedKey); - size = mappedSegment.get(ValueLayout.JAVA_LONG_UNALIGNED, offset); - offset += Long.BYTES; - - if (mismatch == -1) { - MemorySegment slicedValue = mappedSegment.asSlice(offset, size); - BaseEntry entry = new BaseEntry<>(slicedKey, slicedValue); - cachedValues.put(slicedKey, entry); - return entry; - } - offset += size; - } - return null; - } - - private static int compare(MemorySegment segment1, MemorySegment segment2) { - long offset = segment1.mismatch(segment2); - if (offset == -1) { - return 0; - } else if (offset == segment1.byteSize()) { - return -1; - } else if (offset == segment2.byteSize()) { - return 1; - } - byte b1 = segment1.get(ValueLayout.JAVA_BYTE, offset); - byte b2 = segment2.get(ValueLayout.JAVA_BYTE, offset); - return Byte.compare(b1, b2); - } - -} diff --git a/src/main/java/ru/vk/itmo/osokindmitry/MergeIterator.java b/src/main/java/ru/vk/itmo/osokindmitry/MergeIterator.java new file mode 100644 index 000000000..b80fb06cf --- /dev/null +++ b/src/main/java/ru/vk/itmo/osokindmitry/MergeIterator.java @@ -0,0 +1,141 @@ +package ru.vk.itmo.osokindmitry; + +import java.util.Collection; +import java.util.Comparator; +import java.util.Iterator; +import java.util.NoSuchElementException; +import java.util.PriorityQueue; + +public class MergeIterator implements Iterator { + + private final PriorityQueue> priorityQueue; + private final Comparator comparator; + PeekIterator peekIterator; + + private static class PeekIterator implements Iterator { + + public final int id; + private final Iterator delegate; + private T peek; + + private PeekIterator(int id, Iterator delegate) { + this.id = id; + this.delegate = delegate; + } + + @Override + public boolean hasNext() { + if (peek == null) { + return delegate.hasNext(); + } + return true; + } + + @Override + public T next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + T entry = peek(); + peek = null; + return entry; + } + + private T peek() { + if (peek == null) { + if (!delegate.hasNext()) { + return null; + } + peek = delegate.next(); + } + return peek; + } + } + + public MergeIterator(Collection> iterators, Comparator comparator) { + this.comparator = comparator; + Comparator> peekComp = (o1, o2) -> comparator.compare(o1.peek(), o2.peek()); + priorityQueue = new PriorityQueue<>( + iterators.size(), + peekComp.thenComparing(o -> -o.id) + ); + + int id = 0; + for (Iterator iterator : iterators) { + if (iterator.hasNext()) { + priorityQueue.add(new PeekIterator<>(id++, iterator)); + } + } + } + + private PeekIterator peek() { + while (peekIterator == null) { + peekIterator = priorityQueue.poll(); + if (peekIterator == null) { + return null; + } + moveIterator(); + + if (peekIterator.peek() == null) { + peekIterator = null; + continue; + } + + if (skip(peekIterator.peek())) { + peekIterator.next(); + if (peekIterator.hasNext()) { + priorityQueue.add(peekIterator); + } + peekIterator = null; + } + } + + return peekIterator; + } + + private void moveIterator() { + while (true) { + PeekIterator next = priorityQueue.peek(); + if (next == null) { + break; + } + + int compare = comparator.compare(peekIterator.peek(), next.peek()); + if (compare == 0) { + PeekIterator poll = priorityQueue.poll(); + if (poll != null) { + poll.next(); + if (poll.hasNext()) { + priorityQueue.add(poll); + } + } + } else { + break; + } + } + } + + protected boolean skip(T t) { + return t == null; + } + + @Override + public boolean hasNext() { + return peek() != null; + } + + @Override + public T next() { + PeekIterator peek = peek(); + if (peek == null) { + throw new NoSuchElementException(); + } + T next = peek.next(); + this.peekIterator = null; + if (peek.hasNext()) { + priorityQueue.add(peek); + } + return next; + } + +} diff --git a/src/main/java/ru/vk/itmo/osokindmitry/PersistentDao.java b/src/main/java/ru/vk/itmo/osokindmitry/PersistentDao.java new file mode 100644 index 000000000..5660af819 --- /dev/null +++ b/src/main/java/ru/vk/itmo/osokindmitry/PersistentDao.java @@ -0,0 +1,119 @@ +package ru.vk.itmo.osokindmitry; + +import ru.vk.itmo.Config; +import ru.vk.itmo.Dao; +import ru.vk.itmo.Entry; + +import java.io.IOException; +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collections; +import java.util.Iterator; +import java.util.concurrent.ConcurrentNavigableMap; +import java.util.concurrent.ConcurrentSkipListMap; + +public class PersistentDao implements Dao> { + + private final Arena arena; + private final Path path; + private final DiskStorage diskStorage; + private ConcurrentNavigableMap> storage; + + public PersistentDao(Config config) throws IOException { + path = config.basePath().resolve("data"); + Files.createDirectories(path); + + arena = Arena.ofShared(); + storage = new ConcurrentSkipListMap<>(PersistentDao::compare); + + this.diskStorage = new DiskStorage(Utils.loadOrRecover(path), arena); + } + + @Override + public Entry get(MemorySegment key) { + Entry entry = storage.get(key); + if (entry != null) { + if (entry.value() == null) { + return null; + } + return entry; + } + + Iterator> iterator = diskStorage.range(Collections.emptyIterator(), key, null); + + if (!iterator.hasNext()) { + return null; + } + Entry next = iterator.next(); + if (compare(next.key(), key) == 0) { + return next; + } + return null; + } + + static int compare(MemorySegment segment1, MemorySegment segment2) { + long offset = segment1.mismatch(segment2); + if (offset == -1) { + return 0; + } else if (offset == segment1.byteSize()) { + return -1; + } else if (offset == segment2.byteSize()) { + return 1; + } + byte b1 = segment1.get(ValueLayout.JAVA_BYTE, offset); + byte b2 = segment2.get(ValueLayout.JAVA_BYTE, offset); + return Byte.compare(b1, b2); + } + + private Iterator> getInMemory(MemorySegment from, MemorySegment to) { + if (from == null && to == null) { + return storage.values().iterator(); + } + if (from == null) { + return storage.headMap(to).values().iterator(); + } + if (to == null) { + return storage.tailMap(from).values().iterator(); + } + return storage.subMap(from, to).values().iterator(); + } + + @Override + public Iterator> get(MemorySegment from, MemorySegment to) { + return diskStorage.range(getInMemory(from, to), from, to); + } + + @Override + public void upsert(Entry entry) { + storage.put(entry.key(), entry); + } + + @Override + public void flush() throws IOException { + if (!storage.isEmpty()) { + DiskStorage.save(path, storage.values()); + } + } + + @Override + public void compact() throws IOException { + if (!storage.isEmpty()) { + flush(); + storage = new ConcurrentSkipListMap<>(PersistentDao::compare); + } + diskStorage.compact(path); + } + + @Override + public void close() throws IOException { + if (!arena.scope().isAlive()) { + return; + } + arena.close(); + flush(); + } + +} diff --git a/src/main/java/ru/vk/itmo/osokindmitry/SsTable.java b/src/main/java/ru/vk/itmo/osokindmitry/SsTable.java new file mode 100644 index 000000000..13e263128 --- /dev/null +++ b/src/main/java/ru/vk/itmo/osokindmitry/SsTable.java @@ -0,0 +1,146 @@ +package ru.vk.itmo.osokindmitry; + +import ru.vk.itmo.BaseEntry; +import ru.vk.itmo.Entry; + +import java.io.IOException; +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.nio.channels.FileChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.Iterator; +import java.util.NoSuchElementException; + +public class SsTable { + private final MemorySegment segment; + + public SsTable(Path fullPath, Arena arena) throws IOException { + try (FileChannel fileChannel = FileChannel.open(fullPath, StandardOpenOption.READ, StandardOpenOption.WRITE)) { + segment = fileChannel.map( + FileChannel.MapMode.READ_WRITE, + 0, + Files.size(fullPath), + arena + ); + } + } + + public SsTable(MemorySegment segment) { + this.segment = segment; + } + + private static long indexOf(MemorySegment segment, MemorySegment key) { + long recordsCount = recordsCount(segment); + + long left = 0; + long right = recordsCount - 1; + while (left <= right) { + long mid = (left + right) >>> 1; + + long startOfKey = startOfKey(segment, mid); + long endOfKey = endOfKey(segment, mid); + long mismatch = MemorySegment.mismatch(segment, startOfKey, endOfKey, key, 0, key.byteSize()); + if (mismatch == -1) { + return mid; + } + + if (mismatch == key.byteSize()) { + right = mid - 1; + continue; + } + + if (mismatch == endOfKey - startOfKey) { + left = mid + 1; + continue; + } + + int b1 = Byte.toUnsignedInt(segment.get(ValueLayout.JAVA_BYTE, startOfKey + mismatch)); + int b2 = Byte.toUnsignedInt(key.get(ValueLayout.JAVA_BYTE, mismatch)); + if (b1 > b2) { + right = mid - 1; + } else { + left = mid + 1; + } + } + + return tombstone(left); + } + + private static long recordsCount(MemorySegment segment) { + long indexSize = indexSize(segment); + return indexSize / Long.BYTES / 2; + } + + private static long indexSize(MemorySegment segment) { + return segment.get(ValueLayout.JAVA_LONG_UNALIGNED, 0); + } + + public static long tombstone(long offset) { + return 1L << 63 | offset; + } + + private static long normalize(long value) { + return value & ~(1L << 63); + } + + private static MemorySegment slice(MemorySegment page, long start, long end) { + return page.asSlice(start, end - start); + } + + private static long startOfKey(MemorySegment segment, long recordIndex) { + return segment.get(ValueLayout.JAVA_LONG_UNALIGNED, recordIndex * 2 * Long.BYTES); + } + + private static long endOfKey(MemorySegment segment, long recordIndex) { + return normalizedStartOfValue(segment, recordIndex); + } + + private static long normalizedStartOfValue(MemorySegment segment, long recordIndex) { + return normalize(startOfValue(segment, recordIndex)); + } + + private static long startOfValue(MemorySegment segment, long recordIndex) { + return segment.get(ValueLayout.JAVA_LONG_UNALIGNED, recordIndex * 2 * Long.BYTES + Long.BYTES); + } + + private static long endOfValue(MemorySegment segment, long recordIndex, long recordsCount) { + if (recordIndex < recordsCount - 1) { + return startOfKey(segment, recordIndex + 1); + } + return segment.byteSize(); + } + + public Iterator> iterator(MemorySegment from, MemorySegment to) { + long recordIndexFrom = from == null ? 0 : normalize(indexOf(segment, from)); + long recordIndexTo = to == null ? recordsCount(segment) : normalize(indexOf(segment, to)); + long recordsCount = recordsCount(segment); + + return new Iterator<>() { + long index = recordIndexFrom; + + @Override + public boolean hasNext() { + return index < recordIndexTo; + } + + @Override + public Entry next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + MemorySegment key = slice(segment, startOfKey(segment, index), endOfKey(segment, index)); + long startOfValue = startOfValue(segment, index); + MemorySegment value = + startOfValue < 0 + ? null + : slice(segment, startOfValue, endOfValue(segment, index, recordsCount)); + index++; + return new BaseEntry<>(key, value); + } + }; + } + +} diff --git a/src/main/java/ru/vk/itmo/osokindmitry/Utils.java b/src/main/java/ru/vk/itmo/osokindmitry/Utils.java new file mode 100644 index 000000000..901d781d9 --- /dev/null +++ b/src/main/java/ru/vk/itmo/osokindmitry/Utils.java @@ -0,0 +1,44 @@ +package ru.vk.itmo.osokindmitry; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.List; + +public final class Utils { + private static final String INDEX_FILE_NAME = "index"; + private static final String SSTABLE_EXT = ".sstable"; + private static final String TMP_EXT = ".tmp"; + + private Utils() { + throw new UnsupportedOperationException("This is a utility class and cannot be instantiated"); + } + + public static List loadOrRecover(Path storagePath) throws IOException { + final Path indexTmp = storagePath.resolve(INDEX_FILE_NAME + TMP_EXT); + final Path indexFile = storagePath.resolve(INDEX_FILE_NAME + SSTABLE_EXT); + + if (Files.exists(indexTmp)) { + Files.move(indexTmp, indexFile, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + } else { + try { + Files.createFile(indexFile); + } catch (FileAlreadyExistsException ignored) { + // it is ok, actually it is normal state + } + } + + List existedFiles = Files.readAllLines(indexFile, StandardCharsets.UTF_8); + List result = new ArrayList<>(existedFiles.size()); + for (String fileName : existedFiles) { + Path file = storagePath.resolve(fileName); + result.add(file); + } + return result; + } + +} diff --git a/src/main/java/ru/vk/itmo/test/osokindmitry/MyFactory.java b/src/main/java/ru/vk/itmo/test/osokindmitry/DmitFactory.java similarity index 77% rename from src/main/java/ru/vk/itmo/test/osokindmitry/MyFactory.java rename to src/main/java/ru/vk/itmo/test/osokindmitry/DmitFactory.java index 764c8c95c..65943cbc4 100644 --- a/src/main/java/ru/vk/itmo/test/osokindmitry/MyFactory.java +++ b/src/main/java/ru/vk/itmo/test/osokindmitry/DmitFactory.java @@ -3,19 +3,20 @@ import ru.vk.itmo.Config; import ru.vk.itmo.Dao; import ru.vk.itmo.Entry; -import ru.vk.itmo.osokindmitry.InMemoryDao; +import ru.vk.itmo.osokindmitry.PersistentDao; import ru.vk.itmo.test.DaoFactory; +import java.io.IOException; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; import java.nio.charset.StandardCharsets; -@DaoFactory(stage = 2) -public class MyFactory implements DaoFactory.Factory> { +@DaoFactory(stage = 4) +public class DmitFactory implements DaoFactory.Factory> { @Override - public Dao> createDao(Config config) { - return new InMemoryDao(config); + public Dao> createDao(Config config) throws IOException { + return new PersistentDao(config); } @Override From cc5c32f0d495dee00f67888c98bedd921ce3b799 Mon Sep 17 00:00:00 2001 From: Kirill06344 <67016214+Kirill06344@users.noreply.github.com> Date: Tue, 5 Dec 2023 23:21:21 +0300 Subject: [PATCH 18/20] =?UTF-8?q?HW4=20-=20=D0=9F=D0=BE=D0=BB=D0=B8=D1=82?= =?UTF-8?q?=D0=B5=D1=85,=20=D0=91=D0=B0=D0=B6=D0=B5=D0=BD=D0=BE=D0=B2=20?= =?UTF-8?q?=D0=9A=D0=B8=D1=80=D0=B8=D0=BB=D0=BB=20(#230)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Vadim Tsesko Co-authored-by: Anton Lamtev --- .../bazhenovkirill/MemorySegmentUtils.java | 72 +++++ .../vk/itmo/bazhenovkirill/MergeIterator.java | 123 ++++++++ .../ru/vk/itmo/bazhenovkirill/Offset.java | 11 + .../bazhenovkirill/PersistentDaoImpl.java | 147 ++-------- .../ru/vk/itmo/bazhenovkirill/Storage.java | 267 ++++++++++++++++++ .../test/bazhenovkirill/DaoFactoryImpl.java | 2 +- 6 files changed, 505 insertions(+), 117 deletions(-) create mode 100644 src/main/java/ru/vk/itmo/bazhenovkirill/MemorySegmentUtils.java create mode 100644 src/main/java/ru/vk/itmo/bazhenovkirill/MergeIterator.java create mode 100644 src/main/java/ru/vk/itmo/bazhenovkirill/Offset.java create mode 100644 src/main/java/ru/vk/itmo/bazhenovkirill/Storage.java diff --git a/src/main/java/ru/vk/itmo/bazhenovkirill/MemorySegmentUtils.java b/src/main/java/ru/vk/itmo/bazhenovkirill/MemorySegmentUtils.java new file mode 100644 index 000000000..487da9450 --- /dev/null +++ b/src/main/java/ru/vk/itmo/bazhenovkirill/MemorySegmentUtils.java @@ -0,0 +1,72 @@ +package ru.vk.itmo.bazhenovkirill; + +import ru.vk.itmo.BaseEntry; +import ru.vk.itmo.Entry; + +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; + +public final class MemorySegmentUtils { + + private MemorySegmentUtils() { + + } + + public static MemorySegment getSlice(MemorySegment segment, long start, long end) { + return segment.asSlice(start, end - start); + } + + public static long startOfKey(MemorySegment segment, long inx) { + return segment.get(ValueLayout.JAVA_LONG_UNALIGNED, inx * 2 * Long.BYTES); + } + + public static long endOfKey(MemorySegment segment, long inx) { + return normalize(startOfValue(segment, inx)); + } + + public static MemorySegment getKey(MemorySegment segment, long inx) { + return getSlice(segment, startOfKey(segment, inx), endOfKey(segment, inx)); + } + + public static long startOfValue(MemorySegment segment, long inx) { + return segment.get(ValueLayout.JAVA_LONG_UNALIGNED, Long.BYTES + inx * 2 * Long.BYTES); + } + + public static long endOfValue(MemorySegment segment, long inx) { + if (inx < recordsCount(segment) - 1) { + return startOfKey(segment, inx + 1); + } + return segment.byteSize(); + } + + public static Entry getEntry(MemorySegment segment, long inx) { + MemorySegment key = getKey(segment, inx); + MemorySegment value = getValue(segment, inx); + return new BaseEntry<>(key, value); + } + + public static MemorySegment getValue(MemorySegment segment, long inx) { + long start = startOfValue(segment, inx); + if (start < 0) { + return null; + } + return getSlice(segment, start, endOfValue(segment, inx)); + } + + public static long tombstone(long offset) { + return 1L << 63 | offset; + } + + public static long normalize(long offset) { + return offset & ~(1L << 63); + } + + public static long recordsCount(MemorySegment segment) { + return indexSize(segment) / (2 * Long.BYTES); + } + + public static long indexSize(MemorySegment segment) { + return segment.get(ValueLayout.JAVA_LONG_UNALIGNED, 0); + } + +} diff --git a/src/main/java/ru/vk/itmo/bazhenovkirill/MergeIterator.java b/src/main/java/ru/vk/itmo/bazhenovkirill/MergeIterator.java new file mode 100644 index 000000000..4d047a82c --- /dev/null +++ b/src/main/java/ru/vk/itmo/bazhenovkirill/MergeIterator.java @@ -0,0 +1,123 @@ +package ru.vk.itmo.bazhenovkirill; + +import java.util.Collection; +import java.util.Comparator; +import java.util.Iterator; +import java.util.NoSuchElementException; +import java.util.PriorityQueue; + +public class MergeIterator implements Iterator { + + private final PriorityQueue> priorityQueue; + + private final Comparator comparator; + + private PeekIterator peek; + + private static class PeekIterator implements Iterator { + + private final Iterator iterator; + private T next; + private final int id; + + public PeekIterator(int id, Iterator iterator) { + this.id = id; + this.iterator = iterator; + if (iterator.hasNext()) { + next = iterator.next(); + } + } + + private T peek() { + return next; + } + + @Override + public boolean hasNext() { + return next != null || iterator.hasNext(); + } + + @Override + public T next() { + T curr = next; + next = iterator.hasNext() ? iterator.next() : null; + return curr; + } + } + + public MergeIterator(Collection> iterators, Comparator comparator) { + this.comparator = comparator; + Comparator> peekComp = (o1, o2) -> comparator.compare(o1.peek(), o2.peek()); + priorityQueue = new PriorityQueue<>( + iterators.size(), + peekComp.thenComparing(o -> -o.id) + ); + + int id = 0; + for (Iterator iterator : iterators) { + if (iterator.hasNext()) { + priorityQueue.add(new PeekIterator<>(id++, iterator)); + } + } + } + + protected boolean skip(T t) { + return t == null; + } + + private PeekIterator peek() { + while (peek == null) { + peek = priorityQueue.poll(); + if (peek == null) { + return null; + } + + PeekIterator next = priorityQueue.peek(); + while (next != null && comparator.compare(peek.peek(), next.peek()) == 0) { + PeekIterator poll = priorityQueue.poll(); + if (poll != null) { + skipElement(poll); + } + next = priorityQueue.peek(); + } + + if (!peek.hasNext()) { + peek = null; + continue; + } + + if (skip(peek.peek())) { + skipElement(peek); + peek = null; + } + } + + return peek; + } + + private void skipElement(PeekIterator iterator) { + iterator.next(); + if (iterator.hasNext()) { + priorityQueue.add(iterator); + } + } + + @Override + public boolean hasNext() { + return peek() != null; + } + + @Override + public T next() { + PeekIterator peekIterator = peek(); + if (peekIterator == null) { + throw new NoSuchElementException(); + } + T next = peek.next(); + this.peek = null; + if (peekIterator.hasNext()) { + priorityQueue.add(peekIterator); + } + return next; + } +} diff --git a/src/main/java/ru/vk/itmo/bazhenovkirill/Offset.java b/src/main/java/ru/vk/itmo/bazhenovkirill/Offset.java new file mode 100644 index 000000000..7f65af2b6 --- /dev/null +++ b/src/main/java/ru/vk/itmo/bazhenovkirill/Offset.java @@ -0,0 +1,11 @@ +package ru.vk.itmo.bazhenovkirill; + +public class Offset { + long data; + long index; + + Offset(long data, long index) { + this.data = data; + this.index = index; + } +} diff --git a/src/main/java/ru/vk/itmo/bazhenovkirill/PersistentDaoImpl.java b/src/main/java/ru/vk/itmo/bazhenovkirill/PersistentDaoImpl.java index f0e4b770e..a7e24a1e1 100644 --- a/src/main/java/ru/vk/itmo/bazhenovkirill/PersistentDaoImpl.java +++ b/src/main/java/ru/vk/itmo/bazhenovkirill/PersistentDaoImpl.java @@ -1,94 +1,48 @@ package ru.vk.itmo.bazhenovkirill; -import ru.vk.itmo.BaseEntry; import ru.vk.itmo.Config; import ru.vk.itmo.Dao; import ru.vk.itmo.Entry; -import java.io.FileNotFoundException; import java.io.IOException; import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; -import java.lang.foreign.ValueLayout; -import java.nio.channels.FileChannel; -import java.nio.channels.FileChannel.MapMode; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.StandardOpenOption; import java.util.Iterator; -import java.util.Set; import java.util.concurrent.ConcurrentNavigableMap; import java.util.concurrent.ConcurrentSkipListMap; public class PersistentDaoImpl implements Dao> { - - private static final String DATA_FILE = "sstable.db"; - - private static final Set WRITE_OPTIONS = Set.of( - StandardOpenOption.CREATE, - StandardOpenOption.READ, - StandardOpenOption.TRUNCATE_EXISTING, - StandardOpenOption.WRITE - ); - private final ConcurrentNavigableMap> memTable = new ConcurrentSkipListMap<>(new MemorySegmentComparator()); - - private final Path dataPath; - private final Arena arena; - private MemorySegment mappedMS; + private final Path dataPath; + private final Storage storage; public PersistentDaoImpl(Config config) throws IOException { - dataPath = config.basePath().resolve(DATA_FILE); - arena = Arena.ofShared(); - + dataPath = config.basePath().resolve("data"); if (!Files.exists(dataPath)) { - mappedMS = MemorySegment.NULL; - if (!arena.scope().isAlive()) { - arena.close(); - } - return; - } - - boolean segmentMapped = false; - try (FileChannel channel = FileChannel.open(dataPath, StandardOpenOption.READ)) { - mappedMS = channel.map(MapMode.READ_ONLY, - 0, channel.size(), arena).asReadOnly(); - segmentMapped = true; - } catch (FileNotFoundException e) { - mappedMS = MemorySegment.NULL; - } finally { - if (!segmentMapped) { - arena.close(); - } + Files.createDirectories(dataPath); } + arena = Arena.ofShared(); + storage = new Storage(Storage.loadData(dataPath, arena)); } @Override public Iterator> get(MemorySegment from, MemorySegment to) { - if (from == null) { - if (to != null) { - return memTable.headMap(to).values().iterator(); - } - return memTable.values().iterator(); - } else { - if (to == null) { - return memTable.tailMap(from).values().iterator(); - } - return memTable.subMap(from, true, to, false).values().iterator(); - } + return storage.range(getInMemTable(from, to), from, to); } @Override public Entry get(MemorySegment key) { - Entry value = memTable.get(key); - if (value == null) { - return getDataFromSSTable(key); + Entry entry = memTable.get(key); + if (entry == null) { + return storage.get(key); } - return value; + return entry.value() == null ? null : entry; } @Override @@ -98,77 +52,38 @@ public void upsert(Entry entry) { @Override public void flush() throws IOException { - try (FileChannel channel = FileChannel.open(dataPath, WRITE_OPTIONS)) { - try (Arena confinedArena = Arena.ofConfined()) { - MemorySegment dataMemorySegment = channel.map(MapMode.READ_WRITE, 0, - getMemTableSizeInBytes(), confinedArena); - long offset = 0; - for (var entry : memTable.values()) { - offset = writeEntry(entry, dataMemorySegment, offset); - } - } + if (!memTable.isEmpty()) { + Storage.save(dataPath, memTable.values()); + } + } + + @Override + public void compact() throws IOException { + if (storage.compact(dataPath, this::all)) { + memTable.clear(); } } @Override public void close() throws IOException { + flush(); if (!arena.scope().isAlive()) { return; } arena.close(); - - flush(); } - private Entry getDataFromSSTable(MemorySegment key) { - long offset = 0; - long valueSize = 0; - while (offset < mappedMS.byteSize()) { - long keySize = mappedMS.get(ValueLayout.JAVA_LONG_UNALIGNED, offset); - offset += Long.BYTES; - - if (keySize == key.byteSize()) { - MemorySegment possibleKey = mappedMS.asSlice(offset, keySize); - - valueSize = mappedMS.get(ValueLayout.JAVA_LONG_UNALIGNED, offset + keySize); - offset += (keySize + Long.BYTES); - if (key.mismatch(possibleKey) == -1) { - MemorySegment value = mappedMS.asSlice(offset, valueSize); - return new BaseEntry<>(possibleKey, value); - } - } else { - offset += keySize; - valueSize = mappedMS.get(ValueLayout.JAVA_LONG_UNALIGNED, offset); - offset += Long.BYTES; + private Iterator> getInMemTable(MemorySegment from, MemorySegment to) { + if (from == null) { + if (to != null) { + return memTable.headMap(to).values().iterator(); } - - offset += valueSize; - } - return null; - } - - private long getMemTableSizeInBytes() { - long size = memTable.size() * Long.BYTES * 2L; - for (var entry : memTable.values()) { - size += entry.key().byteSize() + entry.value().byteSize(); + return memTable.values().iterator(); + } else { + if (to == null) { + return memTable.tailMap(from).values().iterator(); + } + return memTable.subMap(from, true, to, false).values().iterator(); } - return size; - } - - private long writeEntry(Entry entry, MemorySegment destination, long offset) { - long updatedOffset = writeDataToMemorySegment(entry.key(), destination, offset); - return writeDataToMemorySegment(entry.value(), destination, updatedOffset); - } - - private long writeDataToMemorySegment(MemorySegment entryPart, MemorySegment destination, long offset) { - long currentOffset = offset; - - destination.set(ValueLayout.JAVA_LONG_UNALIGNED, currentOffset, entryPart.byteSize()); - currentOffset += Long.BYTES; - - MemorySegment.copy(entryPart, 0, destination, currentOffset, entryPart.byteSize()); - currentOffset += entryPart.byteSize(); - - return currentOffset; } } diff --git a/src/main/java/ru/vk/itmo/bazhenovkirill/Storage.java b/src/main/java/ru/vk/itmo/bazhenovkirill/Storage.java new file mode 100644 index 000000000..636ab732e --- /dev/null +++ b/src/main/java/ru/vk/itmo/bazhenovkirill/Storage.java @@ -0,0 +1,267 @@ +package ru.vk.itmo.bazhenovkirill; + +import ru.vk.itmo.Entry; + +import java.io.IOException; +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.nio.channels.FileChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.Iterator; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; + +public class Storage { + + private static final AtomicInteger SSTABLE_ID = new AtomicInteger(); + private static final MemorySegmentComparator comparator = new MemorySegmentComparator(); + private static final String INDEX_FILE_NAME = "index.db"; + private static final Set WRITE_OPTIONS = Set.of( + StandardOpenOption.CREATE, + StandardOpenOption.WRITE, + StandardOpenOption.READ + ); + private final List segments; + + public Storage(List segments) { + this.segments = segments; + } + + /* + dataFile: + index -> |key0_offset|value0_offset|...|keyn_offset|valuen_offset| + data -> |ke0|value0|...|keyn|valuen| + index, data in one file + */ + public static void save(Path dataPath, Iterable> values) throws IOException { + Path indexFile = createOrMapIndexFile(dataPath); + List existedFiles = Files.readAllLines(indexFile); + + String fileName = String.valueOf(SSTABLE_ID.incrementAndGet()); + writeDataToSSTable(dataPath.resolve(fileName), values); + + existedFiles.add(fileName); + Files.write(indexFile, + existedFiles, + StandardOpenOption.WRITE, + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING + ); + } + + public boolean compact(Path dataPath, Iterable> values) throws IOException { + Path indexFile = createOrMapIndexFile(dataPath); + List existedFiles = Files.readAllLines(indexFile); + if (existedFiles.isEmpty()) { + return false; + } + + Path compactionTmpFile = getCompactionPath(dataPath); + + compactData(compactionTmpFile, values); + finalizeCompaction(dataPath, existedFiles); + return true; + } + + private void finalizeCompaction(Path dataPath, List existedFiles) throws IOException { + Path compactionTmpFile = getCompactionPath(dataPath); + if (Files.size(compactionTmpFile) == 0) { + return; + } + + for (String name : existedFiles) { + Files.deleteIfExists(dataPath.resolve(name)); + } + + Path indexFile = createOrMapIndexFile(dataPath); + String newSSTableName = String.valueOf(SSTABLE_ID.incrementAndGet()); + Files.write( + indexFile, + Collections.singleton(newSSTableName), + StandardOpenOption.WRITE, + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING + ); + + Files.move(compactionTmpFile, dataPath.resolve(newSSTableName), StandardCopyOption.ATOMIC_MOVE); + } + + private Path getCompactionPath(Path dataPath) { + return dataPath.resolve("compaction.tmp"); + } + + private void compactData(Path ssTablePath, Iterable> values) throws IOException { + long dataSize = 0; + long entriesCount = 0; + + for (Entry entry : values) { + dataSize += entry.key().byteSize(); + MemorySegment value = entry.value(); + dataSize += (value == null) ? 0 : value.byteSize(); + entriesCount++; + } + long indexSize = 2 * Long.BYTES * entriesCount; + + try (FileChannel channel = FileChannel.open(ssTablePath, WRITE_OPTIONS); + Arena arena = Arena.ofConfined()) { + MemorySegment segment = channel.map(FileChannel.MapMode.READ_WRITE, + 0, + dataSize + indexSize, + arena); + + Offset offset = new Offset(indexSize, 0); + for (Entry entry : values) { + writeEntry(entry, segment, offset); + + } + } + } + + private static void writeDataToSSTable(Path ssTablePath, Iterable> values) throws IOException { + long dataSize = 0; + long entriesCount = 0; + for (Entry entry : values) { + dataSize += entry.key().byteSize(); + MemorySegment value = entry.value(); + dataSize += (value == null) ? 0 : value.byteSize(); + entriesCount++; + } + long indexSize = 2 * Long.BYTES * entriesCount; + + try (FileChannel channel = FileChannel.open(ssTablePath, WRITE_OPTIONS); + Arena arena = Arena.ofConfined()) { + MemorySegment segment = channel.map(FileChannel.MapMode.READ_WRITE, + 0, + dataSize + indexSize, + arena); + + Offset offset = new Offset(indexSize, 0); + for (Entry entry : values) { + writeEntry(entry, segment, offset); + } + } + } + + private static void writeEntry(Entry entry, MemorySegment segment, Offset offset) { + MemorySegment key = entry.key(); + segment.set(ValueLayout.JAVA_LONG_UNALIGNED, offset.index, offset.data); + MemorySegment.copy(key, 0, segment, offset.data, key.byteSize()); + offset.data += key.byteSize(); + offset.index += Long.BYTES; + + MemorySegment value = entry.value(); + if (value == null) { + segment.set(ValueLayout.JAVA_LONG_UNALIGNED, offset.index, MemorySegmentUtils.tombstone(offset.data)); + } else { + segment.set(ValueLayout.JAVA_LONG_UNALIGNED, offset.index, offset.data); + MemorySegment.copy(value, 0, segment, offset.data, value.byteSize()); + offset.data += value.byteSize(); + } + offset.index += Long.BYTES; + } + + public static List loadData(Path dataPath, Arena arena) throws IOException { + Path indexFile = createOrMapIndexFile(dataPath); + + List existedFiles = Files.readAllLines(indexFile); + List segments = new ArrayList<>(existedFiles.size()); + for (String name : existedFiles) { + Path dataFile = dataPath.resolve(name); + try (FileChannel channel = FileChannel.open(dataFile, StandardOpenOption.READ, StandardOpenOption.WRITE)) { + MemorySegment segment = channel.map(FileChannel.MapMode.READ_WRITE, 0, channel.size(), arena); + segments.add(segment); + } + } + + return segments; + } + + public Entry get(MemorySegment key) { + for (int i = segments.size() - 1; i >= 0; --i) { + long index = indexOf(segments.get(i), key); + if (index >= 0) { + Entry entry = MemorySegmentUtils.getEntry(segments.get(i), index); + return entry.value() == null ? null : entry; + } + } + return null; + } + + public Iterator> range(Iterator> inMemoryIterator, + MemorySegment from, + MemorySegment to) { + List>> iterators = new ArrayList<>(segments.size() + 1); + for (MemorySegment segment : segments) { + iterators.add(iterator(segment, from, to)); + } + iterators.add(inMemoryIterator); + + return new MergeIterator<>(iterators, Comparator.comparing(Entry::key, comparator)) { + @Override + protected boolean skip(Entry memorySegmentEntry) { + return memorySegmentEntry.value() == null; + } + }; + } + + private static Iterator> iterator(MemorySegment segment, + MemorySegment from, + MemorySegment to) { + long size = MemorySegmentUtils.recordsCount(segment); + long start = (from == null) ? 0 : MemorySegmentUtils.normalize(indexOf(segment, from)); + long end = (to == null) ? size : MemorySegmentUtils.normalize(indexOf(segment, to)); + + return new Iterator<>() { + long inx = start; + + @Override + public boolean hasNext() { + return inx < end; + } + + @Override + public Entry next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + return MemorySegmentUtils.getEntry(segment, inx++); + } + }; + } + + private static long indexOf(MemorySegment segment, MemorySegment key) { + long l = 0; + long r = MemorySegmentUtils.recordsCount(segment) - 1; + while (l <= r) { + long mid = (l + r) >>> 1; + int resultOfComparing = comparator.compare(key, MemorySegmentUtils.getKey(segment, mid)); + if (resultOfComparing == 0) { + return mid; + } else if (resultOfComparing < 0) { + r = mid - 1; + } else { + l = mid + 1; + } + } + return MemorySegmentUtils.tombstone(l); + } + + private static Path createOrMapIndexFile(Path dataPath) throws IOException { + Path indexFile = dataPath.resolve(INDEX_FILE_NAME); + + if (!Files.exists(indexFile)) { + Files.createFile(indexFile); + } + + return indexFile; + } +} diff --git a/src/main/java/ru/vk/itmo/test/bazhenovkirill/DaoFactoryImpl.java b/src/main/java/ru/vk/itmo/test/bazhenovkirill/DaoFactoryImpl.java index 150fb53b7..943d6082b 100644 --- a/src/main/java/ru/vk/itmo/test/bazhenovkirill/DaoFactoryImpl.java +++ b/src/main/java/ru/vk/itmo/test/bazhenovkirill/DaoFactoryImpl.java @@ -11,7 +11,7 @@ import java.lang.foreign.ValueLayout; import java.nio.charset.StandardCharsets; -@DaoFactory(stage = 2) +@DaoFactory(stage = 4) public class DaoFactoryImpl implements DaoFactory.Factory> { @Override From f163bd3944c1f86c95d9f522aa90a32bc1766dcc Mon Sep 17 00:00:00 2001 From: slava_xfce <42910710+GenryEden@users.noreply.github.com> Date: Tue, 5 Dec 2023 23:26:13 +0300 Subject: [PATCH 19/20] =?UTF-8?q?Part4,=20=D0=95=D0=BC=D0=B5=D0=BB=D1=8C?= =?UTF-8?q?=D1=8F=D0=BD=D0=BE=D0=B2=20=D0=92=D0=B8=D1=82=D0=B0=D0=BB=D0=B8?= =?UTF-8?q?=D0=B9,=20=D0=9F=D0=BE=D0=BB=D0=B8=D1=82=D0=B5=D1=85=20(#237)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: vitaliy.emelyanov Co-authored-by: Vadim Tsesko Co-authored-by: vitaliy.emelyanov Co-authored-by: Marashov-Alexander <55152974+Marashov-Alexander@users.noreply.github.com> Co-authored-by: Anton Lamtev --- .../test/emelyanovvitaliy/InMemoryDao.java | 98 +++++++++++++------ .../emelyanovvitaliy/InMemoryDaoFactory.java | 2 +- 2 files changed, 70 insertions(+), 30 deletions(-) diff --git a/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/InMemoryDao.java b/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/InMemoryDao.java index a943c5a9a..7f39643db 100644 --- a/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/InMemoryDao.java +++ b/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/InMemoryDao.java @@ -109,14 +109,48 @@ public void upsert(Entry entry) { public void flush() throws IOException { long currentTimeMillis = System.currentTimeMillis(); long nanoTime = System.nanoTime(); - Path filePath = sstablesPath.resolve( + Path filePath = getFilePath(currentTimeMillis, nanoTime); + dumpToFile(filePath, currentTimeMillis, nanoTime); + } + + private Path getFilePath(long curTime, long nanoTime) { + return sstablesPath.resolve( Path.of( - Long.toString(currentTimeMillis, Character.MAX_RADIX) - + Long.toString(nanoTime, Character.MAX_RADIX) - + SSTABLE_SUFFIX + Long.toString(curTime, Character.MAX_RADIX) + + Long.toString(nanoTime, Character.MAX_RADIX) + + SSTABLE_SUFFIX ) ); - dumpToFile(filePath, currentTimeMillis, nanoTime); + } + + @Override + public void compact() throws IOException { + long fileSize = 2 * Long.BYTES + Integer.BYTES; + int countOfKeys = 0; + Iterator> it = all(); + while (it.hasNext()) { + Entry entry = it.next(); + fileSize += entry.key().byteSize() + entry.value().byteSize(); + countOfKeys++; + } + long curTimeMillis = System.currentTimeMillis(); + long nanoTime = System.nanoTime(); + + fileSize += 2L * countOfKeys * Long.BYTES; + Path filePath = getFilePath(curTimeMillis, nanoTime); + Set openOptions = + Set.of( + StandardOpenOption.CREATE, + StandardOpenOption.READ, + StandardOpenOption.WRITE + ); + try (FileChannel fc = FileChannel.open(filePath, openOptions); Arena writeArena = Arena.ofConfined()) { + MemorySegment mapped = fc.map(READ_WRITE, 0, fileSize, writeArena); + dumpToMemSegment(mapped, all(), countOfKeys, curTimeMillis, nanoTime); + } + for (Path file: filesSet) { + Files.delete(file); + } filesSet.add(filePath); } @@ -168,36 +202,42 @@ private void dumpToFile(Path path, long currentTimeMillis, long nanoTime) throws size += Integer.BYTES + (2L * mappings.size() + 2) * Long.BYTES; try (FileChannel fc = FileChannel.open(path, OPEN_OPTIONS); Arena writeArena = Arena.ofConfined()) { MemorySegment mapped = fc.map(READ_WRITE, 0, size, writeArena); - long offset = 0; - mapped.set(ValueLayout.JAVA_LONG_UNALIGNED, offset, currentTimeMillis); - offset += Long.BYTES; - mapped.set(ValueLayout.JAVA_LONG_UNALIGNED, offset, nanoTime); + dumpToMemSegment(mapped, getFromMemory(null, null), mappings.size(), currentTimeMillis, nanoTime); + } + } + + private void dumpToMemSegment(MemorySegment mapped, Iterator> iterator, + int numOfKeys, long curTime, long nanoTime) { + long offset = 0; + mapped.set(ValueLayout.JAVA_LONG_UNALIGNED, offset, curTime); + offset += Long.BYTES; + mapped.set(ValueLayout.JAVA_LONG_UNALIGNED, offset, nanoTime); + offset += Long.BYTES; + mapped.set(ValueLayout.JAVA_INT_UNALIGNED, offset, numOfKeys); + offset += Integer.BYTES; + long offsetToWrite = Integer.BYTES + (2L * numOfKeys + 2) * Long.BYTES; + while (iterator.hasNext()) { + Entry entry = iterator.next(); + mapped.set(ValueLayout.JAVA_LONG_UNALIGNED, offset, offsetToWrite); offset += Long.BYTES; - mapped.set(ValueLayout.JAVA_INT_UNALIGNED, offset, mappings.size()); - offset += Integer.BYTES; - long offsetToWrite = Integer.BYTES + (2L * mappings.size() + 2) * Long.BYTES; - for (Entry entry: mappings.values()) { + MemorySegment.copy( + entry.key(), 0, + mapped, offsetToWrite, + entry.key().byteSize() + ); + offsetToWrite += entry.key().byteSize(); + if (entry.value() == null) { + mapped.set(ValueLayout.JAVA_LONG_UNALIGNED, offset, -1); + } else { mapped.set(ValueLayout.JAVA_LONG_UNALIGNED, offset, offsetToWrite); - offset += Long.BYTES; MemorySegment.copy( - entry.key(), 0, + entry.value(), 0, mapped, offsetToWrite, - entry.key().byteSize() + entry.value().byteSize() ); - offsetToWrite += entry.key().byteSize(); - if (entry.value() == null) { - mapped.set(ValueLayout.JAVA_LONG_UNALIGNED, offset, -1); - } else { - mapped.set(ValueLayout.JAVA_LONG_UNALIGNED, offset, offsetToWrite); - MemorySegment.copy( - entry.value(), 0, - mapped, offsetToWrite, - entry.value().byteSize() - ); - offsetToWrite += entry.value().byteSize(); - } - offset += Long.BYTES; + offsetToWrite += entry.value().byteSize(); } + offset += Long.BYTES; } } diff --git a/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/InMemoryDaoFactory.java b/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/InMemoryDaoFactory.java index 8dd186d93..8066f180d 100644 --- a/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/InMemoryDaoFactory.java +++ b/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/InMemoryDaoFactory.java @@ -10,7 +10,7 @@ import java.lang.foreign.ValueLayout; import java.nio.charset.StandardCharsets; -@DaoFactory(stage = 3) +@DaoFactory(stage = 4) public class InMemoryDaoFactory implements DaoFactory.Factory> { @Override public String toString(MemorySegment memorySegment) { From f402b3f2db9960263c78fcc0dfe5330e77a0a0fa Mon Sep 17 00:00:00 2001 From: Vadim Tsesko Date: Thu, 7 Dec 2023 18:32:52 +0300 Subject: [PATCH 20/20] Reference implementation (#299) --- .../vk/itmo/reference/ByteArraySegment.java | 48 +++ .../itmo/reference/LiveFilteringIterator.java | 52 ++++ .../java/ru/vk/itmo/reference/MemTable.java | 49 +++ .../reference/MemorySegmentComparator.java | 89 ++++++ .../itmo/reference/MergingEntryIterator.java | 72 +++++ .../ru/vk/itmo/reference/ReferenceDao.java | 292 ++++++++++++++++++ .../java/ru/vk/itmo/reference/SSTable.java | 204 ++++++++++++ .../ru/vk/itmo/reference/SSTableWriter.java | 166 ++++++++++ .../java/ru/vk/itmo/reference/SSTables.java | 162 ++++++++++ .../java/ru/vk/itmo/reference/TableSet.java | 205 ++++++++++++ .../WeightedPeekingEntryIterator.java | 67 ++++ .../test/reference/ReferenceDaoFactory.java | 49 +++ src/test/java/ru/vk/itmo/BaseTest.java | 34 +- .../java/ru/vk/itmo/BasicConcurrentTest.java | 6 +- src/test/java/ru/vk/itmo/BasicTest.java | 8 +- .../ru/vk/itmo/PersistentConcurrentTest.java | 44 +-- src/test/java/ru/vk/itmo/PersistentTest.java | 8 +- 17 files changed, 1508 insertions(+), 47 deletions(-) create mode 100644 src/main/java/ru/vk/itmo/reference/ByteArraySegment.java create mode 100644 src/main/java/ru/vk/itmo/reference/LiveFilteringIterator.java create mode 100644 src/main/java/ru/vk/itmo/reference/MemTable.java create mode 100644 src/main/java/ru/vk/itmo/reference/MemorySegmentComparator.java create mode 100644 src/main/java/ru/vk/itmo/reference/MergingEntryIterator.java create mode 100644 src/main/java/ru/vk/itmo/reference/ReferenceDao.java create mode 100644 src/main/java/ru/vk/itmo/reference/SSTable.java create mode 100644 src/main/java/ru/vk/itmo/reference/SSTableWriter.java create mode 100644 src/main/java/ru/vk/itmo/reference/SSTables.java create mode 100644 src/main/java/ru/vk/itmo/reference/TableSet.java create mode 100644 src/main/java/ru/vk/itmo/reference/WeightedPeekingEntryIterator.java create mode 100644 src/main/java/ru/vk/itmo/test/reference/ReferenceDaoFactory.java diff --git a/src/main/java/ru/vk/itmo/reference/ByteArraySegment.java b/src/main/java/ru/vk/itmo/reference/ByteArraySegment.java new file mode 100644 index 000000000..34f98f61e --- /dev/null +++ b/src/main/java/ru/vk/itmo/reference/ByteArraySegment.java @@ -0,0 +1,48 @@ +package ru.vk.itmo.reference; + +import java.io.IOException; +import java.lang.foreign.MemorySegment; +import java.nio.ByteBuffer; + +/** + * Growable buffer with {@link ByteBuffer} and {@link MemorySegment} interface. + * + * @author incubos + */ +final class ByteArraySegment { + private byte[] array; + private MemorySegment segment; + + ByteArraySegment(final int capacity) { + this.array = new byte[capacity]; + this.segment = MemorySegment.ofArray(array); + } + + void withArray(final ArrayConsumer consumer) throws IOException { + consumer.process(array); + } + + MemorySegment segment() { + return segment; + } + + void ensureCapacity(final long size) { + if (size > Integer.MAX_VALUE) { + throw new IllegalArgumentException("Too big!"); + } + + final int capacity = (int) size; + if (array.length >= capacity) { + return; + } + + // Grow to the nearest bigger power of 2 + final int newSize = Integer.highestOneBit(capacity) << 1; + array = new byte[newSize]; + segment = MemorySegment.ofArray(array); + } + + interface ArrayConsumer { + void process(byte[] array) throws IOException; + } +} diff --git a/src/main/java/ru/vk/itmo/reference/LiveFilteringIterator.java b/src/main/java/ru/vk/itmo/reference/LiveFilteringIterator.java new file mode 100644 index 000000000..676bc0e37 --- /dev/null +++ b/src/main/java/ru/vk/itmo/reference/LiveFilteringIterator.java @@ -0,0 +1,52 @@ +package ru.vk.itmo.reference; + +import ru.vk.itmo.Entry; + +import java.lang.foreign.MemorySegment; +import java.util.Iterator; +import java.util.NoSuchElementException; + +/** + * Filters non tombstone {@link Entry}s. + * + * @author incubos + */ +final class LiveFilteringIterator implements Iterator> { + private final Iterator> delegate; + private Entry next; + + LiveFilteringIterator(final Iterator> delegate) { + this.delegate = delegate; + skipTombstones(); + } + + private void skipTombstones() { + while (delegate.hasNext()) { + final Entry entry = delegate.next(); + if (entry.value() != null) { + this.next = entry; + break; + } + } + } + + @Override + public boolean hasNext() { + return next != null; + } + + @Override + public Entry next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + + // Consume + final Entry result = next; + next = null; + + skipTombstones(); + + return result; + } +} diff --git a/src/main/java/ru/vk/itmo/reference/MemTable.java b/src/main/java/ru/vk/itmo/reference/MemTable.java new file mode 100644 index 000000000..96843c04b --- /dev/null +++ b/src/main/java/ru/vk/itmo/reference/MemTable.java @@ -0,0 +1,49 @@ +package ru.vk.itmo.reference; + +import ru.vk.itmo.Entry; + +import java.lang.foreign.MemorySegment; +import java.util.Iterator; +import java.util.NavigableMap; +import java.util.concurrent.ConcurrentSkipListMap; + +/** + * Memory table. + * + * @author incubos + */ +final class MemTable { + private final NavigableMap> map = + new ConcurrentSkipListMap<>( + MemorySegmentComparator.INSTANCE); + + boolean isEmpty() { + return map.isEmpty(); + } + + Iterator> get( + final MemorySegment from, + final MemorySegment to) { + if (from == null && to == null) { + // All + return map.values().iterator(); + } else if (from == null) { + // Head + return map.headMap(to).values().iterator(); + } else if (to == null) { + // Tail + return map.tailMap(from).values().iterator(); + } else { + // Slice + return map.subMap(from, to).values().iterator(); + } + } + + Entry get(final MemorySegment key) { + return map.get(key); + } + + Entry upsert(final Entry entry) { + return map.put(entry.key(), entry); + } +} diff --git a/src/main/java/ru/vk/itmo/reference/MemorySegmentComparator.java b/src/main/java/ru/vk/itmo/reference/MemorySegmentComparator.java new file mode 100644 index 000000000..33f2936f1 --- /dev/null +++ b/src/main/java/ru/vk/itmo/reference/MemorySegmentComparator.java @@ -0,0 +1,89 @@ +package ru.vk.itmo.reference; + +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.util.Comparator; + +/** + * Compares {@link MemorySegment}s. + * + * @author incubos + */ +final class MemorySegmentComparator implements Comparator { + static final Comparator INSTANCE = + new MemorySegmentComparator(); + + private MemorySegmentComparator() { + // Singleton + } + + @Override + public int compare( + final MemorySegment left, + final MemorySegment right) { + final long mismatch = left.mismatch(right); + if (mismatch == -1L) { + // No mismatch + return 0; + } + + if (mismatch == left.byteSize()) { + // left is prefix of right, so left is smaller + return -1; + } + + if (mismatch == right.byteSize()) { + // right is prefix of left, so left is greater + return 1; + } + + // Compare mismatched bytes as unsigned + return Byte.compareUnsigned( + left.getAtIndex( + ValueLayout.OfByte.JAVA_BYTE, + mismatch), + right.getAtIndex( + ValueLayout.OfByte.JAVA_BYTE, + mismatch)); + } + + static int compare( + final MemorySegment srcSegment, + final long srcFromOffset, + final long srcLength, + final MemorySegment dstSegment, + final long dstFromOffset, + final long dstLength) { + final long mismatch = + MemorySegment.mismatch( + srcSegment, + srcFromOffset, + srcFromOffset + srcLength, + dstSegment, + dstFromOffset, + dstFromOffset + dstLength); + if (mismatch == -1L) { + // No mismatch + return 0; + } + + if (mismatch == srcLength) { + // left is prefix of right, so left is smaller + return -1; + } + + if (mismatch == dstLength) { + // right is prefix of left, so left is greater + return 1; + } + + // Compare mismatched bytes as unsigned + return Byte.compareUnsigned( + srcSegment.getAtIndex( + ValueLayout.OfByte.JAVA_BYTE, + srcFromOffset + mismatch), + dstSegment.getAtIndex( + ValueLayout.OfByte.JAVA_BYTE, + dstFromOffset + mismatch)); + } +} diff --git a/src/main/java/ru/vk/itmo/reference/MergingEntryIterator.java b/src/main/java/ru/vk/itmo/reference/MergingEntryIterator.java new file mode 100644 index 000000000..8130da2cb --- /dev/null +++ b/src/main/java/ru/vk/itmo/reference/MergingEntryIterator.java @@ -0,0 +1,72 @@ +package ru.vk.itmo.reference; + +import ru.vk.itmo.Entry; + +import java.lang.foreign.MemorySegment; +import java.util.Iterator; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.PriorityQueue; +import java.util.Queue; + +/** + * Merges entry {@link Iterator}s. + * + * @author incubos + */ +final class MergingEntryIterator implements Iterator> { + private final Queue iterators; + + MergingEntryIterator(final List iterators) { + assert iterators.stream().allMatch(WeightedPeekingEntryIterator::hasNext); + + this.iterators = new PriorityQueue<>(iterators); + } + + @Override + public boolean hasNext() { + return !iterators.isEmpty(); + } + + @Override + public Entry next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + + final WeightedPeekingEntryIterator top = iterators.remove(); + final Entry result = top.next(); + + if (top.hasNext()) { + // Not exhausted + iterators.add(top); + } + + // Remove older versions of the key + while (true) { + final WeightedPeekingEntryIterator iterator = iterators.peek(); + if (iterator == null) { + // Nothing left + break; + } + + // Skip entries with the same key + final Entry entry = iterator.peek(); + if (MemorySegmentComparator.INSTANCE.compare(result.key(), entry.key()) != 0) { + // Reached another key + break; + } + + // Drop + iterators.remove(); + // Skip + iterator.next(); + if (iterator.hasNext()) { + // Not exhausted + iterators.add(iterator); + } + } + + return result; + } +} diff --git a/src/main/java/ru/vk/itmo/reference/ReferenceDao.java b/src/main/java/ru/vk/itmo/reference/ReferenceDao.java new file mode 100644 index 000000000..1531f78e2 --- /dev/null +++ b/src/main/java/ru/vk/itmo/reference/ReferenceDao.java @@ -0,0 +1,292 @@ +package ru.vk.itmo.reference; + +import ru.vk.itmo.Config; +import ru.vk.itmo.Dao; +import ru.vk.itmo.Entry; + +import java.io.IOException; +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.util.Collections; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +/** + * Reference implementation of {@link Dao}. + * + * @author incubos + */ +public class ReferenceDao implements Dao> { + private final Config config; + private final Arena arena; + + private final ReadWriteLock lock = new ReentrantReadWriteLock(); + // Guarded by lock + private volatile TableSet tableSet; + + private final ExecutorService flusher = + Executors.newSingleThreadExecutor(r -> { + final Thread result = new Thread(r); + result.setName("flusher"); + return result; + }); + private final ExecutorService compactor = + Executors.newSingleThreadExecutor(r -> { + final Thread result = new Thread(r); + result.setName("compactor"); + return result; + }); + + private final AtomicBoolean closed = new AtomicBoolean(); + + public ReferenceDao(final Config config) throws IOException { + this.config = config; + this.arena = Arena.ofShared(); + + // First complete promotion of compacted SSTables + SSTables.promote( + config.basePath(), + 0, + 1); + + this.tableSet = + TableSet.from( + SSTables.discover( + arena, + config.basePath())); + } + + @Override + public Iterator> get( + final MemorySegment from, + final MemorySegment to) { + return new LiveFilteringIterator( + tableSet.get( + from, + to)); + } + + @Override + public Entry get(final MemorySegment key) { + // Without lock, just snapshot of table set + return tableSet.get(key); + } + + @Override + public void upsert(final Entry entry) { + final boolean autoFlush; + lock.readLock().lock(); + try { + if (tableSet.memTableSize.get() > config.flushThresholdBytes() + && tableSet.flushingTable != null) { + throw new IllegalStateException("Can't keep up with flushing!"); + } + + // Upsert + final Entry previous = tableSet.upsert(entry); + + // Update size estimate + final long size = tableSet.memTableSize.addAndGet(sizeOf(entry) - sizeOf(previous)); + autoFlush = size > config.flushThresholdBytes(); + } finally { + lock.readLock().unlock(); + } + + if (autoFlush) { + initiateFlush(true); + } + } + + private static long sizeOf(final Entry entry) { + if (entry == null) { + return 0L; + } + + if (entry.value() == null) { + return entry.key().byteSize(); + } + + return entry.key().byteSize() + entry.value().byteSize(); + } + + private void initiateFlush(final boolean auto) { + flusher.submit(() -> { + final TableSet currentTableSet; + lock.writeLock().lock(); + try { + if (this.tableSet.memTable.isEmpty()) { + // Nothing to flush + return; + } + + if (auto && this.tableSet.memTableSize.get() < config.flushThresholdBytes()) { + // Not enough data to flush + return; + } + + // Switch memTable to flushing + currentTableSet = this.tableSet.flushing(); + this.tableSet = currentTableSet; + } finally { + lock.writeLock().unlock(); + } + + // Write + final int sequence = currentTableSet.nextSequence(); + try { + new SSTableWriter() + .write( + config.basePath(), + sequence, + currentTableSet.flushingTable.get(null, null)); + } catch (IOException e) { + e.printStackTrace(); + Runtime.getRuntime().halt(-1); + return; + } + + // Open + final SSTable flushed; + try { + flushed = SSTables.open( + arena, + config.basePath(), + sequence); + } catch (IOException e) { + e.printStackTrace(); + Runtime.getRuntime().halt(-2); + return; + } + + // Switch + lock.writeLock().lock(); + try { + this.tableSet = this.tableSet.flushed(flushed); + } finally { + lock.writeLock().unlock(); + } + }).state(); + } + + @Override + public void flush() throws IOException { + initiateFlush(false); + } + + @Override + public void compact() throws IOException { + compactor.submit(() -> { + final TableSet currentTableSet; + lock.writeLock().lock(); + try { + currentTableSet = this.tableSet; + if (currentTableSet.ssTables.size() < 2) { + // Nothing to compact + return; + } + } finally { + lock.writeLock().unlock(); + } + + // Compact to 0 + try { + new SSTableWriter() + .write( + config.basePath(), + 0, + new LiveFilteringIterator( + currentTableSet.allSSTableEntries())); + } catch (IOException e) { + e.printStackTrace(); + Runtime.getRuntime().halt(-3); + } + + // Open 0 + final SSTable compacted; + try { + compacted = + SSTables.open( + arena, + config.basePath(), + 0); + } catch (IOException e) { + e.printStackTrace(); + Runtime.getRuntime().halt(-4); + return; + } + + // Replace old SSTables with compacted one to + // keep serving requests + final Set replaced = new HashSet<>(currentTableSet.ssTables); + lock.writeLock().lock(); + try { + this.tableSet = + this.tableSet.compacted( + replaced, + compacted); + } finally { + lock.writeLock().unlock(); + } + + // Remove compacted SSTables starting from the oldest ones. + // If we crash, 0 contains all the data, and + // it will be promoted on reopen. + for (final SSTable ssTable : currentTableSet.ssTables.reversed()) { + try { + SSTables.remove( + config.basePath(), + ssTable.sequence); + } catch (IOException e) { + e.printStackTrace(); + Runtime.getRuntime().halt(-5); + } + } + + // Promote zero to one (possibly replacing) + try { + SSTables.promote( + config.basePath(), + 0, + 1); + } catch (IOException e) { + e.printStackTrace(); + Runtime.getRuntime().halt(-6); + } + + // Replace promoted SSTable + lock.writeLock().lock(); + try { + this.tableSet = + this.tableSet.compacted( + Collections.singleton(compacted), + compacted.withSequence(1)); + } finally { + lock.writeLock().unlock(); + } + }).state(); + } + + @Override + public void close() throws IOException { + if (closed.getAndSet(true)) { + // Already closed + return; + } + + // Maybe flush + flush(); + + // Stop all the threads + flusher.close(); + compactor.close(); + + // Close arena + arena.close(); + } +} diff --git a/src/main/java/ru/vk/itmo/reference/SSTable.java b/src/main/java/ru/vk/itmo/reference/SSTable.java new file mode 100644 index 000000000..02217ab65 --- /dev/null +++ b/src/main/java/ru/vk/itmo/reference/SSTable.java @@ -0,0 +1,204 @@ +package ru.vk.itmo.reference; + +import ru.vk.itmo.BaseEntry; +import ru.vk.itmo.Entry; + +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.util.Collections; +import java.util.Iterator; +import java.util.NoSuchElementException; + +/** + * Persistent SSTable in data file and index file. + * + * @author incubos + * @see SSTables + */ +final class SSTable { + final int sequence; + + private final MemorySegment index; + private final MemorySegment data; + private final long size; + + SSTable( + final int sequence, + final MemorySegment index, + final MemorySegment data) { + this.sequence = sequence; + this.index = index; + this.data = data; + this.size = index.byteSize() / Long.BYTES; + } + + SSTable withSequence(final int sequence) { + return new SSTable( + sequence, + index, + data); + } + + /** + * Returns index of the entry if found; otherwise, (-(insertion point) - 1). + * The insertion point is defined as the point at which the key would be inserted: + * the index of the first element greater than the key, + * or size if all keys are less than the specified key. + * Note that this guarantees that the return value will be >= 0 + * if and only if the key is found. + */ + private long entryBinarySearch(final MemorySegment key) { + long low = 0L; + long high = size - 1; + + while (low <= high) { + final long mid = (low + high) >>> 1; + final long midEntryOffset = entryOffset(mid); + final long midKeyLength = getLength(midEntryOffset); + final int compare = + MemorySegmentComparator.compare( + data, + midEntryOffset + Long.BYTES, // Position at key + midKeyLength, + key, + 0L, + key.byteSize()); + + if (compare < 0) { + low = mid + 1; + } else if (compare > 0) { + high = mid - 1; + } else { + return mid; + } + } + + return -(low + 1); + } + + private long entryOffset(final long entry) { + return index.get( + ValueLayout.OfLong.JAVA_LONG, + entry * Long.BYTES); + } + + private long getLength(final long offset) { + return data.get( + ValueLayout.OfLong.JAVA_LONG_UNALIGNED, + offset); + } + + Iterator> get( + final MemorySegment from, + final MemorySegment to) { + assert from == null || to == null || MemorySegmentComparator.INSTANCE.compare(from, to) <= 0; + + // Slice of SSTable in absolute offsets + final long fromOffset; + final long toOffset; + + // Left offset bound + if (from == null) { + // Start from the beginning + fromOffset = 0L; + } else { + final long fromEntry = entryBinarySearch(from); + if (fromEntry >= 0L) { + fromOffset = entryOffset(fromEntry); + } else if (-fromEntry - 1 == size) { + // No relevant data + return Collections.emptyIterator(); + } else { + // Greater but existing key found + fromOffset = entryOffset(-fromEntry - 1); + } + } + + // Right offset bound + if (to == null) { + // Up to the end + toOffset = data.byteSize(); + } else { + final long toEntry = entryBinarySearch(to); + if (toEntry >= 0L) { + toOffset = entryOffset(toEntry); + } else if (-toEntry - 1 == size) { + // Up to the end + toOffset = data.byteSize(); + } else { + // Greater but existing key found + toOffset = entryOffset(-toEntry - 1); + } + } + + return new SliceIterator(fromOffset, toOffset); + } + + Entry get(final MemorySegment key) { + final long entry = entryBinarySearch(key); + if (entry < 0) { + return null; + } + + // Skip key (will reuse the argument) + long offset = entryOffset(entry); + offset += Long.BYTES + key.byteSize(); + // Extract value length + final long valueLength = getLength(offset); + if (valueLength == SSTables.TOMBSTONE_VALUE_LENGTH) { + // Tombstone encountered + return new BaseEntry<>(key, null); + } else { + // Get value + offset += Long.BYTES; + final MemorySegment value = data.asSlice(offset, valueLength); + return new BaseEntry<>(key, value); + } + } + + private final class SliceIterator implements Iterator> { + private long offset; + private final long toOffset; + + private SliceIterator( + final long offset, + final long toOffset) { + this.offset = offset; + this.toOffset = toOffset; + } + + @Override + public boolean hasNext() { + return offset < toOffset; + } + + @Override + public Entry next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + + // Read key length + final long keyLength = getLength(offset); + offset += Long.BYTES; + + // Read key + final MemorySegment key = data.asSlice(offset, keyLength); + offset += keyLength; + + // Read value length + final long valueLength = getLength(offset); + offset += Long.BYTES; + + // Read value + if (valueLength == SSTables.TOMBSTONE_VALUE_LENGTH) { + // Tombstone encountered + return new BaseEntry<>(key, null); + } else { + final MemorySegment value = data.asSlice(offset, valueLength); + offset += valueLength; + return new BaseEntry<>(key, value); + } + } + } +} diff --git a/src/main/java/ru/vk/itmo/reference/SSTableWriter.java b/src/main/java/ru/vk/itmo/reference/SSTableWriter.java new file mode 100644 index 000000000..fa8b6612e --- /dev/null +++ b/src/main/java/ru/vk/itmo/reference/SSTableWriter.java @@ -0,0 +1,166 @@ +package ru.vk.itmo.reference; + +import ru.vk.itmo.Entry; + +import java.io.BufferedOutputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.nio.channels.FileChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.Iterator; + +/** + * Writes {@link Entry} {@link Iterator} to SSTable on disk. + * + *

Index file {@code .index} contains {@code long} offsets to entries in data file: + * {@code [offset0, offset1, ...]} + * + *

Data file {@code .data} contains serialized entries: + * {@code } + * + *

Tombstones are encoded as {@code valueLength} {@code -1} and no subsequent value. + * + * @author incubos + */ +final class SSTableWriter { + private static final int BUFFER_SIZE = 64 * 1024; + + // Reusable buffers to eliminate allocations. + // But excessive memory copying is still there :( + // Long cell + private final ByteArraySegment longBuffer = new ByteArraySegment(Long.BYTES); + // Growable blob cell + private final ByteArraySegment blobBuffer = new ByteArraySegment(512); + + void write( + final Path baseDir, + final int sequence, + final Iterator> entries) throws IOException { + // Write to temporary files + final Path tempIndexName = SSTables.tempIndexName(baseDir, sequence); + final Path tempDataName = SSTables.tempDataName(baseDir, sequence); + + // Delete temporary files to eliminate tails + Files.deleteIfExists(tempIndexName); + Files.deleteIfExists(tempDataName); + + // Iterate in a single pass! + // Will write through FileChannel despite extra memory copying and + // no buffering (which may be implemented later). + // Looking forward to MemorySegment facilities in FileChannel! + try (OutputStream index = + new BufferedOutputStream( + new FileOutputStream( + tempIndexName.toFile()), + BUFFER_SIZE); + OutputStream data = + new BufferedOutputStream( + new FileOutputStream( + tempDataName.toFile()), + BUFFER_SIZE)) { + long entryOffset = 0L; + + // Iterate and serialize + while (entries.hasNext()) { + // First write offset to the entry + writeLong(entryOffset, index); + + // Then write the entry + final Entry entry = entries.next(); + entryOffset += writeEntry(entry, data); + } + } + + // Publish files atomically + // FIRST index, LAST data + final Path indexName = + SSTables.indexName( + baseDir, + sequence); + Files.move( + tempIndexName, + indexName, + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING); + final Path dataName = + SSTables.dataName( + baseDir, + sequence); + Files.move( + tempDataName, + dataName, + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING); + } + + private void writeLong( + final long value, + final OutputStream os) throws IOException { + longBuffer.segment().set( + ValueLayout.OfLong.JAVA_LONG_UNALIGNED, + 0, + value); + longBuffer.withArray(os::write); + } + + private void writeSegment( + final MemorySegment value, + final OutputStream os) throws IOException { + final long size = value.byteSize(); + blobBuffer.ensureCapacity(size); + MemorySegment.copy( + value, + 0L, + blobBuffer.segment(), + 0L, + size); + blobBuffer.withArray(array -> + os.write( + array, + 0, + (int) size)); + } + + /** + * Writes {@link Entry} to {@link FileChannel}. + * + * @return written bytes + */ + private long writeEntry( + final Entry entry, + final OutputStream os) throws IOException { + final MemorySegment key = entry.key(); + final MemorySegment value = entry.value(); + long result = 0L; + + // Key size + writeLong(key.byteSize(), os); + result += Long.BYTES; + + // Key + writeSegment(key, os); + result += key.byteSize(); + + // Value size and possibly value + if (value == null) { + // Tombstone + writeLong(SSTables.TOMBSTONE_VALUE_LENGTH, os); + result += Long.BYTES; + } else { + // Value length + writeLong(value.byteSize(), os); + result += Long.BYTES; + + // Value + writeSegment(value, os); + result += value.byteSize(); + } + + return result; + } +} diff --git a/src/main/java/ru/vk/itmo/reference/SSTables.java b/src/main/java/ru/vk/itmo/reference/SSTables.java new file mode 100644 index 000000000..b154c6b34 --- /dev/null +++ b/src/main/java/ru/vk/itmo/reference/SSTables.java @@ -0,0 +1,162 @@ +package ru.vk.itmo.reference; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.nio.channels.FileChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.stream.Stream; + +/** + * Provides {@link SSTable} management facilities: dumping and discovery. + * + * @author incubos + */ +final class SSTables { + public static final String INDEX_SUFFIX = ".index"; + public static final String DATA_SUFFIX = ".data"; + public static final long TOMBSTONE_VALUE_LENGTH = -1L; + + private static final String TEMP_SUFFIX = ".tmp"; + + /** + * Can't instantiate. + */ + private SSTables() { + // Only static methods + } + + static Path indexName( + final Path baseDir, + final int sequence) { + return baseDir.resolve(sequence + INDEX_SUFFIX); + } + + static Path dataName( + final Path baseDir, + final int sequence) { + return baseDir.resolve(sequence + DATA_SUFFIX); + } + + static Path tempIndexName( + final Path baseDir, + final int sequence) { + return baseDir.resolve(sequence + INDEX_SUFFIX + TEMP_SUFFIX); + } + + static Path tempDataName( + final Path baseDir, + final int sequence) { + return baseDir.resolve(sequence + DATA_SUFFIX + TEMP_SUFFIX); + } + + /** + * Returns {@link List} of {@link SSTable}s from freshest to oldest. + */ + static List discover( + final Arena arena, + final Path baseDir) throws IOException { + if (!Files.exists(baseDir)) { + return Collections.emptyList(); + } + + final List result = new ArrayList<>(); + try (Stream files = Files.list(baseDir)) { + files.forEach(file -> { + final String fileName = file.getFileName().toString(); + if (!fileName.endsWith(DATA_SUFFIX)) { + // Skip non data + return; + } + + final int sequence = + // .data -> N + Integer.parseInt( + fileName.substring( + 0, + fileName.length() - DATA_SUFFIX.length())); + + try { + result.add(open(arena, baseDir, sequence)); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }); + } + + // Sort from freshest to oldest + result.sort((o1, o2) -> Integer.compare(o2.sequence, o1.sequence)); + + return Collections.unmodifiableList(result); + } + + static SSTable open( + final Arena arena, + final Path baseDir, + final int sequence) throws IOException { + final MemorySegment index = + mapReadOnly( + arena, + indexName(baseDir, sequence)); + final MemorySegment data = + mapReadOnly( + arena, + dataName(baseDir, sequence)); + + return new SSTable( + sequence, + index, + data); + } + + private static MemorySegment mapReadOnly( + final Arena arena, + final Path file) throws IOException { + try (FileChannel channel = + FileChannel.open( + file, + StandardOpenOption.READ)) { + return channel.map( + FileChannel.MapMode.READ_ONLY, + 0L, + Files.size(file), + arena); + } + } + + static void remove( + final Path baseDir, + final int sequence) throws IOException { + // First delete data file to make SSTable invisible + Files.delete(dataName(baseDir, sequence)); + Files.delete(indexName(baseDir, sequence)); + } + + static void promote( + final Path baseDir, + final int from, + final int to) throws IOException { + // Build to progress to the same outcome + if (Files.exists(indexName(baseDir, from))) { + Files.move( + indexName(baseDir, from), + indexName(baseDir, to), + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING); + } + if (Files.exists(dataName(baseDir, from))) { + Files.move( + dataName(baseDir, from), + dataName(baseDir, to), + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING); + } + } +} diff --git a/src/main/java/ru/vk/itmo/reference/TableSet.java b/src/main/java/ru/vk/itmo/reference/TableSet.java new file mode 100644 index 000000000..ab249e8bd --- /dev/null +++ b/src/main/java/ru/vk/itmo/reference/TableSet.java @@ -0,0 +1,205 @@ +package ru.vk.itmo.reference; + +import ru.vk.itmo.Entry; + +import java.lang.foreign.MemorySegment; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Set; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Data set in various tables. + * + * @author incubos + */ +final class TableSet { + final MemTable memTable; + final AtomicLong memTableSize; + // null or read-only + final MemTable flushingTable; + // From freshest to oldest + final List ssTables; + + private TableSet( + final MemTable memTable, + final AtomicLong memTableSize, + final MemTable flushingTable, + final List ssTables) { + this.memTable = memTable; + this.memTableSize = memTableSize; + this.flushingTable = flushingTable; + this.ssTables = ssTables; + } + + static TableSet from(final List ssTables) { + return new TableSet( + new MemTable(), + new AtomicLong(), + null, + ssTables); + } + + int nextSequence() { + return ssTables.stream() + .mapToInt(t -> t.sequence) + .max() + .orElse(0) + 1; + } + + TableSet flushing() { + if (memTable.isEmpty()) { + throw new IllegalStateException("Nothing to flush"); + } + + if (flushingTable != null) { + throw new IllegalStateException("Already flushing"); + } + + return new TableSet( + new MemTable(), + new AtomicLong(), + memTable, + ssTables); + } + + TableSet flushed(final SSTable flushed) { + final List newSSTables = new ArrayList<>(ssTables.size() + 1); + newSSTables.add(flushed); + newSSTables.addAll(ssTables); + return new TableSet( + memTable, + memTableSize, + null, + newSSTables); + } + + TableSet compacted( + final Set replaced, + final SSTable with) { + final List newSsTables = new ArrayList<>(this.ssTables.size() + 1); + + // Keep not replaced SSTables + for (final SSTable ssTable : this.ssTables) { + if (!replaced.contains(ssTable)) { + newSsTables.add(ssTable); + } + } + + // Logically the oldest one + newSsTables.add(with); + + return new TableSet( + memTable, + memTableSize, + flushingTable, + newSsTables); + } + + Iterator> get( + final MemorySegment from, + final MemorySegment to) { + final List iterators = + new ArrayList<>(2 + ssTables.size()); + + // MemTable goes first + final Iterator> memTableIterator = + memTable.get(from, to); + if (memTableIterator.hasNext()) { + iterators.add( + new WeightedPeekingEntryIterator( + Integer.MIN_VALUE, + memTableIterator)); + } + + // Then goes flushing + if (flushingTable != null) { + final Iterator> flushingIterator = + flushingTable.get(from, to); + if (flushingIterator.hasNext()) { + iterators.add( + new WeightedPeekingEntryIterator( + Integer.MIN_VALUE + 1, + flushingIterator)); + } + } + + // Then go all the SSTables + for (int i = 0; i < ssTables.size(); i++) { + final SSTable ssTable = ssTables.get(i); + final Iterator> ssTableIterator = + ssTable.get(from, to); + if (ssTableIterator.hasNext()) { + iterators.add( + new WeightedPeekingEntryIterator( + i, + ssTableIterator)); + } + } + + return switch (iterators.size()) { + case 0 -> Collections.emptyIterator(); + case 1 -> iterators.get(0); + default -> new MergingEntryIterator(iterators); + }; + } + + Entry get(final MemorySegment key) { + // Slightly optimized version not to pollute the heap + + // First check MemTable + Entry result = memTable.get(key); + if (result != null) { + // Transform tombstone + return swallowTombstone(result); + } + + // Then check flushing + if (flushingTable != null) { + result = flushingTable.get(key); + if (result != null) { + // Transform tombstone + return swallowTombstone(result); + } + } + + // At last check SSTables from freshest to oldest + for (final SSTable ssTable : ssTables) { + result = ssTable.get(key); + if (result != null) { + // Transform tombstone + return swallowTombstone(result); + } + } + + // Nothing found + return null; + } + + private static Entry swallowTombstone(final Entry entry) { + return entry.value() == null ? null : entry; + } + + Entry upsert(final Entry entry) { + return memTable.upsert(entry); + } + + Iterator> allSSTableEntries() { + final List iterators = + new ArrayList<>(ssTables.size()); + + for (int i = 0; i < ssTables.size(); i++) { + final SSTable ssTable = ssTables.get(i); + final Iterator> ssTableIterator = + ssTable.get(null, null); + iterators.add( + new WeightedPeekingEntryIterator( + i, + ssTableIterator)); + } + + return new MergingEntryIterator(iterators); + } +} diff --git a/src/main/java/ru/vk/itmo/reference/WeightedPeekingEntryIterator.java b/src/main/java/ru/vk/itmo/reference/WeightedPeekingEntryIterator.java new file mode 100644 index 000000000..683bd1179 --- /dev/null +++ b/src/main/java/ru/vk/itmo/reference/WeightedPeekingEntryIterator.java @@ -0,0 +1,67 @@ +package ru.vk.itmo.reference; + +import ru.vk.itmo.Entry; + +import java.lang.foreign.MemorySegment; +import java.util.Iterator; +import java.util.NoSuchElementException; + +/** + * Peeking {@link Iterator} wrapper. + * + * @author incubos + */ +final class WeightedPeekingEntryIterator + implements Iterator>, + Comparable { + private final int weight; + private final Iterator> delegate; + private Entry next; + + WeightedPeekingEntryIterator( + final int weight, + final Iterator> delegate) { + this.weight = weight; + this.delegate = delegate; + this.next = delegate.hasNext() ? delegate.next() : null; + } + + @Override + public boolean hasNext() { + return next != null; + } + + @Override + public Entry next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + + final Entry result = next; + next = delegate.hasNext() ? delegate.next() : null; + return result; + } + + Entry peek() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + + return next; + } + + @Override + public int compareTo(final WeightedPeekingEntryIterator other) { + // First compare keys + int result = + MemorySegmentComparator.INSTANCE.compare( + peek().key(), + other.peek().key()); + if (result != 0) { + return result; + } + + // Then compare weights if keys are equal + return Integer.compare(weight, other.weight); + } +} diff --git a/src/main/java/ru/vk/itmo/test/reference/ReferenceDaoFactory.java b/src/main/java/ru/vk/itmo/test/reference/ReferenceDaoFactory.java new file mode 100644 index 000000000..5b410908f --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/reference/ReferenceDaoFactory.java @@ -0,0 +1,49 @@ +package ru.vk.itmo.test.reference; + +import ru.vk.itmo.Config; +import ru.vk.itmo.Dao; +import ru.vk.itmo.Entry; +import ru.vk.itmo.reference.ReferenceDao; +import ru.vk.itmo.test.DaoFactory; + +import java.io.IOException; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.nio.charset.StandardCharsets; + +/** + * Instantiates {@link ReferenceDao}. + * + * @author incubos + */ +@DaoFactory(stage = 5) +public class ReferenceDaoFactory implements DaoFactory.Factory> { + @Override + public Dao> createDao(final Config config) throws IOException { + return new ReferenceDao(config); + } + + @Override + public String toString(final MemorySegment memorySegment) { + if (memorySegment == null) { + return null; + } + + final byte[] array = memorySegment.toArray(ValueLayout.JAVA_BYTE); + return new String(array, StandardCharsets.UTF_8); + } + + @Override + public MemorySegment fromString(final String data) { + return data == null + ? null + : MemorySegment.ofArray( + data.getBytes( + StandardCharsets.UTF_8)); + } + + @Override + public Entry fromBaseEntry(final Entry baseEntry) { + return baseEntry; + } +} diff --git a/src/test/java/ru/vk/itmo/BaseTest.java b/src/test/java/ru/vk/itmo/BaseTest.java index 8ecc37b02..c4f38b8bb 100644 --- a/src/test/java/ru/vk/itmo/BaseTest.java +++ b/src/test/java/ru/vk/itmo/BaseTest.java @@ -6,6 +6,7 @@ import ru.vk.itmo.test.DaoFactory; import java.io.IOException; +import java.nio.channels.IllegalBlockingModeException; import java.nio.file.FileVisitResult; import java.nio.file.Files; import java.nio.file.Path; @@ -84,7 +85,7 @@ public void assertValueAt(Dao> dao, int index) throws IOEx assertSame(dao.get(keyAt(index)), entryAt(index)); } - public void sleep(int millis) { + public static void sleep(final int millis) { try { Thread.sleep(millis); } catch (InterruptedException e) { @@ -255,4 +256,35 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IO return result[0]; } + public static void retry( + final long timeoutNanos, + final Runnable runnable) { + long elapsedNanos; + while (true) { + try { + long start = System.nanoTime(); + runnable.run(); + elapsedNanos = System.nanoTime() - start; + break; + } catch (Exception e) { + sleep(100); + } + } + + // Check timeout + if (elapsedNanos > timeoutNanos) { + throw new IllegalBlockingModeException(); + } + } + + public static void retry(Runnable runnable) { + while (true) { + try { + runnable.run(); + break; + } catch (Exception e) { + sleep(100); + } + } + } } diff --git a/src/test/java/ru/vk/itmo/BasicConcurrentTest.java b/src/test/java/ru/vk/itmo/BasicConcurrentTest.java index 0db9afaf4..82e3f435b 100644 --- a/src/test/java/ru/vk/itmo/BasicConcurrentTest.java +++ b/src/test/java/ru/vk/itmo/BasicConcurrentTest.java @@ -12,7 +12,7 @@ public class BasicConcurrentTest extends BaseTest { void test_10_000(Dao> dao) throws Exception { int count = 10_000; List> entries = entries("k", "v", count); - runInParallel(100, count, value -> dao.upsert(entries.get(value))).close(); + runInParallel(4, count, value -> dao.upsert(entries.get(value))).close(); assertSame(dao.all(), entries); } @@ -21,7 +21,7 @@ void test_10_000(Dao> dao) throws Exception { void testConcurrentRW_2_500(Dao> dao) throws Exception { int count = 2_500; List> entries = entries("k", "v", count); - runInParallel(100, count, value -> { + runInParallel(4, count, value -> { dao.upsert(entries.get(value)); assertContains(dao.all(), entries.get(value)); }).close(); @@ -36,7 +36,7 @@ void testConcurrentRead_8_000(Dao> dao) throws Exception { for (Entry entry : entries) { dao.upsert(entry); } - runInParallel(100, count, value -> assertContains(dao.all(), entries.get(value))).close(); + runInParallel(4, count, value -> assertContains(dao.all(), entries.get(value))).close(); assertSame(dao.all(), entries); } diff --git a/src/test/java/ru/vk/itmo/BasicTest.java b/src/test/java/ru/vk/itmo/BasicTest.java index 0231f8374..11a882ffd 100644 --- a/src/test/java/ru/vk/itmo/BasicTest.java +++ b/src/test/java/ru/vk/itmo/BasicTest.java @@ -142,12 +142,10 @@ void testHugeData(Dao> dao) throws Exception { final int entries = 100_000; for (int entry = 0; entry < entries; entry++) { - dao.upsert(entry(keyAt(entry), valueAt(entry))); + final int e = entry; - // Back off after 1K upserts to be able to flush - if (entry % 1000 == 0) { - Thread.sleep(10); - } + // Retry if autoflush is too slow + retry(() -> dao.upsert(entry(keyAt(e), valueAt(e)))); } for (int entry = 0; entry < entries; entry++) { diff --git a/src/test/java/ru/vk/itmo/PersistentConcurrentTest.java b/src/test/java/ru/vk/itmo/PersistentConcurrentTest.java index b26878442..901f8e79f 100644 --- a/src/test/java/ru/vk/itmo/PersistentConcurrentTest.java +++ b/src/test/java/ru/vk/itmo/PersistentConcurrentTest.java @@ -15,13 +15,13 @@ public class PersistentConcurrentTest extends BaseTest { void testConcurrentRW_2_500_2(Dao> dao) throws Exception { int count = 2_500; List> entries = entries("k", "v", count); - runInParallel(100, count, value -> { + runInParallel(4, count, value -> { dao.upsert(entries.get(value)); }).close(); dao.close(); Dao> dao2 = DaoFactory.Factory.reopen(dao); - runInParallel(100, count, value -> { + runInParallel(4, count, value -> { assertSame(dao2.get(entries.get(value).key()), entries.get(value)); }).close(); } @@ -32,10 +32,10 @@ void testConcurrentRW_100_000_compact(Dao> dao) throws Exc List> entries = entries("k", "v", count); long timeoutNanosWarmup = TimeUnit.MILLISECONDS.toNanos(1000); - runInParallel(100, count, value -> { - tryRun(timeoutNanosWarmup, () -> dao.upsert(entries.get(value))); - tryRun(timeoutNanosWarmup, () -> dao.upsert(entry(keyAt(value), null))); - tryRun(timeoutNanosWarmup, () -> dao.upsert(entries.get(value))); + runInParallel(4, count, value -> { + retry(timeoutNanosWarmup, () -> dao.upsert(entries.get(value))); + retry(timeoutNanosWarmup, () -> dao.upsert(entry(keyAt(value), null))); + retry(timeoutNanosWarmup, () -> dao.upsert(entries.get(value))); }, () -> { for (int i = 0; i < 100; i++) { try { @@ -52,10 +52,10 @@ void testConcurrentRW_100_000_compact(Dao> dao) throws Exc // 200ms should be enough considering GC long timeoutNanos = TimeUnit.MILLISECONDS.toNanos(200); - runInParallel(100, count, value -> { - tryRun(timeoutNanos, () -> dao.upsert(entries.get(value))); - tryRun(timeoutNanos, () -> dao.upsert(entry(keyAt(value), null))); - tryRun(timeoutNanos, () -> dao.upsert(entries.get(value))); + runInParallel(4, count, value -> { + retry(timeoutNanos, () -> dao.upsert(entries.get(value))); + retry(timeoutNanos, () -> dao.upsert(entry(keyAt(value), null))); + retry(timeoutNanos, () -> dao.upsert(entries.get(value))); }, () -> { for (int i = 0; i < 100; i++) { try { @@ -72,7 +72,7 @@ void testConcurrentRW_100_000_compact(Dao> dao) throws Exc Dao> dao2 = DaoFactory.Factory.reopen(dao); runInParallel( - 100, + 4, count, value -> assertSame(dao2.get(entries.get(value).key()), entries.get(value))).close(); } @@ -89,26 +89,4 @@ private static void runAndMeasure( throw new IllegalBlockingModeException(); } } - - private static void tryRun( - long timeoutNanos, - Runnable runnable) throws InterruptedException { - long elapsedNanos; - while (true) { - try { - long start = System.nanoTime(); - runnable.run(); - elapsedNanos = System.nanoTime() - start; - break; - } catch (Exception e) { - //noinspection BusyWait - Thread.sleep(100); - } - } - - // Check timeout - if (elapsedNanos > timeoutNanos) { - throw new IllegalBlockingModeException(); - } - } } diff --git a/src/test/java/ru/vk/itmo/PersistentTest.java b/src/test/java/ru/vk/itmo/PersistentTest.java index 9746bfa51..e9b7f2497 100644 --- a/src/test/java/ru/vk/itmo/PersistentTest.java +++ b/src/test/java/ru/vk/itmo/PersistentTest.java @@ -66,12 +66,10 @@ void persistentPreventInMemoryStorage(Dao> dao) throws Exc // Fill List> entries = entries(keys); for (int entry = 0; entry < keys; entry++) { - dao.upsert(entries.get(entry)); + final int e = entry; - // Back off after 1K upserts to be able to flush - if (entry % 1000 == 0) { - Thread.sleep(10); - } + // Retry if autoflush is too slow + retry(() -> dao.upsert(entries.get(e))); } dao.close();