Examples with Range com.apple.foundationdb.Range used on opensource projects

Example 16 with Range

use of com.apple.foundationdb.Range in project fdb-record-layer by FoundationDB.

the class HighContentionAllocator method chooseWindow.

private CompletableFuture<AllocationWindow> chooseWindow(final AllocationWindow currentWindow, boolean wipeOld) {
    byte[] counterKey = counterSubspace.pack(currentWindow.getStart());
    Range oldCounters = new Range(counterSubspace.getKey(), counterKey);
    CompletableFuture<byte[]> newCount;
    synchronized (transaction) {
        if (wipeOld) {
            transaction.clear(oldCounters);
        }
        transaction.mutate(MutationType.ADD, counterKey, LITTLE_ENDIAN_LONG_ONE);
        newCount = transaction.snapshot().get(counterKey);
    }
    return newCount.thenApply(ByteArrayUtil::decodeInt).thenCompose(count -> {
        if (count * 2 > currentWindow.size()) {
            // advance the window and retry
            final AllocationWindow newWindow = AllocationWindow.startingFrom(currentWindow.getEnd());
            return chooseWindow(newWindow, true);
        }
        return CompletableFuture.completedFuture(currentWindow);
    });
}

Example 17 with Range

use of com.apple.foundationdb.Range in project fdb-record-layer by FoundationDB.

the class RangeSetTest method insert.

@Test
@Tag(Tags.Slow)
public void insert() {
    List<byte[]> keys = createKeys();
    List<Range> rangeSource = Arrays.asList(// Step 1: An initial range
    Range.startsWith(new byte[] { 0x10 }), // Step 2: An second disjoint range.
    Range.startsWith(new byte[] { 0x30 }), // Step 3: A third disjoint range between them.
    Range.startsWith(new byte[] { 0x20 }), // Step 4: A range overlapping the first range.
    new Range(new byte[] { 0x05 }, new byte[] { 0x10, 0x10 }), // Step 5: A range overlapping the third range.
    new Range(new byte[] { 0x2f }, new byte[] { 0x30, 0x10 }), // Step 6: A range overlapping the third range.
    new Range(new byte[] { 0x30, 0x10 }, new byte[] { 0x40 }), // Step 7: A range with just a little at the end.
    new Range(new byte[] { 0x05, 0x10 }, new byte[] { 0x12 }), // Step 8: A range that goes between 0x20 and 0x30 ranges.
    new Range(new byte[] { 0x20, 0x14 }, new byte[] { 0x30, 0x00 }), // Step 9: A range that goes over whole range.
    new Range(new byte[] { 0x03 }, new byte[] { 0x42 }), // Step 10: A range entirely within the given ranges.
    new Range(new byte[] { 0x10, 0x11 }, new byte[] { 0x41 }), // Step 11: A range that contains only 1 key.
    new Range(new byte[] { 0x50 }, new byte[] { 0x50, 0x00 }));
    List<Range> ranges = new ArrayList<>();
    int last = 0;
    // check for emptiness.
    for (int i = 0; i < 2; i++) {
        for (Range range : rangeSource) {
            boolean empty = ranges.stream().allMatch(r -> disjoint(range, r));
            boolean changes = rs.insertRange(db, range, i == 1).join();
            if (i == 1) {
                assertEquals(empty, changes, "Changes even though this was non-empty");
                if (!empty) {
                    changes = rs.insertRange(db, range).join();
                }
            }
            ranges.add(range);
            checkIncreasing();
            int present = checkConsistent(ranges, keys);
            // This check works only because each range added adds at least one key from keys.
            assertEquals(present != last, changes, "Keys changed even though ranges were supposedly static: present = " + present + ", last = " + last + ", changes = " + changes);
            last = present;
        }
        last = 0;
        rs.clear(db).join();
        ranges.clear();
    }
}

Example 18 with Range

use of com.apple.foundationdb.Range in project fdb-record-layer by FoundationDB.

the class RangeSetTest method missingRanges.

@Test
public void missingRanges() {
    // First, from empty.
    List<Range> ranges = db.readAsync(tr -> rs.missingRanges(tr).asList()).join();
    assertEquals(1, ranges.size());
    Range r1 = ranges.get(0);
    assertArrayEquals(new byte[] { 0x00 }, r1.begin);
    assertArrayEquals(new byte[] { (byte) 0xff }, r1.end);
    // Now, add some ranges to make the gaps interesting.
    List<Range> toAdd = Arrays.asList(new Range(new byte[] { 0x10 }, new byte[] { 0x20 }), new Range(new byte[] { 0x20 }, new byte[] { 0x20, 0x00 }), new Range(new byte[] { 0x30 }, new byte[] { 0x40 }), new Range(new byte[] { 0x40, 0x00 }, new byte[] { 0x50 }), new Range(new byte[] { 0x60 }, new byte[] { 0x6a }), new Range(new byte[] { 0x62 }, new byte[] { 0x70 }), new Range(new byte[] { 0x71 }, new byte[] { 0x72 }), new Range(new byte[] { 0x72 }, new byte[] { 0x7f }));
    List<Range> expected = Arrays.asList(new Range(new byte[] { 0x00 }, new byte[] { 0x10 }), new Range(new byte[] { 0x20, 0x00 }, new byte[] { 0x30 }), new Range(new byte[] { 0x40 }, new byte[] { 0x40, 0x00 }), new Range(new byte[] { 0x50 }, new byte[] { 0x60 }), new Range(new byte[] { 0x70 }, new byte[] { 0x71 }), new Range(new byte[] { 0x7f }, new byte[] { (byte) 0xff }));
    List<Boolean> added = db.runAsync(tr -> {
        List<CompletableFuture<Boolean>> futures = new ArrayList<>();
        for (Range range : toAdd) {
            futures.add(rs.insertRange(tr, range));
        }
        return AsyncUtil.getAll(futures);
    }).join();
    assertTrue(added.stream().allMatch(b -> b), "Some of the insertions didn't change things");
    ranges = rs.missingRanges(db).join();
    assertEquals(6, ranges.size());
    for (int i = 0; i < ranges.size(); i++) {
        assertArrayEquals(expected.get(i).begin, ranges.get(i).begin);
        assertArrayEquals(expected.get(i).end, ranges.get(i).end);
    }
    ranges = rs.missingRanges(db, new byte[] { 0x00 }, new byte[] { 0x60 }).join();
    assertEquals(4, ranges.size());
    for (int i = 0; i < ranges.size(); i++) {
        assertArrayEquals(expected.get(i).begin, ranges.get(i).begin);
        assertArrayEquals(expected.get(i).end, ranges.get(i).end);
    }
    ranges = rs.missingRanges(db, new Range(new byte[] { 0x00 }, new byte[] { 0x60 })).join();
    assertEquals(4, ranges.size());
    for (int i = 0; i < ranges.size(); i++) {
        assertArrayEquals(expected.get(i).begin, ranges.get(i).begin);
        assertArrayEquals(expected.get(i).end, ranges.get(i).end);
    }
    ranges = rs.missingRanges(db, new byte[] { 0x00 }, new byte[] { 0x60 }, 2).join();
    assertEquals(2, ranges.size());
    for (int i = 0; i < ranges.size(); i++) {
        assertArrayEquals(expected.get(i).begin, ranges.get(i).begin);
        assertArrayEquals(expected.get(i).end, ranges.get(i).end);
    }
    ranges = rs.missingRanges(db, new byte[] { 0x00 }, new byte[] { 0x60 }, 1).join();
    assertEquals(1, ranges.size());
    for (int i = 0; i < ranges.size(); i++) {
        assertArrayEquals(expected.get(i).begin, ranges.get(i).begin);
        assertArrayEquals(expected.get(i).end, ranges.get(i).end);
    }
}

Example 19 with Range

use of com.apple.foundationdb.Range in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreIndexTest method markReadable.

@Test
public void markReadable() throws Exception {
    final String indexName = "MySimpleRecord$str_value_indexed";
    try (FDBRecordContext context = openContext()) {
        uncheckedOpenSimpleRecordStore(context);
        assertThat(recordStore.isIndexWriteOnly(indexName), is(false));
        recordStore.clearAndMarkIndexWriteOnly(indexName).get();
        assertThat(recordStore.isIndexWriteOnly(indexName), is(true));
        context.commit();
    }
    try (FDBRecordContext context = openContext()) {
        uncheckedOpenSimpleRecordStore(context);
        Index index = recordStore.getRecordMetaData().getIndex(indexName);
        assertThat(recordStore.isIndexReadable(index), is(false));
        try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder().setRecordStore(recordStore).setIndex(index).build()) {
            indexBuilder.buildRange(recordStore, null, Key.Evaluated.scalar(1066L)).get();
            Optional<Range> firstUnbuilt = recordStore.firstUnbuiltRange(index).get();
            assertTrue(firstUnbuilt.isPresent());
            assertArrayEquals(Key.Evaluated.scalar(1066).toTuple().pack(), firstUnbuilt.get().begin);
            assertArrayEquals(new byte[] { (byte) 0xff }, firstUnbuilt.get().end);
            recordStore.markIndexReadable(index).handle((built, e) -> {
                assertNotNull(e);
                assertThat(e, instanceOf(CompletionException.class));
                assertNotNull(e.getCause());
                assertThat(e.getCause(), instanceOf(FDBRecordStore.IndexNotBuiltException.class));
                return null;
            }).get();
            assertThat(recordStore.isIndexReadable(index), is(false));
            indexBuilder.buildRange(recordStore, Key.Evaluated.scalar(1066L), null).get();
            assertFalse(recordStore.firstUnbuiltRange(index).get().isPresent());
            assertTrue(recordStore.markIndexReadable(index).get());
            assertFalse(recordStore.markIndexReadable(index).get());
            assertThat(recordStore.isIndexReadable(index), is(true));
        }
    // Not committing.
    }
    try (FDBRecordContext context = openContext()) {
        uncheckedOpenSimpleRecordStore(context);
        Index index = recordStore.getRecordMetaData().getIndex(indexName);
        assertThat(recordStore.isIndexReadable(index), is(false));
        try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder().setRecordStore(recordStore).setIndex(index).build()) {
            indexBuilder.buildRange(recordStore, null, Key.Evaluated.scalar(1066L)).get();
        }
        Optional<Range> firstUnbuilt = recordStore.firstUnbuiltRange(index).get();
        assertTrue(firstUnbuilt.isPresent());
        assertArrayEquals(Key.Evaluated.scalar(1066).toTuple().pack(), firstUnbuilt.get().begin);
        assertArrayEquals(new byte[] { (byte) 0xff }, firstUnbuilt.get().end);
        assertTrue(recordStore.uncheckedMarkIndexReadable(index.getName()).get());
        assertFalse(recordStore.uncheckedMarkIndexReadable(index.getName()).get());
        assertThat(recordStore.isIndexReadable(index), is(true));
        // Purposefully, checking to mark an index readable that is already
        // readable does not throw an error.
        firstUnbuilt = recordStore.firstUnbuiltRange(index).get();
        assertTrue(firstUnbuilt.isPresent());
        assertArrayEquals(Key.Evaluated.scalar(1066).toTuple().pack(), firstUnbuilt.get().begin);
        assertArrayEquals(new byte[] { (byte) 0xff }, firstUnbuilt.get().end);
        assertFalse(recordStore.markIndexReadable(index.getName()).get());
        assertThat(recordStore.isIndexReadable(index), is(true));
    }
}

Example 20 with Range

use of com.apple.foundationdb.Range in project fdb-record-layer by FoundationDB.

the class OnlineIndexerSimpleTest method testOnlineIndexerBuilderWriteLimitBytes.

@Test
public void testOnlineIndexerBuilderWriteLimitBytes() throws Exception {
    List<TestRecords1Proto.MySimpleRecord> records = LongStream.range(0, 200).mapToObj(val -> TestRecords1Proto.MySimpleRecord.newBuilder().setRecNo(val).setNumValue2((int) val + 1).build()).collect(Collectors.toList());
    Index index = new Index("newIndex", field("num_value_2").ungrouped(), IndexTypes.SUM);
    IndexAggregateFunction aggregateFunction = new IndexAggregateFunction(FunctionNames.SUM, index.getRootExpression(), index.getName());
    List<String> indexTypes = Collections.singletonList("MySimpleRecord");
    FDBRecordStoreTestBase.RecordMetaDataHook hook = metaDataBuilder -> metaDataBuilder.addIndex("MySimpleRecord", index);
    openSimpleMetaData();
    try (FDBRecordContext context = openContext()) {
        records.forEach(recordStore::saveRecord);
        context.commit();
    }
    openSimpleMetaData(hook);
    final FDBStoreTimer timer = new FDBStoreTimer();
    try (FDBRecordContext context = openContext()) {
        recordStore.checkVersion(null, FDBRecordStoreBase.StoreExistenceCheck.ERROR_IF_NOT_EXISTS).join();
        // Build in this transaction.
        try (OnlineIndexer indexer = OnlineIndexer.newBuilder().setRecordStore(recordStore).setTimer(timer).setIndex("newIndex").setLimit(100000).setMaxWriteLimitBytes(1).build()) {
            // this call will "flatten" the staccato iterations to a whole range. Testing compatibility.
            indexer.rebuildIndex(recordStore);
        }
        recordStore.markIndexReadable("newIndex").join();
        assertEquals(200, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RECORDS_SCANNED));
        assertEquals(200, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RECORDS_INDEXED));
        assertEquals(199, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RANGES_BY_SIZE));
        // last item
        assertEquals(1, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RANGES_BY_COUNT));
        context.commit();
    }
    try (FDBRecordContext context = openContext()) {
        assertTrue(recordStore.isIndexReadable("newIndex"));
        recordStore.clearAndMarkIndexWriteOnly("newIndex").join();
        context.commit();
    }
    timer.reset();
    try (FDBRecordContext context = openContext()) {
        recordStore.checkVersion(null, FDBRecordStoreBase.StoreExistenceCheck.ERROR_IF_NOT_EXISTS).join();
        // Build in this transaction.
        try (OnlineIndexer indexer = OnlineIndexer.newBuilder().setRecordStore(recordStore).setTimer(timer).setIndex("newIndex").setLimit(100000).setMaxWriteLimitBytes(1).build()) {
            Key.Evaluated key = indexer.buildUnbuiltRange(Key.Evaluated.scalar(0L), Key.Evaluated.scalar(25L)).join();
            assertEquals(1, key.getLong(0));
            assertEquals(1, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RANGES_BY_SIZE));
            assertEquals(0, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RANGES_BY_COUNT));
        }
        recordStore.clearAndMarkIndexWriteOnly("newIndex").join();
        context.commit();
    }
    timer.reset();
    try (OnlineIndexer indexer = OnlineIndexer.newBuilder().setDatabase(fdb).setMetaData(metaData).setSubspace(subspace).setTimer(timer).setIndex(index).setLimit(100000).setMaxWriteLimitBytes(1).setRecordsPerSecond(OnlineIndexer.UNLIMITED).build()) {
        indexer.buildIndex();
    }
    assertEquals(200, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RECORDS_SCANNED));
    assertEquals(200, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RECORDS_INDEXED));
    // this includes two endpoints + one range = total of 3 terminations by count
    // - note that (last, null] endpoint is en empty range
    assertEquals(3, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RANGES_BY_COUNT));
    // this is the range between the endpoints - 199 items in (first, last] interval
    assertEquals(198, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RANGES_BY_SIZE));
}