Examples with Tuple com.apple.foundationdb.tuple.Tuple used on opensource projects

Example 56 with Tuple

use of com.apple.foundationdb.tuple.Tuple in project fdb-record-layer by FoundationDB.

the class OnlineIndexerSimpleTest method buildRangeWithNull.

@Test
public void buildRangeWithNull() {
    List<TestRecords1Proto.MySimpleRecord> records = LongStream.range(0, 200).mapToObj(val -> TestRecords1Proto.MySimpleRecord.newBuilder().setRecNo(val).setNumValue2((int) val + 1).build()).collect(Collectors.toList());
    Index index = new Index("simple$value_2", field("num_value_2").ungrouped(), IndexTypes.SUM);
    IndexAggregateFunction aggregateFunction = new IndexAggregateFunction(FunctionNames.SUM, index.getRootExpression(), index.getName());
    List<String> indexTypes = Collections.singletonList("MySimpleRecord");
    FDBRecordStoreTestBase.RecordMetaDataHook hook = metaDataBuilder -> metaDataBuilder.addIndex("MySimpleRecord", index);
    openSimpleMetaData();
    try (FDBRecordContext context = openContext()) {
        records.forEach(recordStore::saveRecord);
        context.commit();
    }
    final Supplier<Tuple> getAggregate = () -> {
        Tuple ret;
        try (FDBRecordContext context = openContext()) {
            assertTrue(recordStore.uncheckedMarkIndexReadable(index.getName()).join());
            FDBRecordStore recordStore2 = recordStore.asBuilder().setContext(context).uncheckedOpen();
            ret = recordStore2.evaluateAggregateFunction(indexTypes, aggregateFunction, TupleRange.ALL, IsolationLevel.SERIALIZABLE).join();
        // Do NOT commit changes
        }
        return ret;
    };
    openSimpleMetaData(hook);
    try (FDBRecordContext context = openContext()) {
        recordStore.markIndexWriteOnly(index).join();
        context.commit();
    }
    try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder().setDatabase(fdb).setMetaData(metaData).setIndex(index).setSubspace(subspace).build()) {
        indexBuilder.buildRange(null, null).join();
        assertEquals(Tuple.from(20100L), getAggregate.get());
    }
}

Example 57 with Tuple

use of com.apple.foundationdb.tuple.Tuple in project fdb-record-layer by FoundationDB.

the class OnlineIndexerSimpleTest method buildEndpointIdempotency.

@Test
public void buildEndpointIdempotency() {
    List<TestRecords1Proto.MySimpleRecord> records = LongStream.range(0, 10).mapToObj(val -> TestRecords1Proto.MySimpleRecord.newBuilder().setRecNo(val).setNumValue2((int) val + 1).build()).collect(Collectors.toList());
    Index index = new Index("simple$value_2", field("num_value_2").ungrouped(), IndexTypes.SUM);
    IndexAggregateFunction aggregateFunction = new IndexAggregateFunction(FunctionNames.SUM, index.getRootExpression(), index.getName());
    List<String> indexTypes = Collections.singletonList("MySimpleRecord");
    FDBRecordStoreTestBase.RecordMetaDataHook hook = metaDataBuilder -> metaDataBuilder.addIndex("MySimpleRecord", index);
    final Supplier<Tuple> getAggregate = () -> {
        Tuple ret;
        try (FDBRecordContext context = openContext()) {
            assertTrue(recordStore.uncheckedMarkIndexReadable(index.getName()).join());
            FDBRecordStore recordStore2 = recordStore.asBuilder().setContext(context).uncheckedOpen();
            ret = recordStore2.evaluateAggregateFunction(indexTypes, aggregateFunction, TupleRange.ALL, IsolationLevel.SERIALIZABLE).join();
        // Do NOT commit the change.
        }
        return ret;
    };
    openSimpleMetaData();
    try (FDBRecordContext context = openContext()) {
        records.forEach(recordStore::saveRecord);
        context.commit();
    }
    openSimpleMetaData(hook);
    try (FDBRecordContext context = openContext()) {
        recordStore.markIndexWriteOnly(index).join();
        context.commit();
    }
    try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder().setDatabase(fdb).setMetaData(metaData).setIndex(index).setSubspace(subspace).build()) {
        final RangeSet rangeSet = new RangeSet(recordStore.indexRangeSubspace(index));
        // Build the endpoints
        TupleRange range = indexBuilder.buildEndpoints().join();
        assertEquals(Tuple.from(0L), range.getLow());
        assertEquals(Tuple.from(9L), range.getHigh());
        assertEquals(Tuple.from(10L), getAggregate.get());
        assertEquals(Collections.emptyList(), rangeSet.missingRanges(fdb.database(), null, Tuple.from(0L).pack()).join());
        assertEquals(Collections.emptyList(), rangeSet.missingRanges(fdb.database(), Tuple.from(9L).pack(), null).join());
        List<Range> middleRanges = rangeSet.missingRanges(fdb.database()).join();
        assertEquals(Collections.singletonList(Tuple.from(0L)), middleRanges.stream().map(r -> Tuple.fromBytes(r.begin)).collect(Collectors.toList()));
        assertEquals(Collections.singletonList(Tuple.from(9L)), middleRanges.stream().map(r -> Tuple.fromBytes(r.end)).collect(Collectors.toList()));
        // Make sure running this again doesn't change anything.
        range = indexBuilder.buildEndpoints().join();
        assertEquals(Tuple.from(0L), range.getLow());
        assertEquals(Tuple.from(9L), range.getHigh());
        assertEquals(Tuple.from(10L), getAggregate.get());
        assertEquals(Collections.emptyList(), rangeSet.missingRanges(fdb.database(), null, Tuple.from(0L).pack()).join());
        assertEquals(Collections.emptyList(), rangeSet.missingRanges(fdb.database(), Tuple.from(9L).pack(), null).join());
        middleRanges = rangeSet.missingRanges(fdb.database()).join();
        assertEquals(Collections.singletonList(Tuple.from(0L)), middleRanges.stream().map(r -> Tuple.fromBytes(r.begin)).collect(Collectors.toList()));
        assertEquals(Collections.singletonList(Tuple.from(9L)), middleRanges.stream().map(r -> Tuple.fromBytes(r.end)).collect(Collectors.toList()));
        // Remove the first and last records.
        try (FDBRecordContext context = openContext()) {
            recordStore.deleteRecord(Tuple.from(0L));
            recordStore.deleteRecord(Tuple.from(9L));
            context.commit();
        }
        assertEquals(Tuple.from(0L), getAggregate.get());
        // Rerun endpoints with new data.
        range = indexBuilder.buildEndpoints().join();
        assertEquals(Tuple.from(1L), range.getLow());
        assertEquals(Tuple.from(8L), range.getHigh());
        assertEquals(Tuple.from(9L), getAggregate.get());
        assertEquals(Collections.emptyList(), rangeSet.missingRanges(fdb.database(), null, Tuple.from(1L).pack()).join());
        assertEquals(Collections.emptyList(), rangeSet.missingRanges(fdb.database(), Tuple.from(8L).pack(), null).join());
        middleRanges = rangeSet.missingRanges(fdb.database()).join();
        assertEquals(Collections.singletonList(Tuple.from(1L)), middleRanges.stream().map(r -> Tuple.fromBytes(r.begin)).collect(Collectors.toList()));
        assertEquals(Collections.singletonList(Tuple.from(8L)), middleRanges.stream().map(r -> Tuple.fromBytes(r.end)).collect(Collectors.toList()));
        // Run it again to show that nothing has happened.
        range = indexBuilder.buildEndpoints().join();
        assertEquals(Tuple.from(1L), range.getLow());
        assertEquals(Tuple.from(8L), range.getHigh());
        assertEquals(Tuple.from(9L), getAggregate.get());
        assertEquals(Collections.emptyList(), rangeSet.missingRanges(fdb.database(), null, Tuple.from(1L).pack()).join());
        assertEquals(Collections.emptyList(), rangeSet.missingRanges(fdb.database(), Tuple.from(8L).pack(), null).join());
        middleRanges = rangeSet.missingRanges(fdb.database()).join();
        assertEquals(Collections.singletonList(Tuple.from(1L)), middleRanges.stream().map(r -> Tuple.fromBytes(r.begin)).collect(Collectors.toList()));
        assertEquals(Collections.singletonList(Tuple.from(8L)), middleRanges.stream().map(r -> Tuple.fromBytes(r.end)).collect(Collectors.toList()));
        // Add back the previous first and last records.
        try (FDBRecordContext context = openContext()) {
            recordStore.saveRecord(TestRecords1Proto.MySimpleRecord.newBuilder().setRecNo(0L).setNumValue2(1).build());
            recordStore.saveRecord(TestRecords1Proto.MySimpleRecord.newBuilder().setRecNo(9L).setNumValue2(10).build());
            context.commit();
        }
        assertEquals(Tuple.from(20L), getAggregate.get());
        // Rerun endpoints with new data.
        range = indexBuilder.buildEndpoints().join();
        assertEquals(Tuple.from(0L), range.getLow());
        assertEquals(Tuple.from(9L), range.getHigh());
        assertEquals(Tuple.from(20L), getAggregate.get());
        assertEquals(Collections.emptyList(), rangeSet.missingRanges(fdb.database(), null, Tuple.from(1L).pack()).join());
        assertEquals(Collections.emptyList(), rangeSet.missingRanges(fdb.database(), Tuple.from(8L).pack(), null).join());
        middleRanges = rangeSet.missingRanges(fdb.database()).join();
        assertEquals(Collections.singletonList(Tuple.from(1L)), middleRanges.stream().map(r -> Tuple.fromBytes(r.begin)).collect(Collectors.toList()));
        assertEquals(Collections.singletonList(Tuple.from(8L)), middleRanges.stream().map(r -> Tuple.fromBytes(r.end)).collect(Collectors.toList()));
        // Run it again to show that nothing has happened.
        range = indexBuilder.buildEndpoints().join();
        assertEquals(Tuple.from(0L), range.getLow());
        assertEquals(Tuple.from(9L), range.getHigh());
        assertEquals(Tuple.from(20L), getAggregate.get());
        assertEquals(Collections.emptyList(), rangeSet.missingRanges(fdb.database(), null, Tuple.from(1L).pack()).join());
        assertEquals(Collections.emptyList(), rangeSet.missingRanges(fdb.database(), Tuple.from(8L).pack(), null).join());
        middleRanges = rangeSet.missingRanges(fdb.database()).join();
        assertEquals(Collections.singletonList(Tuple.from(1L)), middleRanges.stream().map(r -> Tuple.fromBytes(r.begin)).collect(Collectors.toList()));
        assertEquals(Collections.singletonList(Tuple.from(8L)), middleRanges.stream().map(r -> Tuple.fromBytes(r.end)).collect(Collectors.toList()));
        // Straight up build the whole index.
        indexBuilder.buildIndex(false);
        assertEquals(Tuple.from(55L), getAggregate.get());
        assertEquals(Collections.emptyList(), rangeSet.missingRanges(fdb.database()).join());
    }
}

Example 58 with Tuple

use of com.apple.foundationdb.tuple.Tuple in project fdb-record-layer by FoundationDB.

the class SplitHelperTest method writeDummyRecords.

private List<FDBRawRecord> writeDummyRecords() throws Exception {
    final byte[] globalVersion = "_cushions_".getBytes(Charsets.US_ASCII);
    final List<FDBRawRecord> rawRecords = new ArrayList<>();
    // Generate primary keys using a generalization of the Fibonacci formula: https://oeis.org/A247698
    long currKey = 2308L;
    long nextKey = 4261L;
    try (FDBRecordContext context = openContext()) {
        for (int i = 0; i < 50; i++) {
            FDBRecordVersion version = (i % 2 == 0) ? FDBRecordVersion.complete(globalVersion, context.claimLocalVersion()) : null;
            byte[] rawBytes = (i % 4 < 2) ? SHORT_STRING : MEDIUM_STRING;
            Tuple key = Tuple.from(currKey);
            FDBStoredSizes sizes = writeDummyRecord(context, key, version, (i % 4 < 2) ? 1 : MEDIUM_COPIES, false);
            rawRecords.add(new FDBRawRecord(key, rawBytes, version, sizes));
            long temp = currKey + nextKey;
            currKey = nextKey;
            nextKey = temp;
        }
        commit(context);
    }
    return rawRecords;
}

Example 59 with Tuple

use of com.apple.foundationdb.tuple.Tuple in project fdb-record-layer by FoundationDB.

the class SplitHelperTest method saveWithSplitAndIncompleteVersions.

@MethodSource("splitAndSuffixArgs")
@ParameterizedTest(name = "saveWithSplitAndIncompleteVersions [splitLongRecords = {0}, omitUnsplitSuffix = {1}]")
public void saveWithSplitAndIncompleteVersions(boolean splitLongRecords, boolean omitUnsplitSuffix) throws Exception {
    final byte[] versionstamp;
    final byte[] globalVersion = "whiteroses".getBytes(Charsets.US_ASCII);
    try (FDBRecordContext context = openContext()) {
        // With incomplete version
        saveWithSplit(context, Tuple.from(962L), SHORT_STRING, FDBRecordVersion.incomplete(context.claimLocalVersion()), splitLongRecords, omitUnsplitSuffix);
        saveWithSplit(context, Tuple.from(967L), LONG_STRING, FDBRecordVersion.incomplete(context.claimLocalVersion()), splitLongRecords, omitUnsplitSuffix);
        saveWithSplit(context, Tuple.from(996L), VERY_LONG_STRING, FDBRecordVersion.incomplete(context.claimLocalVersion()), splitLongRecords, omitUnsplitSuffix);
        commit(context);
        versionstamp = context.getVersionStamp();
        if (!omitUnsplitSuffix) {
            assertNotNull(versionstamp);
        } else {
            assertNull(versionstamp);
        }
    }
    if (!omitUnsplitSuffix) {
        try (FDBRecordContext context = openContext()) {
            List<Pair<Tuple, FDBRecordVersion>> keys = Arrays.asList(Pair.of(Tuple.from(962L), FDBRecordVersion.complete(versionstamp, 0)), Pair.of(Tuple.from(967L), FDBRecordVersion.complete(versionstamp, 1)), Pair.of(Tuple.from(996L), FDBRecordVersion.complete(versionstamp, 2)));
            for (int i = 0; i < keys.size(); i++) {
                Tuple key = keys.get(i).getLeft();
                FDBRecordVersion version = keys.get(i).getRight();
                byte[] versionBytes = context.ensureActive().get(subspace.pack(key.add(SplitHelper.RECORD_VERSION))).join();
                if (i % 3 == 0 || splitLongRecords) {
                    assertNotNull(versionBytes);
                    FDBRecordVersion deserializedVersion = SplitHelper.unpackVersion(versionBytes);
                    assertEquals(version, deserializedVersion);
                } else {
                    assertNull(versionBytes);
                }
            }
        }
    }
}

Example 60 with Tuple

use of com.apple.foundationdb.tuple.Tuple in project fdb-record-layer by FoundationDB.

the class OnlineIndexerBuildVersionIndexTest method versionRebuild.

private void versionRebuild(@Nonnull List<TestRecords1Proto.MySimpleRecord> records, @Nullable List<TestRecords1Proto.MySimpleRecord> recordsWhileBuilding, int agents, boolean overlap) {
    final Index index = new Index("newVersionIndex", concat(field("num_value_2"), VersionKeyExpression.VERSION), IndexTypes.VERSION);
    final Function<FDBQueriedRecord<Message>, Tuple> projection = rec -> {
        TestRecords1Proto.MySimpleRecord simple = TestRecords1Proto.MySimpleRecord.newBuilder().mergeFrom(rec.getRecord()).build();
        Integer numValue2 = (simple.hasNumValue2()) ? simple.getNumValue2() : null;
        FDBRecordVersion version = rec.hasVersion() ? rec.getVersion() : null;
        if (version != null) {
            assertTrue(version.isComplete());
        }
        return Tuple.from(numValue2, (version == null) ? null : version.toVersionstamp());
    };
    List<RecordQuery> queries = records.stream().map(record -> {
        Integer value2 = (record.hasNumValue2()) ? record.getNumValue2() : null;
        return RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(value2 != null ? Query.field("num_value_2").equalsValue(record.getNumValue2()) : Query.field("num_value_2").isNull()).setSort(VersionKeyExpression.VERSION).build();
    }).collect(Collectors.toList());
    Function<TestRecords1Proto.MySimpleRecord, Integer> indexValue = msg -> msg.hasNumValue2() ? msg.getNumValue2() : null;
    Map<Integer, List<Message>> valueMap = group(records, indexValue);
    Map<Long, FDBRecordVersion> versionMap = new HashMap<>(records.size() + (recordsWhileBuilding == null ? 0 : recordsWhileBuilding.size()));
    AtomicReference<FDBRecordVersion> greatestVersion = new AtomicReference<>(null);
    final Runnable beforeBuild = () -> {
        try (FDBRecordContext context = openContext()) {
            for (int i = 0; i < queries.size(); i++) {
                Integer value2 = (records.get(i).hasNumValue2()) ? records.get(i).getNumValue2() : null;
                try {
                    executeQuery(queries.get(i), "Index(newVersionIndex [[" + value2 + "],[" + value2 + "])", valueMap.get(value2));
                    fail("somehow executed query with new index before build");
                } catch (RecordCoreException e) {
                    assertEquals("Cannot sort without appropriate index: Version", e.getMessage());
                }
            }
            // Load all the version information for the records that were initially there.
            for (TestRecords1Proto.MySimpleRecord simple : records) {
                recordStore.loadRecordVersion(Tuple.from(simple.getRecNo())).ifPresent(version -> {
                    versionMap.put(simple.getRecNo(), version);
                    if (greatestVersion.get() == null || version.compareTo(greatestVersion.get()) > 0) {
                        greatestVersion.set(version);
                    }
                });
            }
            context.commit();
        }
    };
    List<TestRecords1Proto.MySimpleRecord> updatedRecords;
    List<RecordQuery> updatedQueries;
    Map<Integer, List<Message>> updatedValueMap;
    if (recordsWhileBuilding == null || recordsWhileBuilding.size() == 0) {
        updatedRecords = records;
        updatedQueries = queries;
        updatedValueMap = valueMap;
    } else {
        updatedRecords = updated(records, recordsWhileBuilding);
        updatedQueries = updatedRecords.stream().map(record -> {
            Integer value2 = (record.hasNumValue2()) ? record.getNumValue2() : null;
            return RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(value2 != null ? Query.field("num_value_2").equalsValue(record.getNumValue2()) : Query.field("num_value_2").isNull()).setSort(VersionKeyExpression.VERSION).build();
        }).collect(Collectors.toList());
        updatedValueMap = group(updatedRecords, indexValue);
    }
    Map<Long, FDBRecordVersion> updatedVersionMap = new HashMap<>(versionMap.size());
    Set<Long> newRecordKeys = (recordsWhileBuilding == null) ? Collections.emptySet() : recordsWhileBuilding.stream().map(TestRecords1Proto.MySimpleRecord::getRecNo).collect(Collectors.toSet());
    Runnable afterBuild = new Runnable() {

        @SuppressWarnings("try")
        @Override
        public void run() {
            try (FDBRecordContext context = openContext()) {
                // The build job shouldn't affect the reads.
                for (int i = 0; i < updatedQueries.size(); i++) {
                    Integer value2 = (updatedRecords.get(i).hasNumValue2()) ? updatedRecords.get(i).getNumValue2() : null;
                    try {
                        executeQuery(updatedQueries.get(i), "Index(newVersionIndex [[" + value2 + "],[" + value2 + "])", updatedValueMap.get(value2));
                        fail("somehow executed query with new index before readable");
                    } catch (RecordCoreException e) {
                        assertEquals("Cannot sort without appropriate index: Version", e.getMessage());
                    }
                }
                // Load all the version information for records that are there now and that values are sane.
                for (TestRecords1Proto.MySimpleRecord simple : updatedRecords) {
                    recordStore.loadRecordVersion(Tuple.from(simple.getRecNo())).ifPresent(version -> {
                        assertTrue(version.isComplete());
                        if (newRecordKeys.contains(simple.getRecNo())) {
                            assertThat(version, greaterThan(greatestVersion.get()));
                            if (versionMap.containsKey(simple.getRecNo())) {
                                assertThat(version, greaterThan(versionMap.get(simple.getRecNo())));
                            }
                        } else {
                            if (versionMap.containsKey(simple.getRecNo())) {
                                assertEquals(versionMap.get(simple.getRecNo()), version);
                            }
                        }
                        updatedVersionMap.put(simple.getRecNo(), version);
                    });
                }
            }
        }
    };
    Runnable afterReadable = () -> {
        Descriptors.FieldDescriptor recNoFieldDescriptor = TestRecords1Proto.MySimpleRecord.getDescriptor().findFieldByName("rec_no");
        try (FDBRecordContext context = openContext()) {
            for (int i = 0; i < updatedQueries.size(); i++) {
                Integer value2 = (updatedRecords.get(i).hasNumValue2()) ? updatedRecords.get(i).getNumValue2() : null;
                List<Tuple> sortedValues = updatedValueMap.get(value2).stream().map(msg -> {
                    FDBRecordVersion version = updatedVersionMap.get(((Number) msg.getField(recNoFieldDescriptor)).longValue());
                    return Tuple.from(value2, version == null ? null : version.toVersionstamp());
                }).sorted().collect(Collectors.toList());
                executeQuery(updatedQueries.get(i), "Index(newVersionIndex [[" + value2 + "],[" + value2 + "]])", sortedValues, projection);
            }
            context.commit();
        }
    };
    singleRebuild(records, recordsWhileBuilding, agents, overlap, false, index, beforeBuild, afterBuild, afterReadable);
}