Examples with Tuple com.apple.foundationdb.tuple.Tuple used on opensource projects

Example 51 with Tuple

use of com.apple.foundationdb.tuple.Tuple in project fdb-record-layer by FoundationDB.

the class VersionIndexTest method validateUsingOlderVersionFormat.

private <M extends Message> void validateUsingOlderVersionFormat(@Nonnull List<FDBStoredRecord<M>> storedRecords) {
    // Make sure all of the records have versions in the old keyspace
    final Subspace legacyVersionSubspace = recordStore.getLegacyVersionSubspace();
    RecordCursorIterator<Pair<Tuple, FDBRecordVersion>> versionKeyPairs = KeyValueCursor.Builder.withSubspace(legacyVersionSubspace).setContext(recordStore.getRecordContext()).setScanProperties(ScanProperties.FORWARD_SCAN).build().map(kv -> Pair.of(legacyVersionSubspace.unpack(kv.getKey()), FDBRecordVersion.fromBytes(kv.getValue()))).asIterator();
    for (FDBStoredRecord<M> storedRecord : storedRecords) {
        assertTrue(versionKeyPairs.hasNext());
        Pair<Tuple, FDBRecordVersion> versionPair = versionKeyPairs.next();
        assertEquals(storedRecord.getPrimaryKey(), versionPair.getLeft());
        assertEquals(storedRecord.getVersion(), versionPair.getRight());
    }
    assertFalse(versionKeyPairs.hasNext());
    // Validate that no value in the record subspace begins with the type code for versionstamps
    final Subspace recordsSubspace = recordStore.recordsSubspace();
    KeyValueCursor.Builder.withSubspace(recordsSubspace).setContext(recordStore.getRecordContext()).setScanProperties(ScanProperties.FORWARD_SCAN).build().forEach(kv -> assertNotEquals(VERSIONSTAMP_CODE, kv.getValue()[0])).join();
}

Example 52 with Tuple

use of com.apple.foundationdb.tuple.Tuple in project fdb-record-layer by FoundationDB.

the class VersionIndexTest method upgradeFormatVersions.

@ParameterizedTest(name = "upgradeFormatVersions [formatVersion = {0}, splitLongRecords = {1}]")
@MethodSource("formatVersionArguments")
@SuppressWarnings("try")
public void upgradeFormatVersions(int testFormatVersion, boolean splitLongRecords) {
    formatVersion = testFormatVersion;
    final RecordMetaDataHook hook = metaDataBuilder -> {
        simpleVersionHook.apply(metaDataBuilder);
        metaDataBuilder.setSplitLongRecords(splitLongRecords);
    };
    final List<Message> records = Arrays.asList(MySimpleRecord.newBuilder().setRecNo(1066L).setNumValue2(1).build(), MySimpleRecord.newBuilder().setRecNo(1415L).setNumValue2(1).build(), MySimpleRecord.newBuilder().setRecNo(1776L).setNumValue2(2).build());
    List<FDBStoredRecord<Message>> storedRecords;
    try (FDBRecordContext context = openContext(hook)) {
        List<FDBStoredRecord<Message>> storedRecordsWithIncompletes = records.stream().map(recordStore::saveRecord).collect(Collectors.toList());
        context.commit();
        byte[] globalVersion = context.getVersionStamp();
        storedRecords = storedRecordsWithIncompletes.stream().map(record -> record.withCommittedVersion(globalVersion)).collect(Collectors.toList());
    }
    try (FDBRecordContext context = openContext(hook)) {
        if (testFormatVersion < FDBRecordStore.SAVE_VERSION_WITH_RECORD_FORMAT_VERSION) {
            validateUsingOlderVersionFormat(storedRecords);
        } else {
            validateUsingNewerVersionFormat(storedRecords);
        }
    }
    // Update to the current format version
    formatVersion = FDBRecordStore.MAX_SUPPORTED_FORMAT_VERSION;
    if (testFormatVersion < FDBRecordStore.SAVE_VERSION_WITH_RECORD_FORMAT_VERSION && (splitLongRecords || testFormatVersion >= FDBRecordStore.SAVE_UNSPLIT_WITH_SUFFIX_FORMAT_VERSION)) {
        // After format version upgrade, each record should now store that it has the version inlined
        storedRecords = storedRecords.stream().map(record -> new FDBStoredRecord<>(record.getPrimaryKey(), record.getRecordType(), record.getRecord(), record.getKeyCount() + 1, record.getKeySize() * 2 + 1, record.getValueSize() + 1 + FDBRecordVersion.VERSION_LENGTH, record.isSplit(), true, record.getVersion())).collect(Collectors.toList());
    }
    try (FDBRecordContext context = openContext(hook)) {
        if (!splitLongRecords && testFormatVersion < FDBRecordStore.SAVE_UNSPLIT_WITH_SUFFIX_FORMAT_VERSION) {
            validateUsingOlderVersionFormat(storedRecords);
        } else {
            validateUsingNewerVersionFormat(storedRecords);
        }
        for (FDBStoredRecord<Message> storedRecord : storedRecords) {
            Optional<FDBRecordVersion> loadedVersion = recordStore.loadRecordVersion(storedRecord.getPrimaryKey());
            assertTrue(loadedVersion.isPresent());
            assertEquals(storedRecord.getVersion(), loadedVersion.get());
            RecordQuery query = RecordQuery.newBuilder().setFilter(Query.version().equalsValue(storedRecord.getVersion())).build();
            RecordQueryPlan plan = planner.plan(query);
            final String endpointString = "[" + storedRecord.getVersion().toVersionstamp(false).toString() + "]";
            assertThat(plan, indexScan(allOf(indexName("globalVersion"), bounds(hasTupleString("[" + endpointString + "," + endpointString + "]")))));
            List<FDBStoredRecord<Message>> queriedRecords = recordStore.executeQuery(plan).map(FDBQueriedRecord::getStoredRecord).asList().join();
            assertEquals(Collections.singletonList(storedRecord), queriedRecords);
        }
        assertTrue(recordStore.deleteRecord(storedRecords.get(0).getPrimaryKey()));
        final List<FDBStoredRecord<Message>> fewerRecords = storedRecords.subList(1, storedRecords.size());
        if (!splitLongRecords && testFormatVersion < FDBRecordStore.SAVE_UNSPLIT_WITH_SUFFIX_FORMAT_VERSION) {
            validateUsingOlderVersionFormat(fewerRecords);
        } else {
            validateUsingNewerVersionFormat(fewerRecords);
        }
        recordStore.saveRecord(storedRecords.get(0).getRecord());
        if (!splitLongRecords && testFormatVersion < FDBRecordStore.SAVE_UNSPLIT_WITH_SUFFIX_FORMAT_VERSION) {
            validateUsingOlderVersionFormat(fewerRecords);
        } else {
            validateUsingNewerVersionFormat(fewerRecords);
        }
    // do not commit (so we can do a second upgrade)
    }
    final Index newValueIndex = new Index("MySimpleRecord$num2", field("num_value_2"), IndexTypes.VALUE);
    final Index newVersionIndex = new Index("MySimpleRecord$version-num2", concat(VersionKeyExpression.VERSION, field("num_value_2")), IndexTypes.VERSION);
    final RecordMetaDataHook hookWithNewIndexes = metaDataBuilder -> {
        hook.apply(metaDataBuilder);
        metaDataBuilder.addIndex("MySimpleRecord", newValueIndex);
        metaDataBuilder.addIndex("MySimpleRecord", newVersionIndex);
    };
    final List<FDBStoredRecord<Message>> newStoredRecords;
    try (FDBRecordContext context = openContext(hookWithNewIndexes)) {
        assertTrue(recordStore.getRecordStoreState().isReadable(newValueIndex));
        boolean performedMigration = testFormatVersion < FDBRecordStore.SAVE_VERSION_WITH_RECORD_FORMAT_VERSION && (splitLongRecords || testFormatVersion >= FDBRecordStore.SAVE_UNSPLIT_WITH_SUFFIX_FORMAT_VERSION);
        assertNotEquals(performedMigration, recordStore.getRecordStoreState().isReadable(newVersionIndex));
        if (recordStore.getRecordStoreState().isReadable(newVersionIndex)) {
            // Validate versions are the same for all records in index and in primary store
            List<Pair<Tuple, FDBRecordVersion>> recordScannedValues = recordStore.scanRecords(null, ScanProperties.FORWARD_SCAN).map(record -> Pair.of(record.getPrimaryKey(), record.getVersion())).asList().join();
            List<Pair<Tuple, FDBRecordVersion>> indexedScannedValues = recordStore.scanIndex(newVersionIndex, IndexScanType.BY_VALUE, TupleRange.ALL, null, ScanProperties.FORWARD_SCAN).map(indexEntry -> Pair.of(TupleHelpers.subTuple(indexEntry.getKey(), 2, indexEntry.getKey().size()), FDBRecordVersion.fromVersionstamp(indexEntry.getKey().getVersionstamp(0), false))).asList().join();
            assertEquals(recordScannedValues, indexedScannedValues);
        }
        // Save record at newer version
        assertTrue(recordStore.deleteRecord(storedRecords.get(0).getPrimaryKey()));
        FDBStoredRecord<Message> newRecord0 = recordStore.saveRecord(storedRecords.get(0).getRecord());
        FDBStoredRecord<Message> newRecord2 = recordStore.saveRecord(storedRecords.get(2).getRecord());
        assertEquals(newRecord0.getVersion(), recordStore.loadRecordVersion(newRecord0.getPrimaryKey()).get());
        assertEquals(newRecord2.getVersion(), recordStore.loadRecordVersion(newRecord2.getPrimaryKey()).get());
        context.commit();
        byte[] versionstamp = context.getVersionStamp();
        newStoredRecords = Arrays.asList(newRecord0.withVersion(FDBRecordVersion.complete(versionstamp, 0)), storedRecords.get(1), newRecord2.withVersion(FDBRecordVersion.complete(versionstamp, 1)));
    }
    try (FDBRecordContext context = openContext(hookWithNewIndexes)) {
        if (!splitLongRecords && testFormatVersion < FDBRecordStore.SAVE_UNSPLIT_WITH_SUFFIX_FORMAT_VERSION) {
            validateUsingOlderVersionFormat(newStoredRecords);
        } else {
            validateUsingNewerVersionFormat(newStoredRecords);
        }
    }
}

Example 53 with Tuple

use of com.apple.foundationdb.tuple.Tuple in project fdb-record-layer by FoundationDB.

the class TextIndexTest method saveComplexDocuments.

@Test
public void saveComplexDocuments() throws Exception {
    ComplexDocument complexDocument = ComplexDocument.newBuilder().setGroup(0).setDocId(1066L).setText("Very complex. Not to be trifled with.").build();
    ComplexDocument shakespeareDocument = ComplexDocument.newBuilder().setGroup(0).setDocId(1623L).setText(TextSamples.ROMEO_AND_JULIET_PROLOGUE).addTag("a").addTag("b").build();
    ComplexDocument yiddishDocument = ComplexDocument.newBuilder().setGroup(1).setDocId(1944L).setText(TextSamples.YIDDISH).addTag("c").build();
    try (FDBRecordContext context = openContext()) {
        openRecordStore(context, metaDataBuilder -> metaDataBuilder.addIndex(COMPLEX_DOC, COMPLEX_TEXT_BY_GROUP));
        recordStore.saveRecord(complexDocument);
        int firstKeys = getSaveIndexKeyCount(recordStore);
        assertEquals(complexDocument.getText().split(" ").length, firstKeys);
        recordStore.saveRecord(shakespeareDocument);
        int secondKeys = getSaveIndexKeyCount(recordStore) - firstKeys;
        assertEquals(82, secondKeys);
        recordStore.saveRecord(yiddishDocument);
        int thirdKeys = getSaveIndexKeyCount(recordStore) - secondKeys - firstKeys;
        assertEquals(9, thirdKeys);
        List<Map.Entry<Tuple, List<Integer>>> entryList = scanMapEntries(recordStore, COMPLEX_TEXT_BY_GROUP, Tuple.from(0L, "to"));
        assertEquals(Arrays.asList(entryOf(Tuple.from(1066L), Collections.singletonList(3)), entryOf(Tuple.from(1623L), Arrays.asList(18, 108))), entryList);
        List<Message> recordList = recordStore.scanIndexRecords(COMPLEX_TEXT_BY_GROUP.getName(), BY_TEXT_TOKEN, TupleRange.allOf(Tuple.from(0L, "to")), null, ScanProperties.FORWARD_SCAN).map(FDBIndexedRecord::getRecord).asList().get();
        assertEquals(Arrays.asList(complexDocument, shakespeareDocument), recordList);
        entryList = toMapEntries(scanIndex(recordStore, COMPLEX_TEXT_BY_GROUP, TupleRange.prefixedBy("א").prepend(Tuple.from(1L))), null);
        assertEquals(Arrays.asList(entryOf(Tuple.from(1L, "א", 1944L), Arrays.asList(0, 3)), entryOf(Tuple.from(1L, "און", 1944L), Collections.singletonList(8)), entryOf(Tuple.from(1L, "איז", 1944L), Collections.singletonList(2)), entryOf(Tuple.from(1L, "אן", 1944L), Collections.singletonList(6)), entryOf(Tuple.from(1L, "ארמיי", 1944L), Collections.singletonList(7))), entryList);
        // Read the whole store and make sure the values come back in a somewhat sensible way
        entryList = toMapEntries(scanIndex(recordStore, COMPLEX_TEXT_BY_GROUP, TupleRange.ALL), null);
        assertEquals(firstKeys + secondKeys + thirdKeys, entryList.size());
        int i = 0;
        String last = null;
        for (Map.Entry<Tuple, List<Integer>> entry : entryList) {
            assertEquals(3, entry.getKey().size());
            if (i < firstKeys + secondKeys) {
                assertEquals(0L, entry.getKey().getLong(0));
                assertThat(entry.getKey().getLong(2), anyOf(is(1066L), is(1623L)));
            } else {
                assertEquals(1L, entry.getKey().getLong(0));
                assertEquals(1944L, entry.getKey().getLong(2));
                if (i == firstKeys + secondKeys) {
                    last = null;
                }
            }
            if (last != null) {
                assertThat(entry.getKey().getString(1), greaterThanOrEqualTo(last));
            }
            last = entry.getKey().getString(1);
            i++;
        }
        commit(context);
    }
}

Example 54 with Tuple

use of com.apple.foundationdb.tuple.Tuple in project fdb-record-layer by FoundationDB.

the class GeophileSpatialJoin method getSpatialIndex.

@Nonnull
public SpatialIndex<GeophileRecordImpl> getSpatialIndex(@Nonnull String indexName, @Nonnull ScanComparisons prefixComparisons, @Nonnull BiFunction<IndexEntry, Tuple, GeophileRecordImpl> recordFunction) {
    if (!prefixComparisons.isEquality()) {
        throw new RecordCoreArgumentException("prefix comparisons must only have equality");
    }
    // TODO: Add a FDBRecordStoreBase.getIndexMaintainer String overload to do this.
    final IndexMaintainer indexMaintainer = store.getIndexMaintainer(store.getRecordMetaData().getIndex(indexName));
    final TupleRange prefixRange = prefixComparisons.toTupleRange(store, context);
    // Since this is an equality, will match getHigh(), too.
    final Tuple prefix = prefixRange.getLow();
    final Index<GeophileRecordImpl> index = new GeophileIndexImpl(indexMaintainer, prefix, recordFunction);
    final Space space = ((GeophileIndexMaintainer) indexMaintainer).getSpace();
    try {
        return SpatialIndex.newSpatialIndex(space, index);
    } catch (IOException ex) {
        throw new RecordCoreException("Unexpected IO exception", ex);
    } catch (InterruptedException ex) {
        Thread.currentThread().interrupt();
        throw new RecordCoreException(ex);
    }
}

Example 55 with Tuple

use of com.apple.foundationdb.tuple.Tuple in project fdb-record-layer by FoundationDB.

the class OnlineIndexerSimpleTest method buildRangeTransactional.

@Test
public void buildRangeTransactional() {
    List<TestRecords1Proto.MySimpleRecord> records = LongStream.range(0, 200).mapToObj(val -> TestRecords1Proto.MySimpleRecord.newBuilder().setRecNo(val).setNumValue2((int) val + 1).build()).collect(Collectors.toList());
    Index index = new Index("simple$value_2", field("num_value_2").ungrouped(), IndexTypes.SUM);
    IndexAggregateFunction aggregateFunction = new IndexAggregateFunction(FunctionNames.SUM, index.getRootExpression(), index.getName());
    List<String> indexTypes = Collections.singletonList("MySimpleRecord");
    FDBRecordStoreTestBase.RecordMetaDataHook hook = metaDataBuilder -> metaDataBuilder.addIndex("MySimpleRecord", index);
    openSimpleMetaData();
    try (FDBRecordContext context = openContext()) {
        records.forEach(recordStore::saveRecord);
        context.commit();
    }
    final Supplier<Tuple> getAggregate = () -> {
        Tuple ret;
        try (FDBRecordContext context = openContext()) {
            assertTrue(recordStore.uncheckedMarkIndexReadable(index.getName()).join());
            FDBRecordStore recordStore2 = recordStore.asBuilder().setContext(context).uncheckedOpen();
            ret = recordStore2.evaluateAggregateFunction(indexTypes, aggregateFunction, TupleRange.ALL, IsolationLevel.SERIALIZABLE).join();
        // Do NOT commit changes
        }
        return ret;
    };
    openSimpleMetaData(hook);
    try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder().setDatabase(fdb).setMetaData(metaData).setIndex(index).setSubspace(subspace).build()) {
        try (FDBRecordContext context = openContext()) {
            recordStore.markIndexWriteOnly(index).join();
            context.commit();
        }
        try (FDBRecordContext context = openContext()) {
            indexBuilder.buildRange(recordStore, null, null).join();
            indexBuilder.buildRange(recordStore, null, null).join();
            context.commit();
        }
        assertEquals(Tuple.from(20100L), getAggregate.get());
        clearIndexData(index);
        try (FDBRecordContext context = openContext()) {
            indexBuilder.buildRange(recordStore, null, Key.Evaluated.scalar(130L)).join();
            context.commit();
        }
        assertEquals(Tuple.from(8515L), getAggregate.get());
        try (FDBRecordContext context = openContext()) {
            indexBuilder.buildRange(recordStore, Key.Evaluated.scalar(100L), Key.Evaluated.scalar(130L)).join();
            context.commit();
        }
        assertEquals(Tuple.from(8515L), getAggregate.get());
        try (FDBRecordContext context = openContext()) {
            indexBuilder.buildRange(recordStore, Key.Evaluated.scalar(100L), Key.Evaluated.scalar(150L)).join();
            context.commit();
        }
        assertEquals(Tuple.from(11325L), getAggregate.get());
        try (FDBRecordContext context = openContext()) {
            indexBuilder.buildRange(recordStore, Key.Evaluated.scalar(100L), null).join();
            context.commit();
        }
        assertEquals(Tuple.from(20100L), getAggregate.get());
        clearIndexData(index);
        try (FDBRecordContext context = openContext()) {
            for (long l = 0L; l < 200L; l += 10) {
                indexBuilder.buildRange(recordStore, Key.Evaluated.scalar(l), Key.Evaluated.scalar(l + 5L)).join();
            }
            context.commit();
        }
        assertEquals(Tuple.from(9800L), getAggregate.get());
        try (FDBRecordContext context = openContext()) {
            for (long l = 0L; l < 200L; l += 10) {
                indexBuilder.buildRange(recordStore, null, null).join();
            }
            context.commit();
        }
        assertEquals(Tuple.from(20100L), getAggregate.get());
    }
}