Examples with Transaction com.apple.foundationdb.Transaction used on opensource projects

Example 51 with Transaction

use of com.apple.foundationdb.Transaction in project fdb-record-layer by FoundationDB.

the class KeySpaceDirectoryTest method testListObeysReturnedRowAndScanLimits.

@Test
public void testListObeysReturnedRowAndScanLimits() {
    KeySpace root = new KeySpace(new KeySpaceDirectory("root", KeyType.STRING, "root-" + random.nextInt(Integer.MAX_VALUE)).addSubdirectory(new KeySpaceDirectory("a", KeyType.LONG).addSubdirectory(new KeySpaceDirectory("b", KeyType.LONG))));
    final FDBDatabase database = FDBDatabaseFactory.instance().getDatabase();
    try (FDBRecordContext context = database.openContext()) {
        Transaction tr = context.ensureActive();
        for (int i = 0; i < 5; i++) {
            for (int j = 0; j < 2; j++) {
                tr.set(root.path("root").add("a", i).add("b", j).toTuple(context).pack(), Tuple.from(i + j).pack());
            }
        }
        tr.commit().join();
    }
    doLimitedScan(database, root, 5, Integer.MAX_VALUE, RecordCursor.NoNextReason.RETURN_LIMIT_REACHED);
    doLimitedScan(database, root, Integer.MAX_VALUE, 5, RecordCursor.NoNextReason.SCAN_LIMIT_REACHED);
}

Example 52 with Transaction

use of com.apple.foundationdb.Transaction in project fdb-record-layer by FoundationDB.

the class KeySpaceDirectoryTest method testDeleteAllDataAndHasData.

@Test
public void testDeleteAllDataAndHasData() throws Exception {
    KeySpace root = new KeySpace(new KeySpaceDirectory("root", KeyType.LONG, Math.abs(random.nextLong())).addSubdirectory(new KeySpaceDirectory("dir1", KeyType.STRING, "a").addSubdirectory(new KeySpaceDirectory("dir1_1", KeyType.LONG))).addSubdirectory(new KeySpaceDirectory("dir2", KeyType.STRING, "b").addSubdirectory(new KeySpaceDirectory("dir2_1", KeyType.LONG))).addSubdirectory(new KeySpaceDirectory("dir3", KeyType.STRING, "c").addSubdirectory(new KeySpaceDirectory("dir3_1", KeyType.LONG))));
    final FDBDatabase database = FDBDatabaseFactory.instance().getDatabase();
    try (FDBRecordContext context = database.openContext()) {
        Transaction tr = context.ensureActive();
        for (int i = 0; i < 5; i++) {
            tr.set(root.path("root").add("dir1").add("dir1_1", i).toTuple(context).pack(), Tuple.from(i).pack());
            tr.set(root.path("root").add("dir2").add("dir2_1", i).toTuple(context).pack(), Tuple.from(i).pack());
            tr.set(root.path("root").add("dir3").add("dir3_1", i).toTuple(context).pack(), Tuple.from(i).pack());
        }
        context.commit();
    }
    try (FDBRecordContext context = database.openContext()) {
        // All directories hava data?
        for (int i = 1; i <= 3; i++) {
            assertTrue(root.path("root").add("dir" + i).hasData(context), "dir" + i + " is empty!");
        }
        // Clear out dir2
        root.path("root").add("dir2").deleteAllData(context);
        context.commit();
    }
    try (FDBRecordContext context = database.openContext()) {
        assertTrue(root.path("root").add("dir1").hasData(context), "dir1 is empty!");
        assertFalse(root.path("root").add("dir2").hasData(context), "dir2 has data!");
        assertTrue(root.path("root").add("dir3").hasData(context), "dir3 is empty!");
    }
}

Example 53 with Transaction

use of com.apple.foundationdb.Transaction in project fdb-record-layer by FoundationDB.

the class KeySpaceDirectoryTest method testListPreservesWrapper.

@Test
public void testListPreservesWrapper() {
    KeySpace keySpace = new KeySpace(new KeySpaceDirectory("a", KeyType.STRING, PathA::new).addSubdirectory(new KeySpaceDirectory("b", KeyType.STRING, PathB::new)));
    final FDBDatabase database = FDBDatabaseFactory.instance().getDatabase();
    try (FDBRecordContext context = database.openContext()) {
        Transaction tr = context.ensureActive();
        PathA root = (PathA) keySpace.path("a", "foo");
        tr.set(root.add("b", "one").toTuple(context).pack(), TupleHelpers.EMPTY.pack());
        tr.set(root.add("b", "two").toTuple(context).pack(), TupleHelpers.EMPTY.pack());
        tr.set(root.add("b", "three").toTuple(context).pack(), TupleHelpers.EMPTY.pack());
        tr.commit();
    }
    try (FDBRecordContext context = database.openContext()) {
        List<ResolvedKeySpacePath> paths = keySpace.path("a", "foo").listSubdirectory(context, "b");
        for (ResolvedKeySpacePath path : paths) {
            assertThat("Path should be PathB", path, instanceOf(PathB.class));
            assertThat("parent should be PathA", path.getParent(), instanceOf(PathA.class));
        }
    }
}

Example 54 with Transaction

use of com.apple.foundationdb.Transaction in project fdb-record-layer by FoundationDB.

the class KeySpaceDirectoryTest method testListAcrossTransactions.

@Test
public void testListAcrossTransactions() throws Exception {
    KeySpace root = new KeySpace(new KeySpaceDirectory("a", KeyType.LONG, random.nextLong()).addSubdirectory(new KeySpaceDirectory("b", KeyType.STRING)));
    final FDBDatabase database = FDBDatabaseFactory.instance().getDatabase();
    final List<String> directoryEntries = IntStream.range(0, 10).boxed().map(i -> "val_" + i).collect(Collectors.toList());
    final KeySpacePath rootPath = root.path("a");
    try (final FDBRecordContext context = database.openContext()) {
        final Transaction tr = context.ensureActive();
        directoryEntries.forEach(name -> tr.set(rootPath.add("b", name).toTuple(context).pack(), TupleHelpers.EMPTY.pack()));
        context.commit();
    }
    byte[] continuation = null;
    int idx = 0;
    do {
        try (final FDBRecordContext context = database.openContext()) {
            final RecordCursor<ResolvedKeySpacePath> cursor = rootPath.listSubdirectoryAsync(context, "b", continuation, new ScanProperties(ExecuteProperties.newBuilder().setReturnedRowLimit(2).build()));
            List<ResolvedKeySpacePath> subdirs = context.asyncToSync(FDBStoreTimer.Waits.WAIT_KEYSPACE_LIST, cursor.asList());
            if (!subdirs.isEmpty()) {
                assertEquals(2, subdirs.size(), "Wrong number of path entries returned");
                assertEquals("val_" + idx, subdirs.get(0).getResolvedValue());
                assertEquals("val_" + (idx + 1), subdirs.get(1).getResolvedValue());
                idx += 2;
                continuation = cursor.getNext().getContinuation().toBytes();
                System.out.println(continuation == null ? "null" : Tuple.fromBytes(continuation));
            } else {
                continuation = cursor.getNext().getContinuation().toBytes();
                assertNull(continuation);
            }
        }
    } while (continuation != null);
    assertEquals(directoryEntries.size(), idx);
}

Example 55 with Transaction

use of com.apple.foundationdb.Transaction in project fdb-record-layer by FoundationDB.

the class BunchedMapTest method stressTest.

private void stressTest(final Random r, final int trTotal, final int opTotal, final int keyCount, final int workerCount, boolean addBytesToValue, AtomicLong globalTrCount, int mapCount) throws InterruptedException, ExecutionException {
    final long initialTrCount = globalTrCount.get();
    final Subspace logSubspace = DirectoryLayer.getDefault().createOrOpen(db, PathUtil.from(getClass().getName(), "log")).get();
    db.run(tr -> {
        tr.clear(bmSubspace.range());
        tr.clear(logSubspace.range());
        // If the database is empty, putting these here stop scans from hitting the log subspace within a transaction
        tr.set(logSubspace.getKey(), new byte[0]);
        tr.set(ByteArrayUtil.join(logSubspace.getKey(), new byte[] { (byte) 0xff }), new byte[0]);
        return null;
    });
    final List<CompletableFuture<Void>> workers = Stream.generate(() -> {
        int bunchSize = r.nextInt(15) + 1;
        BunchedMap<Tuple, Tuple> workerMap = new BunchedMap<>(serializer, Comparator.naturalOrder(), bunchSize);
        AtomicInteger trCount = new AtomicInteger(0);
        return AsyncUtil.whileTrue(() -> {
            final Transaction tr = db.createTransaction();
            tr.options().setDebugTransactionIdentifier("stress-tr-" + globalTrCount.getAndIncrement());
            tr.options().setLogTransaction();
            final AtomicInteger opCount = new AtomicInteger(0);
            final AtomicInteger localOrder = new AtomicInteger(0);
            return AsyncUtil.whileTrue(() -> {
                int opCode = r.nextInt(4);
                CompletableFuture<?> op;
                if (opCode == 0) {
                    // Random put
                    CompletableFuture<?>[] futures = new CompletableFuture<?>[mapCount];
                    for (int i = 0; i < mapCount; i++) {
                        if (r.nextBoolean()) {
                            Tuple key = Tuple.from(r.nextInt(keyCount));
                            Tuple value;
                            if (addBytesToValue) {
                                int byteLength = r.nextInt(5000);
                                byte[] bytes = new byte[byteLength];
                                r.nextBytes(bytes);
                                value = Tuple.from(r.nextLong(), bytes);
                            } else {
                                value = Tuple.from(r.nextLong());
                            }
                            tr.mutate(MutationType.SET_VERSIONSTAMPED_KEY, getLogKey(logSubspace, i, localOrder), Tuple.from("PUT", key, value).pack());
                            futures[i] = workerMap.put(tr, bmSubspace.subspace(Tuple.from(i)), key, value);
                        } else {
                            futures[i] = AsyncUtil.DONE;
                        }
                    }
                    op = CompletableFuture.allOf(futures);
                } else if (opCode == 1) {
                    // Read a random key.
                    int mapIndex = r.nextInt(mapCount);
                    Tuple key = Tuple.from(r.nextInt(keyCount));
                    op = workerMap.get(tr, bmSubspace.get(mapIndex), key).thenAccept(optionalValue -> tr.mutate(MutationType.SET_VERSIONSTAMPED_KEY, getLogKey(logSubspace, mapIndex, localOrder), Tuple.from("GET", key, optionalValue.orElse(null)).pack()));
                } else if (opCode == 2) {
                    // Check contains key
                    int mapIndex = r.nextInt(mapCount);
                    Tuple key = Tuple.from(r.nextInt(keyCount));
                    op = workerMap.containsKey(tr, bmSubspace.subspace(Tuple.from(mapIndex)), key).thenAccept(wasPresent -> tr.mutate(MutationType.SET_VERSIONSTAMPED_KEY, getLogKey(logSubspace, mapIndex, localOrder), Tuple.from("CONTAINS_KEY", key, wasPresent).pack()));
                } else {
                    // Remove a random key
                    int mapIndex = r.nextInt(mapCount);
                    Tuple key = Tuple.from(r.nextInt(keyCount));
                    op = workerMap.remove(tr, bmSubspace.subspace(Tuple.from(mapIndex)), key).thenAccept(oldValue -> tr.mutate(MutationType.SET_VERSIONSTAMPED_KEY, getLogKey(logSubspace, mapIndex, localOrder), Tuple.from("REMOVE", key, oldValue.orElse(null)).pack()));
                }
                return op.thenApply(ignore -> opCount.incrementAndGet() < opTotal);
            }).thenCompose(vignore -> tr.commit()).handle((vignore, err) -> {
                tr.close();
                if (err != null) {
                    FDBException fdbE = unwrapException(err);
                    if (fdbE != null) {
                        if (fdbE.getCode() != FDBError.NOT_COMMITTED.code() && fdbE.getCode() != FDBError.TRANSACTION_TOO_OLD.code()) {
                            throw fdbE;
                        }
                    } else {
                        if (err instanceof RuntimeException) {
                            throw (RuntimeException) err;
                        } else {
                            throw new RuntimeException("verification error", err);
                        }
                    }
                }
                return trCount.incrementAndGet() < trTotal;
            });
        });
    }).limit(workerCount).collect(Collectors.toList());
    final AtomicBoolean stillWorking = new AtomicBoolean(true);
    final CompletableFuture<Void> verifierWorker = AsyncUtil.whileTrue(() -> {
        Transaction tr = db.createTransaction();
        AtomicLong versionRef = new AtomicLong(-1L);
        return tr.getReadVersion().thenCompose(version -> {
            versionRef.set(version);
            // Grab the mutation list.
            AtomicInteger mapIndex = new AtomicInteger(0);
            return AsyncUtil.whileTrue(() -> {
                Subspace mapSubspace = bmSubspace.subspace(Tuple.from(mapIndex.get()));
                Subspace mapLogSubspace = logSubspace.subspace(Tuple.from(mapIndex.get()));
                CompletableFuture<List<Tuple>> logFuture = AsyncUtil.mapIterable(tr.getRange(mapLogSubspace.range()), kv -> Tuple.fromBytes(kv.getValue())).asList();
                // Verify integrity and then grab all of the keys and values.
                CompletableFuture<List<Map.Entry<Tuple, Tuple>>> contentFuture = AsyncUtil.collectRemaining(map.scan(tr, mapSubspace));
                CompletableFuture<Void> integrityFuture = map.verifyIntegrity(tr, mapSubspace);
                return integrityFuture.thenCompose(vignore -> contentFuture.thenCombine(logFuture, (mapContents, logEntries) -> {
                    Map<Tuple, Tuple> mapCopy = new TreeMap<>();
                    for (Tuple logEntry : logEntries) {
                        String op = logEntry.getString(0);
                        if (op.equals("PUT")) {
                            mapCopy.put(logEntry.getNestedTuple(1), logEntry.getNestedTuple(2));
                        } else if (op.equals("GET")) {
                            assertEquals(logEntry.getNestedTuple(2), mapCopy.get(logEntry.getNestedTuple(1)));
                        } else if (op.equals("CONTAINS_KEY")) {
                            assertEquals(logEntry.getBoolean(2), mapCopy.containsKey(logEntry.getNestedTuple(1)));
                        } else if (op.equals("REMOVE")) {
                            Tuple oldValue = mapCopy.remove(logEntry.getNestedTuple(1));
                            assertEquals(logEntry.getNestedTuple(2), oldValue);
                        } else {
                            fail("Unexpected operation " + op);
                        }
                    }
                    assertEquals(new ArrayList<>(mapCopy.entrySet()), mapContents);
                    return mapIndex.incrementAndGet() < mapCount;
                })).handle((res, err) -> {
                    // Report error information unless it was just a transaction timeout (in which case we'll retry).
                    FDBException fdbE = unwrapException(err);
                    if (err != null && (fdbE == null || fdbE.getCode() != FDBError.TRANSACTION_TOO_OLD.code())) {
                        System.err.println("Error verifying consistency: " + err);
                        err.printStackTrace();
                        List<Map.Entry<Tuple, Tuple>> contents = contentFuture.join();
                        System.err.println("Map contents:");
                        contents.forEach(entry -> System.err.println("  " + entry.getKey() + " -> " + entry.getValue()));
                        System.err.println("DB contents:");
                        List<KeyValue> rangeKVs = tr.getRange(bmSubspace.range()).asList().join();
                        rangeKVs.forEach(kv -> {
                            Tuple boundaryKey = bmSubspace.unpack(kv.getKey());
                            System.err.println("  " + boundaryKey + " -> " + serializer.deserializeEntries(boundaryKey, kv.getValue()));
                        });
                        List<Tuple> logEntries = logFuture.join();
                        System.err.println("Log contents:");
                        logEntries.forEach(logEntry -> System.err.println(" " + logEntry));
                        if (err instanceof RuntimeException) {
                            throw (RuntimeException) err;
                        } else {
                            throw new LoggableException("unable to complete consistency check", err);
                        }
                    }
                    return res;
                });
            });
        }).whenComplete((v, t) -> tr.close()).thenApply(vignore -> stillWorking.get());
    });
    AtomicInteger mapIndex = new AtomicInteger(0);
    CompletableFuture<Void> compactingWorker = AsyncUtil.whileTrue(() -> {
        AtomicReference<byte[]> continuation = new AtomicReference<>(null);
        return AsyncUtil.whileTrue(() -> map.compact(db, bmSubspace.subspace(Tuple.from(mapIndex.get())), 5, continuation.get()).thenApply(nextContinuation -> {
            continuation.set(nextContinuation);
            return nextContinuation != null;
        })).thenApply(vignore -> {
            mapIndex.getAndUpdate(oldIndex -> (oldIndex + 1) % mapCount);
            return stillWorking.get();
        });
    });
    // Wait for all workers to stop working.
    AsyncUtil.whenAll(workers).whenComplete((vignore, err) -> stillWorking.set(false)).thenAcceptBoth(verifierWorker, (vignore1, vignore2) -> {
    }).thenAcceptBoth(compactingWorker, (vignore1, vignore2) -> {
    }).whenComplete((vignore, err) -> {
        System.out.printf("Completed stress test with %d workers, %d keys, and %d transactions %s (large values=%s).%n", workerCount, keyCount, globalTrCount.get() - initialTrCount, (err == null ? "successfully" : "with an error"), addBytesToValue);
        if (err != null) {
            err.printStackTrace();
        }
        for (int i = 0; i < mapCount; i++) {
            System.out.println(" Map " + i + ":");
            Subspace mapSubspace = bmSubspace.subspace(Tuple.from(i));
            List<KeyValue> rangeKVs = inconsistentScan(db, mapSubspace);
            System.out.println("  Boundary keys: " + rangeKVs.stream().map(kv -> mapSubspace.unpack(kv.getKey())).collect(Collectors.toList()));
            System.out.println("  Boundary info:");
            rangeKVs.forEach(kv -> {
                Tuple boundaryKey = mapSubspace.unpack(kv.getKey());
                System.out.printf("    %s: %d - %s%n", boundaryKey, serializer.deserializeEntries(boundaryKey, kv.getValue()).size(), serializer.deserializeKeys(boundaryKey, kv.getValue()));
            });
        }
        int opsCount = inconsistentScan(db, logSubspace).size();
        System.out.println("  Committed ops: " + opsCount);
    }).get();
}