Examples with Versioned org.apache.bookkeeper.versioning.Versioned used on opensource projects

Example 41 with Versioned

use of org.apache.bookkeeper.versioning.Versioned in project bookkeeper by apache.

the class MetadataAsyncIteratorTest method testOneLedgerErrorsOnRead.

@Test
public void testOneLedgerErrorsOnRead() throws Exception {
    MockLedgerManager lm = new MockLedgerManager() {

        @Override
        public CompletableFuture<Versioned<LedgerMetadata>> readLedgerMetadata(long ledgerId) {
            if (ledgerId == 403) {
                return FutureUtils.exception(new BKException.ZKException());
            } else {
                return super.readLedgerMetadata(ledgerId);
            }
        }
    };
    ConcurrentHashMap<Long, LedgerMetadata> added = addLedgers(lm, 10000);
    MetadataAsyncIterator iterator = new MetadataAsyncIterator(Schedulers.io(), lm, 100, /* inflight */
    3, /* timeout */
    TimeUnit.SECONDS);
    try {
        iterator.forEach((ledgerId, metadata) -> removeFromMap(added, ledgerId, metadata)).get(10, TimeUnit.SECONDS);
    } catch (ExecutionException ee) {
        assertThat(ee.getCause(), instanceOf(BKException.ZKException.class));
    }
}

Example 42 with Versioned

use of org.apache.bookkeeper.versioning.Versioned in project bookkeeper by apache.

the class EtcdLedgerManagerTest method testRegisterLedgerMetadataListener.

@Test
public void testRegisterLedgerMetadataListener() throws Exception {
    long ledgerId = System.currentTimeMillis();
    // create a ledger metadata
    LedgerMetadata metadata = LedgerMetadataBuilder.create().withId(ledgerId).withEnsembleSize(3).withWriteQuorumSize(3).withAckQuorumSize(2).withPassword("test-password".getBytes(UTF_8)).withDigestType(DigestType.CRC32C.toApiDigestType()).newEnsembleEntry(0L, createNumBookies(3)).build();
    result(lm.createLedgerMetadata(ledgerId, metadata));
    Versioned<LedgerMetadata> readMetadata = lm.readLedgerMetadata(ledgerId).get();
    log.info("Create ledger metadata : {}", readMetadata.getValue());
    // register first listener
    LinkedBlockingQueue<Versioned<LedgerMetadata>> metadataQueue1 = new LinkedBlockingQueue<>();
    LedgerMetadataListener listener1 = (lid, m) -> {
        log.info("[listener1] Received ledger {} metadata : {}", lid, m);
        metadataQueue1.add(m);
    };
    log.info("Registered first listener for ledger {}", ledgerId);
    lm.registerLedgerMetadataListener(ledgerId, listener1);
    // we should receive a metadata notification when a ledger is created
    Versioned<LedgerMetadata> notifiedMetadata = metadataQueue1.take();
    assertEquals(readMetadata, notifiedMetadata);
    ValueStream<LedgerMetadata> lms = lm.getLedgerMetadataStream(ledgerId);
    assertNotNull(lms.waitUntilWatched());
    assertNotNull(result(lms.waitUntilWatched()));
    // register second listener
    LinkedBlockingQueue<Versioned<LedgerMetadata>> metadataQueue2 = new LinkedBlockingQueue<>();
    LedgerMetadataListener listener2 = (lid, m) -> {
        log.info("[listener2] Received ledger {} metadata : {}", lid, m);
        metadataQueue2.add(m);
    };
    log.info("Registered second listener for ledger {}", ledgerId);
    lm.registerLedgerMetadataListener(ledgerId, listener2);
    Versioned<LedgerMetadata> notifiedMetadata2 = metadataQueue2.take();
    assertEquals(readMetadata, notifiedMetadata2);
    assertNotNull(lm.getLedgerMetadataStream(ledgerId));
    // update the metadata
    lm.writeLedgerMetadata(ledgerId, LedgerMetadataBuilder.from(metadata).newEnsembleEntry(10L, createNumBookies(3)).build(), notifiedMetadata.getVersion()).get();
    readMetadata = lm.readLedgerMetadata(ledgerId).get();
    assertEquals(readMetadata, metadataQueue1.take());
    assertEquals(readMetadata, metadataQueue2.take());
    lms = lm.getLedgerMetadataStream(ledgerId);
    assertNotNull(lms);
    assertEquals(2, lms.getNumConsumers());
    // remove listener2
    lm.unregisterLedgerMetadataListener(ledgerId, listener2);
    lms = lm.getLedgerMetadataStream(ledgerId);
    assertNotNull(lms);
    assertEquals(1, lms.getNumConsumers());
    // update the metadata again
    lm.writeLedgerMetadata(ledgerId, LedgerMetadataBuilder.from(metadata).newEnsembleEntry(20L, createNumBookies(3)).build(), readMetadata.getVersion()).get();
    readMetadata = lm.readLedgerMetadata(ledgerId).get();
    assertEquals(readMetadata, metadataQueue1.take());
    assertNull(metadataQueue2.poll());
    // remove listener1
    lm.unregisterLedgerMetadataListener(ledgerId, listener1);
    // the value stream will be removed
    while (lm.getLedgerMetadataStream(ledgerId) != null) {
        TimeUnit.MILLISECONDS.sleep(100);
    }
    assertEquals(0, lms.getNumConsumers());
    // update the metadata again
    lm.writeLedgerMetadata(ledgerId, LedgerMetadataBuilder.from(metadata).newEnsembleEntry(30L, createNumBookies(3)).build(), readMetadata.getVersion()).get();
    readMetadata = lm.readLedgerMetadata(ledgerId).get();
    assertNull(metadataQueue1.poll());
    assertNull(metadataQueue2.poll());
    log.info("Registered first listener for ledger {} again", ledgerId);
    lm.registerLedgerMetadataListener(ledgerId, listener1);
    notifiedMetadata = metadataQueue1.take();
    assertEquals(readMetadata, notifiedMetadata);
    lms = lm.getLedgerMetadataStream(ledgerId);
    assertNotNull(lms);
    assertEquals(1, lms.getNumConsumers());
    // delete the ledger
    lm.removeLedgerMetadata(ledgerId, readMetadata.getVersion()).get();
    // the listener will eventually be removed
    while (lm.getLedgerMetadataStream(ledgerId) != null) {
        TimeUnit.MILLISECONDS.sleep(100);
    }
    assertEquals(1, lms.getNumConsumers());
    assertNull(metadataQueue1.poll());
    assertNull(metadataQueue2.poll());
}

Example 43 with Versioned

use of org.apache.bookkeeper.versioning.Versioned in project bookkeeper by apache.

the class EtcdRegistrationTest method testWatchBookies.

private void testWatchBookies(boolean readonly) throws Exception {
    LinkedBlockingQueue<Versioned<Set<BookieId>>> writableChanges = new LinkedBlockingQueue<>();
    LinkedBlockingQueue<Versioned<Set<BookieId>>> readonlyChanges = new LinkedBlockingQueue<>();
    result(regClient.watchReadOnlyBookies(newRegistrationListener(readonlyChanges)));
    result(regClient.watchWritableBookies(newRegistrationListener(writableChanges)));
    Versioned<Set<BookieId>> versionedBookies = writableChanges.take();
    assertTrue(versionedBookies.getValue().isEmpty());
    versionedBookies = readonlyChanges.take();
    assertTrue(versionedBookies.getValue().isEmpty());
    final int numBookies = 3;
    final List<EtcdRegistrationManager> bookies = createNumBookies(readonly, numBookies, scope, 1);
    LinkedBlockingQueue<Versioned<Set<BookieId>>> changes;
    if (readonly) {
        changes = readonlyChanges;
    } else {
        changes = writableChanges;
    }
    Version preVersion = new LongVersion(-1);
    Set<BookieId> expectedBookies = new HashSet<>();
    for (int i = 0; i < numBookies; i++) {
        BookieId address = newBookie(i);
        expectedBookies.add(address);
        versionedBookies = changes.take();
        Version curVersion = versionedBookies.getVersion();
        assertEquals(Occurred.AFTER, curVersion.compare(preVersion));
        assertEquals(expectedBookies, versionedBookies.getValue());
        preVersion = curVersion;
    }
    bookies.forEach(EtcdRegistrationManager::close);
    for (int i = 0; i < numBookies; i++) {
        versionedBookies = changes.take();
        Version curVersion = versionedBookies.getVersion();
        assertEquals(Occurred.AFTER, curVersion.compare(preVersion));
        assertEquals(numBookies - i - 1, versionedBookies.getValue().size());
        preVersion = curVersion;
    }
    if (readonly) {
        assertEquals(0, writableChanges.size());
    } else {
        assertEquals(0, readonlyChanges.size());
    }
}

Example 44 with Versioned

use of org.apache.bookkeeper.versioning.Versioned in project bookkeeper by apache.

the class KeySetReaderTest method testWatchKeySet.

@Test
public void testWatchKeySet() throws Exception {
    String prefix = RandomStringUtils.randomAlphabetic(16);
    ByteSequence beginKeyBs = ByteSequence.from(prefix + "-000", StandardCharsets.UTF_8);
    ByteSequence endKeyBs = ByteSequence.from(prefix + "-999", StandardCharsets.UTF_8);
    KeySetReader<String> ksReader = null;
    try {
        ksReader = new KeySetReader<>(etcdClient, BYTE_SEQUENCE_STRING_FUNCTION, beginKeyBs, endKeyBs);
        LinkedBlockingQueue<Versioned<Set<String>>> notifications = new LinkedBlockingQueue<>();
        Consumer<Versioned<Set<String>>> keyConsumer = consumeVersionedKeySet(notifications);
        // key not exists
        Versioned<Set<String>> versionedKeys = FutureUtils.result(ksReader.readAndWatch(keyConsumer));
        assertTrue("VersionedKeys : " + versionedKeys, ((LongVersion) versionedKeys.getVersion()).getLongVersion() > 0L);
        assertEquals(0, versionedKeys.getValue().size());
        assertTrue(ksReader.isWatcherSet());
        // keys should be cached
        assertEquals(versionedKeys, ksReader.getLocalValue());
        Versioned<Set<String>> newVersionedKey = notifications.take();
        assertEquals(Occurred.CONCURRENTLY, newVersionedKey.getVersion().compare(versionedKeys.getVersion()));
        assertEquals(versionedKeys, newVersionedKey);
        versionedKeys = newVersionedKey;
        Set<String> expectedKeySet = new HashSet<>();
        for (int i = 0; i < 20; i++) {
            // update a value
            String key = String.format("%s-%03d", prefix, i);
            String value = RandomStringUtils.randomAlphabetic(32);
            ByteSequence keyBs = ByteSequence.from(key, StandardCharsets.UTF_8);
            ByteSequence valueBs = ByteSequence.from(value, StandardCharsets.UTF_8);
            expectedKeySet.add(key);
            FutureUtils.result(etcdClient.getKVClient().put(keyBs, valueBs));
            // we should get notified with updated key set
            newVersionedKey = notifications.take();
            assertEquals(Occurred.AFTER, newVersionedKey.getVersion().compare(versionedKeys.getVersion()));
            assertEquals(expectedKeySet, newVersionedKey.getValue());
            // local value should be changed
            assertEquals(newVersionedKey, ksReader.getLocalValue());
            versionedKeys = newVersionedKey;
        }
        for (int i = 0; i < 20; i++) {
            // delete the key
            String key = String.format("%s-%03d", prefix, i);
            ByteSequence keyBs = ByteSequence.from(key, StandardCharsets.UTF_8);
            expectedKeySet.remove(key);
            FutureUtils.result(etcdClient.getKVClient().delete(keyBs));
            // we should get notified with updated key set
            newVersionedKey = notifications.take();
            assertEquals(Occurred.AFTER, newVersionedKey.getVersion().compare(versionedKeys.getVersion()));
            assertEquals(expectedKeySet, newVersionedKey.getValue());
            // local value should be changed
            assertEquals(newVersionedKey, ksReader.getLocalValue());
            versionedKeys = newVersionedKey;
        }
    } finally {
        if (null != ksReader) {
            ksReader.close();
        }
    }
    assertNotNull(ksReader);
    assertFalse(ksReader.isWatcherSet());
}

Example 45 with Versioned

use of org.apache.bookkeeper.versioning.Versioned in project bookkeeper by apache.

the class KeySetReaderTest method testWatchSingleKeyWithTTL.

@Test
public void testWatchSingleKeyWithTTL() throws Exception {
    String key = RandomStringUtils.randomAlphabetic(16);
    ByteSequence keyBs = ByteSequence.from(key, StandardCharsets.UTF_8);
    KeySetReader<String> ksReader = null;
    try {
        ksReader = new KeySetReader<>(etcdClient, BYTE_SEQUENCE_STRING_FUNCTION, keyBs, null);
        LinkedBlockingQueue<Versioned<Set<String>>> notifications = new LinkedBlockingQueue<>();
        Consumer<Versioned<Set<String>>> keyConsumer = consumeVersionedKeySet(notifications);
        // key not exists
        Versioned<Set<String>> versionedKeys = FutureUtils.result(ksReader.readAndWatch(keyConsumer));
        assertTrue("VersionedKeys : " + versionedKeys, ((LongVersion) versionedKeys.getVersion()).getLongVersion() > 0L);
        assertEquals(0, versionedKeys.getValue().size());
        assertTrue(ksReader.isWatcherSet());
        // keys should be cached
        assertEquals(versionedKeys, ksReader.getLocalValue());
        // no watch event should be issued
        Versioned<Set<String>> newVersionedKey = notifications.take();
        assertEquals(Occurred.CONCURRENTLY, newVersionedKey.getVersion().compare(versionedKeys.getVersion()));
        assertEquals(versionedKeys, newVersionedKey);
        versionedKeys = newVersionedKey;
        // create a key with ttl
        long leaseId = FutureUtils.result(etcdClient.getLeaseClient().grant(1)).getID();
        String value = RandomStringUtils.randomAlphabetic(32);
        ByteSequence valueBs = ByteSequence.from(value, StandardCharsets.UTF_8);
        FutureUtils.result(etcdClient.getKVClient().put(keyBs, valueBs, PutOption.newBuilder().withLeaseId(leaseId).build()));
        // we should get notified with updated key set
        newVersionedKey = notifications.take();
        assertEquals(Occurred.AFTER, newVersionedKey.getVersion().compare(versionedKeys.getVersion()));
        assertEquals(1, newVersionedKey.getValue().size());
        assertEquals(Sets.newHashSet(key), newVersionedKey.getValue());
        // local value should be changed
        assertEquals(newVersionedKey, ksReader.getLocalValue());
        versionedKeys = newVersionedKey;
        // the key will be deleted after TTL
        newVersionedKey = notifications.take();
        assertEquals(Occurred.AFTER, newVersionedKey.getVersion().compare(versionedKeys.getVersion()));
        assertEquals(0, newVersionedKey.getValue().size());
        // local value should be changed
        assertEquals(newVersionedKey, ksReader.getLocalValue());
    } finally {
        if (null != ksReader) {
            ksReader.close();
        }
    }
    assertNotNull(ksReader);
    assertFalse(ksReader.isWatcherSet());
}