Examples with RangeResponse com.ibm.etcd.api.RangeResponse used on opensource projects

Example 6 with RangeResponse

use of com.ibm.etcd.api.RangeResponse in project etcd-java by IBM.

the class RangeCache method strongIterator.

/**
 * Iterator whose contents is guaranteed to be sequentially consistent
 * with remote updates to the cached range.
 *
 * @return an {@link Iterator} over the {@link KeyValue}s of this cache
 */
public Iterator<KeyValue> strongIterator() {
    long seenUpTo = seenUpToRev.get();
    if (seenUpTo == 0L) {
        ListenableFuture<Boolean> startFut;
        synchronized (this) {
            startFut = startFuture;
        }
        if (startFut == null) {
            // cache has not yet been started
            return kvClient.get(fromKey).rangeEnd(toKey).timeout(120_000L).sync().getKvsList().iterator();
        } else
            try {
                startFut.get(2L, TimeUnit.MINUTES);
                // now started
                seenUpTo = seenUpToRev.get();
            } catch (TimeoutException te) {
                throw Status.DEADLINE_EXCEEDED.asRuntimeException();
            } catch (ExecutionException e) {
                throw Status.UNKNOWN.withCause(e).asRuntimeException();
            } catch (InterruptedException | CancellationException e) {
                throw Status.CANCELLED.withCause(e).asRuntimeException();
            }
    }
    /* 
         * This logic is similar to that in fullRefreshCache(), but
         * it includes an optimistic initial comparison of counts
         * to identify cases where no deletions have been missed and
         * thus a retrieval of all the keys isn't required.
         */
    RangeRequest.Builder rangeReqBld = RangeRequest.newBuilder().setKey(fromKey).setRangeEnd(toKey);
    RangeRequest curCountReq = rangeReqBld.setCountOnly(true).setMaxCreateRevision(seenUpTo).build();
    RangeRequest seenCountReq = rangeReqBld.clearMaxCreateRevision().setRevision(seenUpTo).build();
    RangeRequest newModsReq = rangeReqBld.clearRevision().clearCountOnly().setMinModRevision(seenUpTo + 1).build();
    // first, attempt to get:
    // 0- kvs modified since seenUpTo
    // 1- current count excluding those created since seenUpTo
    // 2- count at revision seenUpTo (this could potentially
    // fail with compaction error, see below)
    TxnResponse txnResp;
    try {
        txnResp = kvClient.batch().get(newModsReq).get(curCountReq).get(seenCountReq).timeout(8000L).sync();
    } catch (RuntimeException e) {
        Code code = Status.fromThrowable(e).getCode();
        if (code != Code.OUT_OF_RANGE) {
            throw e;
        }
        // if (2) above fails due to compaction, also retrieve all current keys
        RangeRequest otherKeysReq = rangeReqBld.clearMinModRevision().setMaxModRevision(seenUpTo).setKeysOnly(true).build();
        txnResp = kvClient.batch().get(newModsReq).get(otherKeysReq).timeout(60_000L).sync();
    }
    long revNow = txnResp.getHeader().getRevision();
    if (revNow > seenUpToRev.get()) {
        RangeResponse newModKvs = txnResp.getResponses(0).getResponseRange();
        List<KeyValue> otherKeys;
        if (txnResp.getResponsesCount() == 2) {
            // this means we must have taken the compacted exception path above
            otherKeys = txnResp.getResponses(1).getResponseRange().getKvsList();
        } else if (// <- latest count
        txnResp.getResponses(1).getResponseRange().getCount() < txnResp.getResponses(2).getResponseRange().getCount()) {
            // <- count at seenUpTo
            // if counts don't match, there must have been deletions since seenUpTo,
            // so additionally retrieve all current keys
            RangeRequest otherKeysReq = rangeReqBld.clearMinModRevision().setMaxModRevision(seenUpTo).setKeysOnly(true).build();
            // longer timeout
            otherKeys = waitFor(kvClient.get(otherKeysReq), 60_000L).getKvsList();
        } else {
            otherKeys = null;
        }
        boolean newKvs = newModKvs.getKvsCount() > 0;
        if (otherKeys != null) {
            // if this is true, there *might* be deletions to process
            if (otherKeys.isEmpty() && !newKvs) {
                return Collections.emptyIterator();
            }
            // bring cache up to date with recently deleted kvs
            Set<ByteString> keys = Stream.concat(otherKeys.stream(), newModKvs.getKvsList().stream()).map(kv -> kv.getKey()).collect(Collectors.toSet());
            entries.values().stream().filter(kv -> kv.getModRevision() < revNow && !keys.contains(kv.getKey())).forEach(kv -> offerDelete(kv.getKey(), revNow));
        }
        // bring cache up to date with recently modified kvs
        if (newKvs) {
            newModKvs.getKvsList().forEach(kv -> offerUpdate(kv, false));
        }
        if (revNow > seenUpToRev.get()) {
            listenerExecutor.execute(() -> revisionUpdate(revNow));
        }
    }
    return iterator();
}

Example 7 with RangeResponse

use of com.ibm.etcd.api.RangeResponse in project etcd-java by IBM.

the class RangeCache method setupWatch.

// called only from listenerExecutor context
private void setupWatch(List<RangeResponse> rrs, boolean firstTime, SettableFuture<Boolean> promise) {
    if (closed) {
        throw new CancellationException();
    }
    Set<ByteString> snapshot = firstTime && entries.isEmpty() ? null : new HashSet<>();
    RangeResponse toUpdate = rrs.get(0);
    if (toUpdate.getKvsCount() > 0) {
        for (KeyValue kv : toUpdate.getKvsList()) {
            if (snapshot != null) {
                snapshot.add(kv.getKey());
            }
            offerUpdate(kv, true);
        }
    }
    long snapshotRev = toUpdate.getHeader().getRevision();
    if (firstTime) {
        notifyListeners(EventType.INITIALIZED, null, true);
    }
    if (snapshot != null) {
        if (rrs.size() > 1) {
            for (KeyValue kv : rrs.get(1).getKvsList()) {
                snapshot.add(kv.getKey());
            }
        }
        // prune deleted entries
        KeyValue.Builder kvBld = null;
        for (ByteString key : entries.keySet()) {
            if (!snapshot.contains(key)) {
                if (kvBld == null) {
                    kvBld = KeyValue.newBuilder().setVersion(0L).setModRevision(snapshotRev);
                }
                offerUpdate(kvBld.setKey(key).build(), true);
            }
        }
    }
    revisionUpdate(snapshotRev);
    StreamObserver<WatchUpdate> watchObserver = new StreamObserver<WatchUpdate>() {

        @Override
        public void onNext(WatchUpdate update) {
            List<Event> events = update.getEvents();
            int eventCount = events != null ? events.size() : 0;
            if (eventCount > 0) {
                for (Event event : events) {
                    KeyValue kv = event.getKv();
                    // event.getPrevKv(); //TBD
                    switch(event.getType()) {
                        case DELETE:
                            if (kv.getVersion() != 0L) {
                                kv = KeyValue.newBuilder(kv).setVersion(0L).clearValue().build();
                            }
                        // fall-thru
                        case PUT:
                            offerUpdate(kv, true);
                            break;
                        case UNRECOGNIZED:
                        default:
                            logger.warn("Unrecognized event for key " + kv.getKey().toStringUtf8());
                            break;
                    }
                }
            }
            revisionUpdate(eventCount == 0 ? update.getHeader().getRevision() - 1L : events.get(eventCount - 1).getKv().getModRevision());
        }

        @Override
        public void onCompleted() {
            // should only happen after external close()
            if (!closed) {
                if (!client.isClosed()) {
                    logger.warn("Watch completed unexpectedly (not closed) (fromKey = " + fromKey.toStringUtf8() + ")");
                }
                close();
            }
        }

        @Override
        public void onError(Throwable t) {
            if (closed) {
                promise.setException(new CancellationException());
                return;
            }
            assert startFuture == promise;
            boolean isDone = promise.isDone();
            if (isDone && promise.isCancelled()) {
                return;
            }
            if (!(t instanceof RevisionCompactedException)) {
                logger.error("Watch failed with exception (fromKey = " + fromKey.toStringUtf8() + ")", t);
                promise.setException(t);
                return;
            }
            // Refresh the cache, which will renew the watch
            ListenableFuture<Boolean> refresh;
            if (isDone) {
                refresh = fullRefreshCache();
            } else {
                // If this is a watch creation failure, delay the attempt for 1 second
                refresh = Futures.scheduleAsync(RangeCache.this::fullRefreshCache, 1L, TimeUnit.SECONDS, client.internalScheduledExecutor());
                if (promise.setFuture(refresh)) {
                    refresh = null;
                }
            }
            synchronized (RangeCache.this) {
                if (!closed) {
                    if (refresh != null) {
                        startFuture = refresh;
                        refresh = null;
                    }
                    watch = null;
                }
            }
            if (refresh == null) {
                logger.warn("Performing full refresh (fromKey = " + fromKey.toStringUtf8() + ") following watch compaction error: " + t);
            } else {
                assert closed;
                refresh.cancel(false);
            }
        }
    };
    FluentWatchRequest watchRequest = // .prevKv() //TODO TBD
    kvClient.watch(fromKey).rangeEnd(toKey).progressNotify().startRevision(snapshotRev + 1).executor(listenerExecutor);
    Watch newWatch;
    synchronized (RangeCache.this) {
        if (closed) {
            throw new CancellationException();
        }
        if (promise.isCancelled()) {
            return;
        }
        watch = newWatch = watchRequest.start(watchObserver);
    }
    Futures.addCallback(newWatch, (FutureListener<Boolean>) (v, t) -> {
        if (t != null && !newWatch.isCancelled()) {
            // Error cases are handled by onError above
            return;
        }
        if (!Boolean.TRUE.equals(v) && closed) {
            promise.setException(new CancellationException());
        } else {
            promise.set(v);
        }
    }, directExecutor());
}

Example 8 with RangeResponse

use of com.ibm.etcd.api.RangeResponse in project etcd-java by IBM.

the class KvTest method testKvOps.

@Test
public void testKvOps() throws Exception {
    proxy.start();
    try (KvStoreClient directClient = EtcdClient.forEndpoint("localhost", 2379).withPlainText().build();
        KvStoreClient client = EtcdClient.forEndpoint("localhost", 2391).withPlainText().build()) {
        KvClient kvc = client.getKvClient();
        ByteString a = bs("a"), b = bs("b"), v1 = bs("v1"), v2 = bs("v2");
        // clean up first
        kvc.batch().delete(kvc.delete(a).asRequest()).delete(kvc.delete(b).asRequest()).sync();
        // basic put
        assertEquals(KeyValue.getDefaultInstance(), kvc.put(a, v2).prevKv().sync().getPrevKv());
        assertTrue(kvc.put(a, v2).sync().getHeader().getRevision() > 0);
        assertEquals(v2, kvc.put(a, v1).prevKv().sync().getPrevKv().getValue());
        // put with non-existent leaseId
        try {
            kvc.put(b, v1, 12345L).sync();
            fail("lease should not exist");
        } catch (StatusRuntimeException sre) {
            assertEquals(Code.NOT_FOUND, sre.getStatus().getCode());
        }
        // basic get
        RangeResponse rr = kvc.get(bs("a")).sync();
        assertEquals(1L, rr.getCount());
        assertEquals(v1, rr.getKvs(0).getValue());
        // basic delete
        assertEquals(0L, kvc.delete(bs("notthere")).sync().getDeleted());
        assertEquals(v1, kvc.delete(a).prevKv().sync().getPrevKvs(0).getValue());
        assertEquals(0, kvc.get(bs("a")).sync().getCount());
        PutRequest pr1 = kvc.put(a, v1).asRequest(), pr2 = kvc.put(b, v2).asRequest();
        // batch put
        assertEquals(2, kvc.batch().put(pr1).put(pr2).sync().getResponsesCount());
        assertEquals(v1, kvc.get(a).sync().getKvs(0).getValue());
        assertEquals(v2, kvc.get(b).sync().getKvs(0).getValue());
        // basic transaction
        ListenableFuture<TxnResponse> tresp = kvc.txnIf().cmpEqual(a).value(v1).and().cmpNotEqual(b).version(10).then().put(kvc.put(bs("new"), bs("newval")).asRequest()).async();
        assertNotNull(tresp.get().getResponses(0).getResponsePut().getHeader());
        // test disconnected behaviour
        proxy.kill();
        Thread.sleep(200L);
        // should fail
        ListenableFuture<RangeResponse> rrFut1 = kvc.get(bs("new")).async();
        ListenableFuture<RangeResponse> rrFut2 = kvc.get(bs("new")).deadline(Deadline.after(20, TimeUnit.SECONDS)).backoffRetry().async();
        try {
            rrFut1.get(1000, TimeUnit.SECONDS);
            fail("expected get to fail while disconnected");
        } catch (Exception e) {
            // TODO
            System.out.println("failed with: " + e);
        }
        // this one should still be retrying
        assertFalse(rrFut2.isDone());
        // reconnect
        proxy.start();
        // should succeed once network path is there again
        long before = System.nanoTime();
        RangeResponse rr2 = rrFut2.get(2000, TimeUnit.SECONDS);
        long took = (System.nanoTime() - before) / 1000_000L;
        assertEquals(bs("newval"), rr2.getKvs(0).getValue());
        System.out.println("took " + took + "ms after network was reestablished");
    } finally {
        proxy.close();
    }
}

Example 9 with RangeResponse

use of com.ibm.etcd.api.RangeResponse in project SORMAS-Project by hzi-braunschweig.

the class EtcdCentralClient method get.

public <T> T get(String key, Class<T> clazz) throws IOException {
    // use resource auto-close
    try (KvStoreClient etcdClient = createEtcdClient()) {
        if (etcdClient == null) {
            LOGGER.error("Etcd could not be accessed.");
            return null;
        }
        KvClient etcd = etcdClient.getKvClient();
        if (etcd == null) {
            LOGGER.error("Could not create an etcd KV client.");
            return null;
        }
        RangeResponse range = etcd.get(ByteString.copyFromUtf8(key)).sync();
        if (range.getCount() == 0) {
            LOGGER.error("There is no value available for key {}", key);
            return null;
        }
        return deserialize(range.getKvs(0), clazz);
    }
}