Examples with ByteString org.apache.beam.vendor.grpc.v1p26p0.com.google.protobuf.ByteString used on opensource projects

Example 56 with ByteString

use of org.apache.beam.vendor.grpc.v1p26p0.com.google.protobuf.ByteString in project elephant-bird by twitter.

the class BinaryBlockWriter method serialize.

protected void serialize() throws IOException {
    out_.write(Protobufs.KNOWN_GOOD_POSITION_MARKER);
    Message block = SerializedBlock.newInstance(innerClass_.getCanonicalName(), protoBlobs_).getMessage();
    protoBlobs_ = new ArrayList<ByteString>(numRecordsPerBlock_);
    writeRawLittleEndian32(block.getSerializedSize());
    block.writeTo(out_);
}

Example 57 with ByteString

use of org.apache.beam.vendor.grpc.v1p26p0.com.google.protobuf.ByteString in project BIMserver by opensourceBIM.

the class ProtocolBuffersConverter method convertFieldValue.

private Object convertFieldValue(SField field, Object val) throws ConvertException {
    if (val instanceof EnumValueDescriptor) {
        EnumValueDescriptor enumValueDescriptor = (EnumValueDescriptor) val;
        Class<?> enumClass;
        try {
            enumClass = Class.forName("org.bimserver.interfaces.objects." + enumValueDescriptor.getType().getName());
            for (Object v : enumClass.getEnumConstants()) {
                Enum<?> e = (Enum<?>) v;
                if (e.ordinal() == enumValueDescriptor.getNumber()) {
                    val = e;
                    break;
                }
            }
        } catch (ClassNotFoundException e) {
            LOGGER.error("", e);
        }
        return val;
    } else if (field.getType().getInstanceClass() == Date.class) {
        return new Date((Long) val);
    } else if (field.getType().getInstanceClass() == DataHandler.class) {
        ByteString byteString = (ByteString) val;
        ByteArrayDataSource byteArrayDataSource = new ByteArrayDataSource("test", byteString.toByteArray());
        return new DataHandler(byteArrayDataSource);
    } else if (val instanceof DynamicMessage) {
        return convertProtocolBuffersMessageToSObject((DynamicMessage) val, null, field.isAggregate() ? field.getGenericType() : field.getType());
    } else {
        return val;
    }
}

Example 58 with ByteString

use of org.apache.beam.vendor.grpc.v1p26p0.com.google.protobuf.ByteString in project etcd-java by IBM.

the class LeaderElectionTest method basicLeaderElectionTest.

@Test
public void basicLeaderElectionTest() throws Exception {
    try (EtcdClient client1 = EtcdClient.forEndpoint("localhost", 2379).withPlainText().build()) {
        ByteString electionKey = bs("/electiontest");
        EtcdLeaderElection observer = new EtcdLeaderElection(client1, electionKey);
        observer.start();
        assertNull(observer.getId());
        assertNull(observer.getLeaderId());
        EtcdLeaderElection alice = new EtcdLeaderElection(client1, electionKey, "alice");
        EtcdLeaderElection bob = new EtcdLeaderElection(client1, electionKey, "bob");
        EtcdLeaderElection claire = new EtcdLeaderElection(client1, electionKey, "claire");
        bob.start();
        Thread.sleep(500L);
        assertEquals("bob", bob.getId());
        assertEquals("bob", bob.getLeaderId());
        assertEquals("bob", observer.getLeaderId());
        alice.start();
        Thread.sleep(500L);
        assertEquals("alice", alice.getId());
        assertEquals("bob", observer.getLeaderId());
        assertEquals("bob", bob.getLeaderId());
        assertEquals("bob", alice.getLeaderId());
        assertTrue(bob.isLeader());
        assertFalse(alice.isLeader());
        claire.start();
        Thread.sleep(500L);
        assertEquals("bob", claire.getLeaderId());
        assertEquals("bob", observer.getLeaderId());
        bob.close();
        Thread.sleep(500L);
        assertEquals("alice", observer.getLeaderId());
        assertTrue(alice.isLeader());
        assertFalse(bob.isLeader());
        assertFalse(claire.isLeader());
        assertEquals("alice", claire.getLeaderId());
    }
}

Example 59 with ByteString

use of org.apache.beam.vendor.grpc.v1p26p0.com.google.protobuf.ByteString in project etcd-java by IBM.

the class RangeCache method fullRefreshCache.

// internal method - should not be called while watch is active
protected ListenableFuture<Boolean> fullRefreshCache() {
    ListenableFuture<List<RangeResponse>> rrfut;
    long seenUpTo = seenUpToRev;
    boolean firstTime = (seenUpTo == 0L);
    if (firstTime || entries.size() <= 20) {
        // TODO *maybe* chunking (for large caches)
        ListenableFuture<RangeResponse> rrf = kvClient.get(fromKey).rangeEnd(toKey).backoffRetry(() -> !closed).timeout(300_000L).async();
        rrfut = Futures.transform(rrf, (Function<RangeResponse, List<RangeResponse>>) rr -> Collections.singletonList(rr));
    } else {
        // in case the local cache is large, reduce data transfer by requesting
        // just keys, and full key+value only for those modified since seenUpToRev
        RangeRequest.Builder rangeReqBld = RangeRequest.newBuilder().setKey(fromKey).setRangeEnd(toKey);
        RangeRequest newModsReq = rangeReqBld.setMinModRevision(seenUpTo + 1).build();
        RangeRequest otherKeysReq = rangeReqBld.clearMinModRevision().setMaxModRevision(seenUpTo).setKeysOnly(true).build();
        ListenableFuture<TxnResponse> trf = kvClient.batch().get(newModsReq).get(otherKeysReq).backoffRetry(() -> !closed).timeout(300_000L).async();
        rrfut = Futures.transform(trf, (Function<TxnResponse, List<RangeResponse>>) tr -> tr.getResponsesList().stream().map(r -> r.getResponseRange()).collect(Collectors.toList()));
    }
    return Futures.transformAsync(rrfut, rrs -> {
        if (closed)
            throw new CancellationException();
        Set<ByteString> snapshot = firstTime ? null : new HashSet<>();
        RangeResponse toUpdate = rrs.get(0);
        if (toUpdate.getKvsCount() > 0)
            for (KeyValue kv : toUpdate.getKvsList()) {
                if (!firstTime)
                    snapshot.add(kv.getKey());
                offerUpdate(kv, true);
            }
        long snapshotRev = toUpdate.getHeader().getRevision();
        if (firstTime)
            notifyListeners(EventType.INITIALIZED, null, true);
        else {
            if (rrs.size() > 1)
                for (KeyValue kv : rrs.get(1).getKvsList()) {
                    snapshot.add(kv.getKey());
                }
            // prune deleted entries
            KeyValue.Builder kvBld = null;
            for (ByteString key : entries.keySet()) if (!snapshot.contains(key)) {
                if (kvBld == null)
                    kvBld = KeyValue.newBuilder().setVersion(0L).setModRevision(snapshotRev);
                offerUpdate(kvBld.setKey(key).build(), true);
            }
        }
        revisionUpdate(snapshotRev);
        Watch newWatch = // .prevKv() //TODO TBD
        kvClient.watch(fromKey).rangeEnd(toKey).progressNotify().startRevision(snapshotRev + 1).executor(listenerExecutor).start(new StreamObserver<WatchUpdate>() {

            @Override
            public void onNext(WatchUpdate update) {
                List<Event> events = update.getEvents();
                int eventCount = events != null ? events.size() : 0;
                if (eventCount > 0)
                    for (Event event : events) {
                        KeyValue kv = event.getKv();
                        // event.getPrevKv(); //TBD
                        switch(event.getType()) {
                            case DELETE:
                                if (kv.getVersion() != 0L)
                                    kv = KeyValue.newBuilder(kv).setVersion(0L).clearValue().build();
                            // fall-thru
                            case PUT:
                                offerUpdate(kv, true);
                                break;
                            case UNRECOGNIZED:
                            default:
                                logger.warn("Unrecognized event for key " + kv.getKey().toStringUtf8());
                                break;
                        }
                    }
                revisionUpdate(eventCount == 0 ? update.getHeader().getRevision() - 1L : events.get(eventCount - 1).getKv().getModRevision());
            }

            @Override
            public void onCompleted() {
                // should only happen after external close()
                if (!closed) {
                    if (!client.isClosed()) {
                        logger.error("Watch completed unexpectedly (not closed)");
                    }
                    close();
                }
            }

            @Override
            public void onError(Throwable t) {
                logger.error("Watch failed with exception ", t);
                if (t instanceof RevisionCompactedException)
                    synchronized (RangeCache.this) {
                        // fail if happens during start, otherwise refresh
                        if (!closed && startFuture != null && startFuture.isDone()) {
                            // will renew watch
                            startFuture = fullRefreshCache();
                        }
                    }
            }
        });
        synchronized (this) {
            if (closed)
                throw new CancellationException();
            return watch = newWatch;
        }
    }, listenerExecutor);
}

Example 60 with ByteString

use of org.apache.beam.vendor.grpc.v1p26p0.com.google.protobuf.ByteString in project etcd-java by IBM.

the class RangeCache method offerUpdate.

/**
 * @param keyValue
 * @param watchThread if being called from background watch context
 * @return the provided value, <i>or a newer one</i>
 */
protected KeyValue offerUpdate(final KeyValue keyValue, boolean watchThread) {
    final long modRevision = keyValue.getModRevision();
    if (modRevision <= seenUpToRev)
        return kvOrNullIfDeleted(keyValue);
    final ByteString key = keyValue.getKey();
    final boolean isDeleted = isDeleted(keyValue);
    // a possible (but unlikely) race with deletions
    if (watchThread && !isDeleted) {
        // optimized non-deletion path
        KeyValue newKv = entries.merge(key, keyValue, (k, v) -> (modRevision > v.getModRevision() ? keyValue : v));
        if (newKv == keyValue)
            notifyListeners(EventType.UPDATED, keyValue, true);
        return kvOrNullIfDeleted(newKv);
    }
    KeyValue existKv = entries.get(key);
    while (true) {
        if (existKv != null) {
            long existModRevision = existKv.getModRevision();
            if (existModRevision >= modRevision)
                return kvOrNullIfDeleted(existKv);
            KeyValue newKv = entries.computeIfPresent(key, (k, v) -> (existModRevision == v.getModRevision() ? keyValue : v));
            if (newKv != keyValue) {
                existKv = newKv;
                // update failed
                continue;
            }
            // update succeeded
            if (isDeleted) {
                deletionQueue.add(keyValue);
                if (!isDeleted(existKv)) {
                    // previous value
                    notifyListeners(EventType.DELETED, existKv, watchThread);
                }
                return null;
            } else {
                // added or updated
                notifyListeners(EventType.UPDATED, keyValue, false);
                return keyValue;
            }
        }
        // here existKv == null
        if (modRevision <= seenUpToRev)
            return null;
        if ((existKv = entries.putIfAbsent(key, keyValue)) == null) {
            // update succeeded
            if (isDeleted) {
                deletionQueue.add(keyValue);
                return null;
            } else {
                notifyListeners(EventType.UPDATED, keyValue, false);
                return keyValue;
            }
        }
    }
}