Examples with ByteString org.apache.beam.vendor.grpc.v1p26p0.com.google.protobuf.ByteString used on opensource projects

Example 11 with ByteString

use of org.apache.beam.vendor.grpc.v1p26p0.com.google.protobuf.ByteString in project heron by twitter.

the class BoltOutputCollectorImpl method admitBoltTuple.

/////////////////////////////////////////////////////////
// Following private methods are internal implementations
/////////////////////////////////////////////////////////
private List<Integer> admitBoltTuple(String streamId, Collection<Tuple> anchors, List<Object> tuple) {
    // First check whether this tuple is sane
    helper.checkOutputSchema(streamId, tuple);
    // customGroupingTargetTaskIds will be null if this stream is not CustomStreamGrouping
    List<Integer> customGroupingTargetTaskIds = helper.chooseTasksForCustomStreamGrouping(streamId, tuple);
    // Invoke user-defined emit task hook
    helper.getTopologyContext().invokeHookEmit(tuple, streamId, customGroupingTargetTaskIds);
    // Start construct the data tuple
    HeronTuples.HeronDataTuple.Builder bldr = HeronTuples.HeronDataTuple.newBuilder();
    // set the key. This is mostly ignored
    bldr.setKey(0);
    if (customGroupingTargetTaskIds != null) {
        // It is a CustomStreamGrouping
        for (Integer taskId : customGroupingTargetTaskIds) {
            bldr.addDestTaskIds(taskId);
        }
    }
    // Set the anchors for a tuple
    if (anchors != null) {
        // This message is rooted
        Set<HeronTuples.RootId> mergedRoots = new HashSet<HeronTuples.RootId>();
        for (Tuple tpl : anchors) {
            if (tpl instanceof TupleImpl) {
                TupleImpl t = (TupleImpl) tpl;
                mergedRoots.addAll(t.getRoots());
            }
        }
        for (HeronTuples.RootId rt : mergedRoots) {
            bldr.addRoots(rt);
        }
    }
    long tupleSizeInBytes = 0;
    long startTime = System.nanoTime();
    // Serialize it
    for (Object obj : tuple) {
        byte[] b = serializer.serialize(obj);
        ByteString bstr = ByteString.copyFrom(b);
        bldr.addValues(bstr);
        tupleSizeInBytes += b.length;
    }
    long latency = System.nanoTime() - startTime;
    boltMetrics.serializeDataTuple(streamId, latency);
    // submit to outputter
    outputter.addDataTuple(streamId, bldr, tupleSizeInBytes);
    // Update metrics
    boltMetrics.emittedTuple(streamId);
    // TODO:- remove this after changing the api
    return null;
}

Example 12 with ByteString

use of org.apache.beam.vendor.grpc.v1p26p0.com.google.protobuf.ByteString in project heron by twitter.

the class SpoutOutputCollectorImpl method admitSpoutTuple.

/////////////////////////////////////////////////////////
// Following private methods are internal implementations
/////////////////////////////////////////////////////////
private List<Integer> admitSpoutTuple(String streamId, List<Object> tuple, Object messageId) {
    // First check whether this tuple is sane
    helper.checkOutputSchema(streamId, tuple);
    // customGroupingTargetTaskIds will be null if this stream is not CustomStreamGrouping
    List<Integer> customGroupingTargetTaskIds = helper.chooseTasksForCustomStreamGrouping(streamId, tuple);
    // Invoke user-defined emit task hook
    helper.getTopologyContext().invokeHookEmit(tuple, streamId, customGroupingTargetTaskIds);
    // Start construct the data tuple
    HeronTuples.HeronDataTuple.Builder bldr = HeronTuples.HeronDataTuple.newBuilder();
    // set the key. This is mostly ignored
    bldr.setKey(0);
    if (customGroupingTargetTaskIds != null) {
        // It is a CustomStreamGrouping
        for (Integer taskId : customGroupingTargetTaskIds) {
            bldr.addDestTaskIds(taskId);
        }
    }
    if (messageId != null) {
        RootTupleInfo tupleInfo = new RootTupleInfo(streamId, messageId);
        if (ackingEnabled) {
            // This message is rooted
            HeronTuples.RootId.Builder rtbldr = EstablishRootId(tupleInfo);
            bldr.addRoots(rtbldr);
        } else {
            immediateAcks.offer(tupleInfo);
        }
    }
    long tupleSizeInBytes = 0;
    long startTime = System.nanoTime();
    // Serialize it
    for (Object obj : tuple) {
        byte[] b = serializer.serialize(obj);
        ByteString bstr = ByteString.copyFrom(b);
        bldr.addValues(bstr);
        tupleSizeInBytes += b.length;
    }
    long latency = System.nanoTime() - startTime;
    spoutMetrics.serializeDataTuple(streamId, latency);
    // submit to outputter
    outputter.addDataTuple(streamId, bldr, tupleSizeInBytes);
    totalTuplesEmitted++;
    spoutMetrics.emittedTuple(streamId);
    // TODO:- remove this after changing the api
    return null;
}

Example 13 with ByteString

use of org.apache.beam.vendor.grpc.v1p26p0.com.google.protobuf.ByteString in project beam by apache.

the class BoundedSourceRunnerTest method testStart.

@Test
public void testStart() throws Exception {
    List<WindowedValue<Long>> outValues = new ArrayList<>();
    Map<String, Collection<ThrowingConsumer<WindowedValue<Long>>>> outputMap = ImmutableMap.of("out", ImmutableList.of(outValues::add));
    ByteString encodedSource = ByteString.copyFrom(SerializableUtils.serializeToByteArray(CountingSource.upTo(3)));
    BoundedSourceRunner<BoundedSource<Long>, Long> runner = new BoundedSourceRunner<>(PipelineOptionsFactory.create(), BeamFnApi.FunctionSpec.newBuilder().setData(Any.pack(BytesValue.newBuilder().setValue(encodedSource).build())).build(), outputMap);
    runner.start();
    assertThat(outValues, contains(valueInGlobalWindow(0L), valueInGlobalWindow(1L), valueInGlobalWindow(2L)));
}

Example 14 with ByteString

use of org.apache.beam.vendor.grpc.v1p26p0.com.google.protobuf.ByteString in project phoenix by apache.

the class MetaDataEndpointImpl method getFunctions.

@Override
public void getFunctions(RpcController controller, GetFunctionsRequest request, RpcCallback<MetaDataResponse> done) {
    MetaDataResponse.Builder builder = MetaDataResponse.newBuilder();
    byte[] tenantId = request.getTenantId().toByteArray();
    List<String> functionNames = new ArrayList<>(request.getFunctionNamesCount());
    try {
        Region region = env.getRegion();
        List<ByteString> functionNamesList = request.getFunctionNamesList();
        List<Long> functionTimestampsList = request.getFunctionTimestampsList();
        List<byte[]> keys = new ArrayList<byte[]>(request.getFunctionNamesCount());
        List<Pair<byte[], Long>> functions = new ArrayList<Pair<byte[], Long>>(request.getFunctionNamesCount());
        for (int i = 0; i < functionNamesList.size(); i++) {
            byte[] functionName = functionNamesList.get(i).toByteArray();
            functionNames.add(Bytes.toString(functionName));
            byte[] key = SchemaUtil.getFunctionKey(tenantId, functionName);
            MetaDataMutationResult result = checkFunctionKeyInRegion(key, region);
            if (result != null) {
                done.run(MetaDataMutationResult.toProto(result));
                return;
            }
            functions.add(new Pair<byte[], Long>(functionName, functionTimestampsList.get(i)));
            keys.add(key);
        }
        long currentTime = EnvironmentEdgeManager.currentTimeMillis();
        List<PFunction> functionsAvailable = doGetFunctions(keys, request.getClientTimestamp());
        if (functionsAvailable == null) {
            builder.setReturnCode(MetaDataProtos.MutationCode.FUNCTION_NOT_FOUND);
            builder.setMutationTime(currentTime);
            done.run(builder.build());
            return;
        }
        builder.setReturnCode(MetaDataProtos.MutationCode.FUNCTION_ALREADY_EXISTS);
        builder.setMutationTime(currentTime);
        for (PFunction function : functionsAvailable) {
            builder.addFunction(PFunction.toProto(function));
        }
        done.run(builder.build());
        return;
    } catch (Throwable t) {
        logger.error("getFunctions failed", t);
        ProtobufUtil.setControllerException(controller, ServerUtil.createIOException(functionNames.toString(), t));
    }
}

Example 15 with ByteString

use of org.apache.beam.vendor.grpc.v1p26p0.com.google.protobuf.ByteString in project bitcask-java by krestenkrab.

the class BitCask method scan_key_files.

static void scan_key_files(File[] files, final BitCaskKeyDir keydir) throws Exception {
    for (File f : files) {
        final BitCaskFile file = BitCaskFile.open(f);
        file.fold_keys(new KeyIter<Void>() {

            @Override
            public Void each(ByteString key, int tstamp, long entryPos, int entrySize, Void acc) throws Exception {
                keydir.put(key, new BitCaskEntry(file.file_id, tstamp, entryPos, entrySize));
                return null;
            }
        }, null);
        file.close();
    }
}