Examples with Streamer io.crate.Streamer used on opensource projects

Example 6 with Streamer

use of io.crate.Streamer in project crate by crate.

the class JobSetup method registerContextPhases.

private void registerContextPhases(Iterable<? extends NodeOperation> nodeOperations, Context context) {
    for (NodeOperation nodeOperation : nodeOperations) {
        // context for nodeOperations without dependencies can be built immediately (e.g. FetchPhase)
        if (nodeOperation.downstreamExecutionPhaseId() == NodeOperation.NO_DOWNSTREAM) {
            LOGGER.trace("Building context for nodeOp without downstream: {}", nodeOperation);
            createContexts(nodeOperation.executionPhase(), context);
            context.opCtx.builtNodeOperations.set(nodeOperation.executionPhase().phaseId());
        }
        if (ExecutionPhases.hasDirectResponseDownstream(nodeOperation.downstreamNodes())) {
            var executionPhase = nodeOperation.executionPhase();
            CircuitBreaker breaker = breaker();
            int ramAccountingBlockSizeInBytes = BlockBasedRamAccounting.blockSizeInBytes(breaker.getLimit());
            var ramAccounting = new BlockBasedRamAccounting(b -> breaker.addEstimateBytesAndMaybeBreak(b, executionPhase.label()), ramAccountingBlockSizeInBytes);
            Streamer<?>[] streamers = StreamerVisitor.streamersFromOutputs(executionPhase);
            SingleBucketBuilder bucketBuilder = new SingleBucketBuilder(streamers, ramAccounting);
            context.directResponseFutures.add(bucketBuilder.completionFuture().whenComplete((res, err) -> ramAccounting.close()));
            context.registerBatchConsumer(nodeOperation.downstreamExecutionPhaseId(), bucketBuilder);
        }
    }
}

Example 7 with Streamer

use of io.crate.Streamer in project crate by crate.

the class HyperLogLogDistinctAggregationTest method testStreaming.

@Test
public void testStreaming() throws Exception {
    HyperLogLogDistinctAggregation.HllState hllState1 = new HyperLogLogDistinctAggregation.HllState(DataTypes.IP, true);
    hllState1.init(memoryManager, HyperLogLogPlusPlus.DEFAULT_PRECISION);
    BytesStreamOutput out = new BytesStreamOutput();
    Streamer streamer = HyperLogLogDistinctAggregation.HllStateType.INSTANCE.streamer();
    streamer.writeValueTo(out, hllState1);
    StreamInput in = out.bytes().streamInput();
    HyperLogLogDistinctAggregation.HllState hllState2 = (HyperLogLogDistinctAggregation.HllState) streamer.readValueFrom(in);
    // test that murmur3hash and HLL++ is correctly initialized with streamed dataType and version
    hllState1.add("127.0.0.1");
    hllState2.add("127.0.0.1");
    assertThat(hllState2.value(), is(hllState1.value()));
}

Example 8 with Streamer

use of io.crate.Streamer in project crate by crate.

the class TransportAnalyzeAction method fetchSamples.

private CompletableFuture<Samples> fetchSamples(RelationName relationName, List<Reference> columns) {
    FutureActionListener<FetchSampleResponse, Samples> listener = new FutureActionListener<>(FetchSampleResponse::samples);
    List<DiscoveryNode> nodesOn41OrAfter = StreamSupport.stream(clusterService.state().nodes().spliterator(), false).filter(x -> x.getVersion().onOrAfter(Version.V_4_1_0)).collect(Collectors.toList());
    MultiActionListener<FetchSampleResponse, ?, FetchSampleResponse> multiListener = new MultiActionListener<>(nodesOn41OrAfter.size(), Collectors.reducing(new FetchSampleResponse(Samples.EMPTY), (FetchSampleResponse s1, FetchSampleResponse s2) -> FetchSampleResponse.merge(TransportAnalyzeAction.NUM_SAMPLES, s1, s2)), listener);
    List<Streamer> streamers = Arrays.asList(Symbols.streamerArray(columns));
    ActionListenerResponseHandler<FetchSampleResponse> responseHandler = new ActionListenerResponseHandler<>(multiListener, in -> new FetchSampleResponse(streamers, in), ThreadPool.Names.SAME);
    for (DiscoveryNode node : nodesOn41OrAfter) {
        transportService.sendRequest(node, FETCH_SAMPLES, new FetchSampleRequest(relationName, columns, TransportAnalyzeAction.NUM_SAMPLES), responseHandler);
    }
    return listener;
}

Example 9 with Streamer

use of io.crate.Streamer in project crate by crate.

the class RowType method writeValueTo.

@SuppressWarnings({ "rawtypes", "unchecked" })
@Override
public void writeValueTo(StreamOutput out, Row row) throws IOException {
    assert row.numColumns() == fieldTypes.size() : "Row that should be streamed must have the same number of columns as the rowType contains fieldTypes";
    for (int i = 0; i < fieldTypes.size(); i++) {
        Streamer streamer = fieldTypes.get(i).streamer();
        streamer.writeValueTo(out, row.get(i));
    }
}

Example 10 with Streamer

use of io.crate.Streamer in project crate by crate.

the class DistributedResultRequestTest method testStreaming.

@Test
public void testStreaming() throws Exception {
    Streamer<?>[] streamers = new Streamer[] { DataTypes.STRING.streamer() };
    UUID uuid = UUID.randomUUID();
    StreamBucket.Builder builder = new StreamBucket.Builder(streamers, RamAccounting.NO_ACCOUNTING);
    builder.add(new RowN(new Object[] { "ab" }));
    builder.add(new RowN(new Object[] { null }));
    builder.add(new RowN(new Object[] { "cd" }));
    DistributedResultRequest r1 = new DistributedResultRequest(uuid, 1, (byte) 3, 1, builder.build(), false);
    BytesStreamOutput out = new BytesStreamOutput();
    r1.writeTo(out);
    StreamInput in = out.bytes().streamInput();
    DistributedResultRequest r2 = new DistributedResultRequest(in);
    assertEquals(r1.readRows(streamers).size(), r2.readRows(streamers).size());
    assertThat(r1.isLast(), is(r2.isLast()));
    assertThat(r1.executionPhaseInputId(), is(r2.executionPhaseInputId()));
    assertThat(r2.readRows(streamers), contains(isRow("ab"), isNullRow(), isRow("cd")));
}