Examples with JsConsumer io.deephaven.web.shared.fu.JsConsumer used on opensource projects

Example 1 with JsConsumer

use of io.deephaven.web.shared.fu.JsConsumer in project deephaven-core by deephaven.

the class WorkerConnection method flush.

private void flush() {
    // LATER: instead of running a bunch of serial operations,
    // condense these all into a single batch operation.
    // All three server calls made by this method are _only_ called by this method,
    // so we can reasonably merge all three into a single batched operation.
    ArrayList<ClientTableState> statesToFlush = new ArrayList<>(flushable);
    flushable.clear();
    for (ClientTableState state : statesToFlush) {
        if (state.hasNoSubscriptions()) {
            // yet completed (we leave orphaned nodes paused until a request completes).
            if (state.isSubscribed()) {
                state.setSubscribed(false);
                if (state.getHandle().isConnected()) {
                    BiDiStream<FlightData, FlightData> stream = subscriptionStreams.remove(state);
                    if (stream != null) {
                        stream.end();
                        stream.cancel();
                    }
                }
            }
            if (state.isEmpty()) {
                // completely empty; perform release
                final ClientTableState.ResolutionState previousState = state.getResolution();
                state.setResolution(ClientTableState.ResolutionState.RELEASED);
                state.setSubscribed(false);
                if (previousState != ClientTableState.ResolutionState.RELEASED) {
                    cache.release(state);
                    JsLog.debug("Releasing state", state, LazyString.of(state.getHandle()));
                    // don't send a release message to the server if the table isn't really there
                    if (state.getHandle().isConnected()) {
                        releaseHandle(state.getHandle());
                    }
                }
            }
        } else {
            List<TableSubscriptionRequest> vps = new ArrayList<>();
            state.forActiveSubscriptions((table, subscription) -> {
                assert table.isActive(state) : "Inactive table has a viewport still attached";
                vps.add(new TableSubscriptionRequest(table.getSubscriptionId(), subscription.getRows(), subscription.getColumns()));
            });
            boolean isViewport = vps.stream().allMatch(req -> req.getRows() != null);
            assert isViewport || vps.stream().noneMatch(req -> req.getRows() != null) : "All subscriptions to a given handle must be consistently viewport or non-viewport";
            BitSet includedColumns = vps.stream().map(TableSubscriptionRequest::getColumns).reduce((bs1, bs2) -> {
                BitSet result = new BitSet();
                result.or(bs1);
                result.or(bs2);
                return result;
            }).orElseThrow(() -> new IllegalStateException("Cannot call subscribe with zero subscriptions"));
            String[] columnTypes = Arrays.stream(state.getTableDef().getColumns()).map(ColumnDefinition::getType).toArray(String[]::new);
            state.setSubscribed(true);
            Builder subscriptionReq = new Builder(1024);
            double columnsOffset = BarrageSubscriptionRequest.createColumnsVector(subscriptionReq, makeUint8ArrayFromBitset(includedColumns));
            double viewportOffset = 0;
            if (isViewport) {
                viewportOffset = BarrageSubscriptionRequest.createViewportVector(subscriptionReq, serializeRanges(vps.stream().map(TableSubscriptionRequest::getRows).collect(Collectors.toSet())));
            }
            // TODO #188 support minUpdateIntervalMs
            double serializationOptionsOffset = BarrageSubscriptionOptions.createBarrageSubscriptionOptions(subscriptionReq, ColumnConversionMode.Stringify, true, 1000, 0);
            double tableTicketOffset = BarrageSubscriptionRequest.createTicketVector(subscriptionReq, state.getHandle().getTicket());
            BarrageSubscriptionRequest.startBarrageSubscriptionRequest(subscriptionReq);
            BarrageSubscriptionRequest.addColumns(subscriptionReq, columnsOffset);
            BarrageSubscriptionRequest.addSubscriptionOptions(subscriptionReq, serializationOptionsOffset);
            BarrageSubscriptionRequest.addViewport(subscriptionReq, viewportOffset);
            BarrageSubscriptionRequest.addTicket(subscriptionReq, tableTicketOffset);
            subscriptionReq.finish(BarrageSubscriptionRequest.endBarrageSubscriptionRequest(subscriptionReq));
            FlightData request = new FlightData();
            request.setAppMetadata(BarrageUtils.wrapMessage(subscriptionReq, BarrageMessageType.BarrageSubscriptionRequest));
            BiDiStream<FlightData, FlightData> stream = this.<FlightData, FlightData>streamFactory().create(headers -> flightServiceClient.doExchange(headers), (first, headers) -> browserFlightServiceClient.openDoExchange(first, headers), (next, headers, c) -> browserFlightServiceClient.nextDoExchange(next, headers, c::apply), new FlightData());
            stream.send(request);
            stream.onData(new JsConsumer<FlightData>() {

                @Override
                public void apply(FlightData data) {
                    ByteBuffer body = typedArrayToLittleEndianByteBuffer(data.getDataBody_asU8());
                    Message headerMessage = Message.getRootAsMessage(new io.deephaven.javascript.proto.dhinternal.flatbuffers.ByteBuffer(data.getDataHeader_asU8()));
                    if (body.limit() == 0 && headerMessage.headerType() != MessageHeader.RecordBatch) {
                        // TODO hang on to the schema to better handle the now-Utf8 columns
                        return;
                    }
                    RecordBatch header = headerMessage.header(new RecordBatch());
                    BarrageMessageWrapper barrageMessageWrapper = BarrageMessageWrapper.getRootAsBarrageMessageWrapper(new io.deephaven.javascript.proto.dhinternal.flatbuffers.ByteBuffer(data.getAppMetadata_asU8()));
                    if (barrageMessageWrapper.msgType() == BarrageMessageType.None) {
                        // continue previous message, just read RecordBatch
                        appendAndMaybeFlush(header, body);
                    } else {
                        assert barrageMessageWrapper.msgType() == BarrageMessageType.BarrageUpdateMetadata;
                        BarrageUpdateMetadata barrageUpdate = BarrageUpdateMetadata.getRootAsBarrageUpdateMetadata(new io.deephaven.javascript.proto.dhinternal.flatbuffers.ByteBuffer(new Uint8Array(barrageMessageWrapper.msgPayloadArray())));
                        startAndMaybeFlush(barrageUpdate.isSnapshot(), header, body, barrageUpdate, isViewport, columnTypes);
                    }
                }

                private DeltaUpdatesBuilder nextDeltaUpdates;

                private void appendAndMaybeFlush(RecordBatch header, ByteBuffer body) {
                    // using existing barrageUpdate, append to the current snapshot/delta
                    assert nextDeltaUpdates != null;
                    boolean shouldFlush = nextDeltaUpdates.appendRecordBatch(header, body);
                    if (shouldFlush) {
                        incrementalUpdates(state.getHandle(), nextDeltaUpdates.build());
                        nextDeltaUpdates = null;
                    }
                }

                private void startAndMaybeFlush(boolean isSnapshot, RecordBatch header, ByteBuffer body, BarrageUpdateMetadata barrageUpdate, boolean isViewport, String[] columnTypes) {
                    if (isSnapshot) {
                        TableSnapshot snapshot = createSnapshot(header, body, barrageUpdate, isViewport, columnTypes);
                        // for now we always expect snapshots to arrive in a single payload
                        initialSnapshot(state.getHandle(), snapshot);
                    } else {
                        nextDeltaUpdates = deltaUpdates(barrageUpdate, isViewport, columnTypes);
                        appendAndMaybeFlush(header, body);
                    }
                }
            });
            stream.onStatus(err -> {
                checkStatus(err);
                if (!err.isOk()) {
                    // TODO (core#1181): fix this hack that enables barrage errors to propagate to the UI widget
                    state.forActiveSubscriptions((table, subscription) -> {
                        table.failureHandled(err.getDetails());
                    });
                }
            });
            BiDiStream<FlightData, FlightData> oldStream = subscriptionStreams.put(state, stream);
            if (oldStream != null) {
                // cancel any old stream, we presently expect a fresh instance
                oldStream.end();
                oldStream.cancel();
            }
        }
    }
}

Example 2 with JsConsumer

use of io.deephaven.web.shared.fu.JsConsumer in project deephaven-core by deephaven.

the class RequestBatcher method sendRequest.

public Promise<Void> sendRequest() {
    return new Promise<>((resolve, reject) -> {
        // calling buildRequest will change the state of each table, so we first check what the state was
        // of each table before we do that, in case we are actually not making a call to the server
        ClientTableState prevState = table.lastVisibleState();
        final BatchTableRequest request = buildRequest();
        // let callbacks peek at entire request about to be sent.
        for (JsConsumer<BatchTableRequest> send : onSend) {
            send.apply(request);
        }
        onSend.clear();
        sent = true;
        if (request.getOpsList().length == 0) {
            // This is like tableLoop below, except no failure is possible, since we already have the results
            if (table.isAlive()) {
                final ClientTableState active = table.state();
                assert active.isRunning() : active;
                boolean sortChanged = !prevState.getSorts().equals(active.getSorts());
                boolean filterChanged = !prevState.getFilters().equals(active.getFilters());
                boolean customColumnChanged = !prevState.getCustomColumns().equals(active.getCustomColumns());
                table.fireEvent(HasEventHandling.EVENT_REQUEST_SUCCEEDED);
                // TODO think more about the order of events, and what kinds of things one might bind to each
                if (sortChanged) {
                    table.fireEvent(JsTable.EVENT_SORTCHANGED);
                }
                if (filterChanged) {
                    table.fireEvent(JsTable.EVENT_FILTERCHANGED);
                }
                if (customColumnChanged) {
                    table.fireEvent(JsTable.EVENT_CUSTOMCOLUMNSCHANGED);
                }
            }
            orphans.forEach(ClientTableState::cleanup);
            resolve.onInvoke((Void) null);
            // if there's no outgoing operation, user may have removed an operation and wants to return to where
            // they left off
            table.maybeReviveSubscription();
            finished = true;
            return;
        }
        final BatchOp operationHead = builder.getFirstOp();
        assert operationHead != null : "A non-empty request must have a firstOp!";
        final ClientTableState source = operationHead.getAppendTo();
        assert source != null : "A non-empty request must have a source state!";
        JsLog.debug("Sending request", LazyString.of(request), request, " based on ", this);
        ResponseStreamWrapper<ExportedTableCreationResponse> batchStream = ResponseStreamWrapper.of(connection.tableServiceClient().batch(request, connection.metadata()));
        batchStream.onData(response -> {
            TableReference resultid = response.getResultId();
            if (!resultid.hasTicket()) {
                // thanks for telling us, but we don't at this time have a nice way to indicate this
                return;
            }
            Ticket ticket = resultid.getTicket();
            if (!response.getSuccess()) {
                String fail = response.getErrorInfo();
                // any table which has that state active should fire a failed event
                ClientTableState state = allStates().filter(cts -> cts.getHandle().makeTicket().getTicket_asB64().equals(ticket.getTicket_asB64())).first();
                if (state.isEmpty()) {
                    // nobody cares about this state anymore
                    JsLog.debug("Ignoring empty state", state);
                    return;
                }
                state.getHandle().setState(TableTicket.State.FAILED);
                for (JsTable table : allInterestedTables().filter(t -> t.state() == state)) {
                    // fire the failed event
                    failTable(table, fail);
                }
                // mark state as failed (his has to happen after table is marked as failed, it will trigger rollback
                state.setResolution(ClientTableState.ResolutionState.FAILED, fail);
                return;
            }
            // find the state that applies to this ticket
            ClientTableState state = allStates().filter(cts -> cts.getHandle().makeTicket().getTicket_asB64().equals(ticket.getTicket_asB64())).first();
            if (state.isEmpty()) {
                // response.
                return;
            }
            // Before we mark it as successfully running and give it its new schema, track the previous CTS
            // that each table was using. Identify which table to watch based on its current state, even if
            // not visible, but then track the last visible state, so we know which events to fire.
            Map<JsTable, ClientTableState> activeTablesAndStates = StreamSupport.stream(allInterestedTables().spliterator(), false).filter(JsTable::isAlive).filter(t -> t.state() == state).collect(Collectors.toMap(Function.identity(), JsTable::lastVisibleState));
            // Mark the table as ready to go
            state.applyTableCreationResponse(response);
            state.forActiveTables(t -> t.maybeRevive(state));
            state.setResolution(ClientTableState.ResolutionState.RUNNING);
            // Let each table know that what has changed since it has a previous state
            for (JsTable table : activeTablesAndStates.keySet()) {
                ClientTableState lastVisibleState = activeTablesAndStates.get(table);
                // fire any events that are necessary
                boolean sortChanged = !lastVisibleState.getSorts().equals(state.getSorts());
                boolean filterChanged = !lastVisibleState.getFilters().equals(state.getFilters());
                boolean customColumnChanged = !lastVisibleState.getCustomColumns().equals(state.getCustomColumns());
                table.fireEvent(HasEventHandling.EVENT_REQUEST_SUCCEEDED);
                // TODO think more about the order of events, and what kinds of things one might bind to each
                if (sortChanged) {
                    table.fireEvent(JsTable.EVENT_SORTCHANGED);
                }
                if (filterChanged) {
                    table.fireEvent(JsTable.EVENT_FILTERCHANGED);
                }
                if (customColumnChanged) {
                    table.fireEvent(JsTable.EVENT_CUSTOMCOLUMNSCHANGED);
                }
            }
        });
        batchStream.onEnd(status -> {
            // request is complete
            if (status.isOk()) {
                resolve.onInvoke((Void) null);
            } else {
                failed(reject, status.getDetails());
            }
            // Tell anybody who was orphaned to check if they should release their subscriptions / handles
            orphans.forEach(ClientTableState::cleanup);
        });
    });
}