Examples with ColumnDefinition io.deephaven.web.client.api.barrage.def.ColumnDefinition used on opensource projects

Example 1 with ColumnDefinition

use of io.deephaven.web.client.api.barrage.def.ColumnDefinition in project deephaven-core by deephaven.

the class WorkerConnection method flush.

private void flush() {
    // LATER: instead of running a bunch of serial operations,
    // condense these all into a single batch operation.
    // All three server calls made by this method are _only_ called by this method,
    // so we can reasonably merge all three into a single batched operation.
    ArrayList<ClientTableState> statesToFlush = new ArrayList<>(flushable);
    flushable.clear();
    for (ClientTableState state : statesToFlush) {
        if (state.hasNoSubscriptions()) {
            // yet completed (we leave orphaned nodes paused until a request completes).
            if (state.isSubscribed()) {
                state.setSubscribed(false);
                if (state.getHandle().isConnected()) {
                    BiDiStream<FlightData, FlightData> stream = subscriptionStreams.remove(state);
                    if (stream != null) {
                        stream.end();
                        stream.cancel();
                    }
                }
            }
            if (state.isEmpty()) {
                // completely empty; perform release
                final ClientTableState.ResolutionState previousState = state.getResolution();
                state.setResolution(ClientTableState.ResolutionState.RELEASED);
                state.setSubscribed(false);
                if (previousState != ClientTableState.ResolutionState.RELEASED) {
                    cache.release(state);
                    JsLog.debug("Releasing state", state, LazyString.of(state.getHandle()));
                    // don't send a release message to the server if the table isn't really there
                    if (state.getHandle().isConnected()) {
                        releaseHandle(state.getHandle());
                    }
                }
            }
        } else {
            List<TableSubscriptionRequest> vps = new ArrayList<>();
            state.forActiveSubscriptions((table, subscription) -> {
                assert table.isActive(state) : "Inactive table has a viewport still attached";
                vps.add(new TableSubscriptionRequest(table.getSubscriptionId(), subscription.getRows(), subscription.getColumns()));
            });
            boolean isViewport = vps.stream().allMatch(req -> req.getRows() != null);
            assert isViewport || vps.stream().noneMatch(req -> req.getRows() != null) : "All subscriptions to a given handle must be consistently viewport or non-viewport";
            BitSet includedColumns = vps.stream().map(TableSubscriptionRequest::getColumns).reduce((bs1, bs2) -> {
                BitSet result = new BitSet();
                result.or(bs1);
                result.or(bs2);
                return result;
            }).orElseThrow(() -> new IllegalStateException("Cannot call subscribe with zero subscriptions"));
            String[] columnTypes = Arrays.stream(state.getTableDef().getColumns()).map(ColumnDefinition::getType).toArray(String[]::new);
            state.setSubscribed(true);
            Builder subscriptionReq = new Builder(1024);
            double columnsOffset = BarrageSubscriptionRequest.createColumnsVector(subscriptionReq, makeUint8ArrayFromBitset(includedColumns));
            double viewportOffset = 0;
            if (isViewport) {
                viewportOffset = BarrageSubscriptionRequest.createViewportVector(subscriptionReq, serializeRanges(vps.stream().map(TableSubscriptionRequest::getRows).collect(Collectors.toSet())));
            }
            // TODO #188 support minUpdateIntervalMs
            double serializationOptionsOffset = BarrageSubscriptionOptions.createBarrageSubscriptionOptions(subscriptionReq, ColumnConversionMode.Stringify, true, 1000, 0);
            double tableTicketOffset = BarrageSubscriptionRequest.createTicketVector(subscriptionReq, state.getHandle().getTicket());
            BarrageSubscriptionRequest.startBarrageSubscriptionRequest(subscriptionReq);
            BarrageSubscriptionRequest.addColumns(subscriptionReq, columnsOffset);
            BarrageSubscriptionRequest.addSubscriptionOptions(subscriptionReq, serializationOptionsOffset);
            BarrageSubscriptionRequest.addViewport(subscriptionReq, viewportOffset);
            BarrageSubscriptionRequest.addTicket(subscriptionReq, tableTicketOffset);
            subscriptionReq.finish(BarrageSubscriptionRequest.endBarrageSubscriptionRequest(subscriptionReq));
            FlightData request = new FlightData();
            request.setAppMetadata(BarrageUtils.wrapMessage(subscriptionReq, BarrageMessageType.BarrageSubscriptionRequest));
            BiDiStream<FlightData, FlightData> stream = this.<FlightData, FlightData>streamFactory().create(headers -> flightServiceClient.doExchange(headers), (first, headers) -> browserFlightServiceClient.openDoExchange(first, headers), (next, headers, c) -> browserFlightServiceClient.nextDoExchange(next, headers, c::apply), new FlightData());
            stream.send(request);
            stream.onData(new JsConsumer<FlightData>() {

                @Override
                public void apply(FlightData data) {
                    ByteBuffer body = typedArrayToLittleEndianByteBuffer(data.getDataBody_asU8());
                    Message headerMessage = Message.getRootAsMessage(new io.deephaven.javascript.proto.dhinternal.flatbuffers.ByteBuffer(data.getDataHeader_asU8()));
                    if (body.limit() == 0 && headerMessage.headerType() != MessageHeader.RecordBatch) {
                        // TODO hang on to the schema to better handle the now-Utf8 columns
                        return;
                    }
                    RecordBatch header = headerMessage.header(new RecordBatch());
                    BarrageMessageWrapper barrageMessageWrapper = BarrageMessageWrapper.getRootAsBarrageMessageWrapper(new io.deephaven.javascript.proto.dhinternal.flatbuffers.ByteBuffer(data.getAppMetadata_asU8()));
                    if (barrageMessageWrapper.msgType() == BarrageMessageType.None) {
                        // continue previous message, just read RecordBatch
                        appendAndMaybeFlush(header, body);
                    } else {
                        assert barrageMessageWrapper.msgType() == BarrageMessageType.BarrageUpdateMetadata;
                        BarrageUpdateMetadata barrageUpdate = BarrageUpdateMetadata.getRootAsBarrageUpdateMetadata(new io.deephaven.javascript.proto.dhinternal.flatbuffers.ByteBuffer(new Uint8Array(barrageMessageWrapper.msgPayloadArray())));
                        startAndMaybeFlush(barrageUpdate.isSnapshot(), header, body, barrageUpdate, isViewport, columnTypes);
                    }
                }

                private DeltaUpdatesBuilder nextDeltaUpdates;

                private void appendAndMaybeFlush(RecordBatch header, ByteBuffer body) {
                    // using existing barrageUpdate, append to the current snapshot/delta
                    assert nextDeltaUpdates != null;
                    boolean shouldFlush = nextDeltaUpdates.appendRecordBatch(header, body);
                    if (shouldFlush) {
                        incrementalUpdates(state.getHandle(), nextDeltaUpdates.build());
                        nextDeltaUpdates = null;
                    }
                }

                private void startAndMaybeFlush(boolean isSnapshot, RecordBatch header, ByteBuffer body, BarrageUpdateMetadata barrageUpdate, boolean isViewport, String[] columnTypes) {
                    if (isSnapshot) {
                        TableSnapshot snapshot = createSnapshot(header, body, barrageUpdate, isViewport, columnTypes);
                        // for now we always expect snapshots to arrive in a single payload
                        initialSnapshot(state.getHandle(), snapshot);
                    } else {
                        nextDeltaUpdates = deltaUpdates(barrageUpdate, isViewport, columnTypes);
                        appendAndMaybeFlush(header, body);
                    }
                }
            });
            stream.onStatus(err -> {
                checkStatus(err);
                if (!err.isOk()) {
                    // TODO (core#1181): fix this hack that enables barrage errors to propagate to the UI widget
                    state.forActiveSubscriptions((table, subscription) -> {
                        table.failureHandled(err.getDetails());
                    });
                }
            });
            BiDiStream<FlightData, FlightData> oldStream = subscriptionStreams.put(state, stream);
            if (oldStream != null) {
                // cancel any old stream, we presently expect a fresh instance
                oldStream.end();
                oldStream.cancel();
            }
        }
    }
}

Example 2 with ColumnDefinition

use of io.deephaven.web.client.api.barrage.def.ColumnDefinition in project deephaven-core by deephaven.

the class ClientTableState method setTableDef.

private void setTableDef(InitialTableDefinition tableDef) {
    boolean create = this.tableDef != tableDef;
    this.tableDef = tableDef;
    if (create) {
        ColumnDefinition[] columnDefinitions = tableDef.getColumns();
        // iterate through the columns, combine format columns into the normal model
        Map<String, ColumnDefinition> byNameMap = Arrays.stream(columnDefinitions).collect(columnCollector(false));
        Column[] columns1 = new Column[0];
        allColumns = new Column[0];
        for (ColumnDefinition definition : columnDefinitions) {
            if (definition.isForRow()) {
                // special case for the row format column
                setRowFormatColumn(makeColumn(-1, definition, null, null, false, null, null, false));
                continue;
            }
            String name = definition.getName();
            ColumnDefinition format = byNameMap.get(definition.getFormatColumnName());
            ColumnDefinition style = byNameMap.get(definition.getStyleColumnName());
            boolean isPartitionColumn = definition.isPartitionColumn();
            // note the use of columns.length as jsIndex is accurate for visible columns
            allColumns[allColumns.length] = makeColumn(columns1.length, definition, format == null || !format.isNumberFormatColumn() ? null : format.getColumnIndex(), style == null ? null : style.getColumnIndex(), isPartitionColumn, format == null || format.isNumberFormatColumn() ? null : format.getColumnIndex(), definition.getDescription(), definition.isInputTableKeyColumn());
            if (definition.isVisible()) {
                columns1[columns1.length] = allColumns[allColumns.length - 1];
            }
        }
        this.columns = JsObject.freeze(columns1);
        this.columnLookup = resetLookup();
    }
}

Example 3 with ColumnDefinition

use of io.deephaven.web.client.api.barrage.def.ColumnDefinition in project deephaven-core by deephaven.

the class ClientTableState method applyTableCreationResponse.

public void applyTableCreationResponse(ExportedTableCreationResponse def) {
    assert def.getResultId().getTicket().getTicket_asB64().equals(getHandle().makeTicket().getTicket_asB64()) : "Ticket is incompatible with the table details";
    // by definition, the ticket is now exported and connected
    handle.setState(TableTicket.State.EXPORTED);
    handle.setConnected(true);
    // we conform to flight's schema representation of:
    // - IPC_CONTINUATION_TOKEN (4-byte int of -1)
    // - message size (4-byte int)
    // - a Message wrapping the schema
    ByteBuffer bb = new ByteBuffer(def.getSchemaHeader_asU8());
    bb.setPosition(bb.position() + 8);
    Message headerMessage = Message.getRootAsMessage(bb);
    assert headerMessage.headerType() == MessageHeader.Schema;
    Schema schema = headerMessage.header(new Schema());
    ColumnDefinition[] cols = new ColumnDefinition[(int) schema.fieldsLength()];
    for (int i = 0; i < schema.fieldsLength(); i++) {
        cols[i] = new ColumnDefinition();
        Field f = schema.fields(i);
        Map<String, String> fieldMetadata = keyValuePairs("deephaven:", f.customMetadataLength(), f::customMetadata);
        cols[i].setName(f.name().asString());
        cols[i].setColumnIndex(i);
        cols[i].setType(fieldMetadata.get("type"));
        cols[i].setStyleColumn("true".equals(fieldMetadata.get("isStyle")));
        cols[i].setFormatColumn("true".equals(fieldMetadata.get("isDateFormat")) || "true".equals(fieldMetadata.get("isNumberFormat")));
        cols[i].setForRow("true".equals(fieldMetadata.get("isRowStyle")));
        String formatColumnName = fieldMetadata.get("dateFormatColumn");
        if (formatColumnName == null) {
            formatColumnName = fieldMetadata.get("numberFormatColumn");
        }
        cols[i].setFormatColumnName(formatColumnName);
        cols[i].setStyleColumnName(fieldMetadata.get("styleColumn"));
        if (fieldMetadata.containsKey("inputtable.isKey")) {
            cols[i].setInputTableKeyColumn(Boolean.parseBoolean(fieldMetadata.get("inputtable.isKey")));
        }
        cols[i].setDescription(fieldMetadata.get("description"));
    }
    TableAttributesDefinition attributes = new TableAttributesDefinition(keyValuePairs("deephaven:attribute.", schema.customMetadataLength(), schema::customMetadata), keyValuePairs("deephaven:unsent.attribute.", schema.customMetadataLength(), schema::customMetadata).keySet());
    setTableDef(new InitialTableDefinition().setAttributes(attributes).setColumns(cols));
    setResolution(ResolutionState.RUNNING);
    setSize(Long.parseLong(def.getSize()));
}

Example 4 with ColumnDefinition

use of io.deephaven.web.client.api.barrage.def.ColumnDefinition in project deephaven-core by deephaven.

the class JsTable method handleSnapshot.

public void handleSnapshot(TableTicket handle, TableSnapshot snapshot) {
    if (!handle.equals(state().getHandle())) {
        return;
    }
    Viewport viewport = getBinding().getSubscription();
    if (viewport == null || viewport.getRows() == null || viewport.getRows().size() == 0) {
        // check out if we have a non-viewport sub attached
        if (nonViewportSub != null) {
            nonViewportSub.handleSnapshot(snapshot);
        }
        return;
    }
    RangeSet viewportRows = viewport.getRows();
    JsLog.debug("handleSnapshot on " + viewportRows, handle, snapshot, viewport);
    RangeSet includedRows = snapshot.getIncludedRows();
    ColumnData[] dataColumns = snapshot.getDataColumns();
    JsArray[] remappedData = new JsArray[dataColumns.length];
    // remap dataColumns to the expected range for that table's viewport
    long lastRow = -1;
    for (int col = viewport.getColumns().nextSetBit(0); col >= 0; col = viewport.getColumns().nextSetBit(col + 1)) {
        ColumnData dataColumn = dataColumns[col];
        if (dataColumn == null) {
            // TODO when IDS-2138 is fixed stop throwing this data away
            return;
        }
        Object columnData = dataColumn.getData();
        final ColumnDefinition def = state().getTableDef().getColumns()[col];
        remappedData[col] = JsData.newArray(def.getType());
        PrimitiveIterator.OfLong viewportIterator = viewportRows.indexIterator();
        PrimitiveIterator.OfLong includedRowsIterator = includedRows.indexIterator();
        int dataIndex = 0;
        while (viewportIterator.hasNext()) {
            long viewportIndex = viewportIterator.nextLong();
            if (viewportIndex >= snapshot.getTableSize()) {
                // reached or passed the end of the table, we'll still make a snapshot
                break;
            }
            if (!includedRowsIterator.hasNext()) {
                // TODO when IDS-2138 is fixed stop throwing this data away
                return;
            }
            long possibleMatch = includedRowsIterator.nextLong();
            while (includedRowsIterator.hasNext() && possibleMatch < viewportIndex) {
                // skip, still seeking to the next item
                dataIndex++;
                possibleMatch = includedRowsIterator.nextLong();
            }
            if (!includedRowsIterator.hasNext() && possibleMatch < viewportIndex) {
                // we didn't find any items which match, just give up
                return;
            }
            if (possibleMatch > viewportIndex) {
                // rest of this table entirely, a later update will get us caught up
                return;
            }
            Object data = Js.<JsArray<Object>>uncheckedCast(columnData).getAt(dataIndex);
            remappedData[col].push(data);
            // increment for the next row
            dataIndex++;
            // Track how many rows were actually present, allowing the snapshot to stop before the viewport's end
            lastRow = Math.max(lastRow, possibleMatch);
        }
    }
    // TODO correct this - assumes max one range per table viewport, and nothing skipped
    RangeSet actualViewport = lastRow == -1 ? RangeSet.empty() : RangeSet.ofRange(viewportRows.indexIterator().nextLong(), lastRow);
    handleSnapshot(handle, snapshot.getSnapshotType(), actualViewport, remappedData, viewport.getColumns(), viewportRows.size());
}

Example 5 with ColumnDefinition

use of io.deephaven.web.client.api.barrage.def.ColumnDefinition in project deephaven-core by deephaven.

the class TableViewportSubscription method snapshot.

@JsMethod
public Promise<TableData> snapshot(JsRangeSet rows, Column[] columns) {
    // TODO #1039 slice rows and drop columns
    return copy.then(table -> {
        final ClientTableState state = table.state();
        String[] columnTypes = Arrays.stream(state.getTableDef().getColumns()).map(ColumnDefinition::getType).toArray(String[]::new);
        final BitSet columnBitset = table.lastVisibleState().makeBitset(columns);
        return Callbacks.<TableSnapshot, String>promise(this, callback -> {
            WorkerConnection connection = table.getConnection();
            BiDiStream<FlightData, FlightData> stream = connection.<FlightData, FlightData>streamFactory().create(headers -> connection.flightServiceClient().doExchange(headers), (first, headers) -> connection.browserFlightServiceClient().openDoExchange(first, headers), (next, headers, c) -> connection.browserFlightServiceClient().nextDoExchange(next, headers, c::apply), new FlightData());
            Builder doGetRequest = new Builder(1024);
            double columnsOffset = BarrageSubscriptionRequest.createColumnsVector(doGetRequest, makeUint8ArrayFromBitset(columnBitset));
            double viewportOffset = BarrageSubscriptionRequest.createViewportVector(doGetRequest, serializeRanges(Collections.singleton(rows.getRange())));
            double serializationOptionsOffset = BarrageSnapshotOptions.createBarrageSnapshotOptions(doGetRequest, ColumnConversionMode.Stringify, true, 0);
            double tableTicketOffset = BarrageSubscriptionRequest.createTicketVector(doGetRequest, state.getHandle().getTicket());
            BarrageSnapshotRequest.startBarrageSnapshotRequest(doGetRequest);
            BarrageSnapshotRequest.addTicket(doGetRequest, tableTicketOffset);
            BarrageSnapshotRequest.addColumns(doGetRequest, columnsOffset);
            BarrageSnapshotRequest.addSnapshotOptions(doGetRequest, serializationOptionsOffset);
            BarrageSnapshotRequest.addViewport(doGetRequest, viewportOffset);
            doGetRequest.finish(BarrageSnapshotRequest.endBarrageSnapshotRequest(doGetRequest));
            FlightData request = new FlightData();
            request.setAppMetadata(BarrageUtils.wrapMessage(doGetRequest, BarrageMessageType.BarrageSnapshotRequest));
            stream.send(request);
            stream.end();
            stream.onData(flightData -> {
                Message message = Message.getRootAsMessage(new ByteBuffer(flightData.getDataHeader_asU8()));
                if (message.headerType() == MessageHeader.Schema) {
                    // ignore for now, we'll handle this later
                    return;
                }
                assert message.headerType() == MessageHeader.RecordBatch;
                RecordBatch header = message.header(new RecordBatch());
                Uint8Array appMetadataBytes = flightData.getAppMetadata_asU8();
                BarrageUpdateMetadata update = null;
                if (appMetadataBytes.length != 0) {
                    BarrageMessageWrapper barrageMessageWrapper = BarrageMessageWrapper.getRootAsBarrageMessageWrapper(new io.deephaven.javascript.proto.dhinternal.flatbuffers.ByteBuffer(appMetadataBytes));
                    update = BarrageUpdateMetadata.getRootAsBarrageUpdateMetadata(new ByteBuffer(new Uint8Array(barrageMessageWrapper.msgPayloadArray())));
                }
                TableSnapshot snapshot = BarrageUtils.createSnapshot(header, BarrageUtils.typedArrayToLittleEndianByteBuffer(flightData.getDataBody_asU8()), update, true, columnTypes);
                callback.onSuccess(snapshot);
            });
            stream.onStatus(status -> {
                if (!status.isOk()) {
                    callback.onFailure(status.getDetails());
                }
            });
        }).then(defer()).then(snapshot -> {
            SubscriptionTableData pretendSubscription = new SubscriptionTableData(Js.uncheckedCast(columns), state.getRowFormatColumn() == null ? NO_ROW_FORMAT_COLUMN : state.getRowFormatColumn().getIndex(), null);
            TableData data = pretendSubscription.handleSnapshot(snapshot);
            return Promise.resolve(data);
        }).then(defer());
    });
}