Examples with Assert io.deephaven.base.verify.Assert used on opensource projects

Example 1 with Assert

use of io.deephaven.base.verify.Assert in project deephaven-core by deephaven.

the class CrossJoinHelper method zeroKeyColumnsJoin.

@NotNull
private static Table zeroKeyColumnsJoin(QueryTable leftTable, QueryTable rightTable, MatchPair[] columnsToAdd, int numRightBitsToReserve, String listenerDescription) {
    // we are a single value join, we do not need to do any hash-related work
    validateZeroKeyIndexSpace(leftTable, rightTable, numRightBitsToReserve);
    final CrossJoinShiftState crossJoinState = new CrossJoinShiftState(Math.max(numRightBitsToReserve, CrossJoinShiftState.getMinBits(rightTable)));
    final TrackingWritableRowSet resultRowSet = RowSetFactory.empty().toTracking();
    final QueryTable result = makeResult(leftTable, rightTable, columnsToAdd, crossJoinState, resultRowSet, cs -> new BitMaskingColumnSource<>(crossJoinState, cs));
    final ModifiedColumnSet.Transformer leftTransformer = leftTable.newModifiedColumnSetTransformer(result, leftTable.getDefinition().getColumnNamesArray());
    final ModifiedColumnSet.Transformer rightTransformer = rightTable.newModifiedColumnSetTransformer(result, columnsToAdd);
    final BiConsumer<TableUpdate, TableUpdate> onUpdate = (leftUpdate, rightUpdate) -> {
        final boolean leftChanged = leftUpdate != null;
        final boolean rightChanged = rightUpdate != null;
        final int prevRightBits = crossJoinState.getNumShiftBits();
        final int currRightBits = Math.max(prevRightBits, CrossJoinShiftState.getMinBits(rightTable));
        validateZeroKeyIndexSpace(leftTable, rightTable, currRightBits);
        if (currRightBits != prevRightBits) {
            crossJoinState.setNumShiftBitsAndUpdatePrev(currRightBits);
        }
        final TableUpdateImpl downstream = new TableUpdateImpl();
        downstream.added = RowSetFactory.empty();
        downstream.removed = RowSetFactory.empty();
        downstream.modified = RowSetFactory.empty();
        downstream.modifiedColumnSet = result.getModifiedColumnSetForUpdates();
        downstream.modifiedColumnSet.clear();
        final RowSetShiftData.Builder shiftBuilder = new RowSetShiftData.Builder();
        try (final SafeCloseableList closer = new SafeCloseableList()) {
            if (rightChanged && rightUpdate.modified().isNonempty()) {
                rightTransformer.transform(rightUpdate.modifiedColumnSet(), downstream.modifiedColumnSet);
            }
            if (leftChanged && leftUpdate.modified().isNonempty()) {
                leftTransformer.transform(leftUpdate.modifiedColumnSet(), downstream.modifiedColumnSet);
            }
            // how far currRight has been shifted
            long currRightShift = 0;
            final WritableRowSet currRight = closer.add(rightTable.getRowSet().copy());
            if (rightChanged) {
                // Must touch every left row. (Note: this code is accessible iff right changed.)
                final TrackingRowSet currLeft = leftTable.getRowSet();
                final RowSet prevLeft = closer.add(currLeft.copyPrev());
                // how far prevRight has been shifted
                long prevRightShift = 0;
                final WritableRowSet prevRight = closer.add(rightTable.getRowSet().copyPrev());
                // how far rmRight has been shifted
                long rmRightShift = 0;
                final WritableRowSet rmRight = closer.add(rightUpdate.removed().copy());
                // how far addRight has been shifted
                long addRightShift = 0;
                final WritableRowSet addRight = closer.add(rightUpdate.added().copy());
                // how far modRight has been shifted
                long modRightShift = 0;
                final WritableRowSet modRight = closer.add(rightUpdate.modified().copy());
                // how far existingRight has been shifted
                long existingRightShift = 0;
                final WritableRowSet existingRight = closer.add(currRight.minus(rightUpdate.added()));
                final boolean rightHasAdds = addRight.isNonempty();
                final boolean rightHasRemoves = rmRight.isNonempty();
                final boolean rightHasModifies = modRight.isNonempty();
                // Do note that add/mod's are in post-shift keyspace.
                final RowSet.SearchIterator leftAddIter = leftChanged ? leftUpdate.added().searchIterator() : null;
                final RowSet.SearchIterator leftRmIter = leftChanged ? leftUpdate.removed().searchIterator() : null;
                final RowSet.SearchIterator leftModIter = leftChanged ? leftUpdate.modified().searchIterator() : null;
                boolean moreLeftAdd = leftChanged && advanceIterator(leftAddIter);
                boolean moreLeftRm = leftChanged && advanceIterator(leftRmIter);
                boolean moreLeftMod = leftChanged && advanceIterator(leftModIter);
                // Prepare left-side iterators.
                final RowSet.SearchIterator leftPrevIter = prevLeft.searchIterator();
                final RowSet.SearchIterator leftCurrIter = leftTable.getRowSet().searchIterator();
                boolean moreLeftPrev = advanceIterator(leftPrevIter);
                boolean moreLeftCurr = advanceIterator(leftCurrIter);
                // It is more efficient to completely rebuild this RowSet, than to modify each row to right mapping.
                resultRowSet.clear();
                final long prevCardinality = 1L << prevRightBits;
                final long currCardinality = 1L << currRightBits;
                // Note: This assumes that shifts are not allowed to re-order data.
                while (moreLeftPrev) {
                    final long currPrevIdx = leftPrevIter.currentValue();
                    final long prevResultOffset = currPrevIdx << prevRightBits;
                    moreLeftPrev = advanceIterator(leftPrevIter);
                    if (moreLeftRm && currPrevIdx == leftRmIter.currentValue()) {
                        // currPrevIdx is a left remove.
                        moreLeftRm = advanceIterator(leftRmIter);
                        prevRightShift = furtherShiftIndex(prevRight, prevRightShift, prevResultOffset);
                        downstream.removed().writableCast().insert(prevRight);
                        continue;
                    }
                    // Note: Pre-existing row was not removed, therefore there must be an entry in curr RowSet.
                    Assert.eqTrue(moreLeftCurr, "moreLeftCurr");
                    long currCurrIdx = leftCurrIter.currentValue();
                    long currResultOffset = currCurrIdx << currRightBits;
                    moreLeftCurr = advanceIterator(leftCurrIter);
                    // Insert adds until we find our currCurrIdx that matches currPrevIdx.
                    while (moreLeftAdd && currCurrIdx == leftAddIter.currentValue()) {
                        // currCurrIdx is a left add.
                        moreLeftAdd = advanceIterator(leftAddIter);
                        currRightShift = furtherShiftIndex(currRight, currRightShift, currResultOffset);
                        downstream.added().writableCast().insert(currRight);
                        resultRowSet.insert(currRight);
                        // Advance left current iterator.
                        Assert.eqTrue(moreLeftCurr, "moreLeftCurr");
                        currCurrIdx = leftCurrIter.currentValue();
                        currResultOffset = currCurrIdx << currRightBits;
                        moreLeftCurr = advanceIterator(leftCurrIter);
                    }
                    if (rightHasRemoves) {
                        rmRightShift = furtherShiftIndex(rmRight, rmRightShift, prevResultOffset);
                        downstream.removed().writableCast().insert(rmRight);
                    }
                    if (rightHasAdds) {
                        addRightShift = furtherShiftIndex(addRight, addRightShift, currResultOffset);
                        downstream.added().writableCast().insert(addRight);
                    }
                    if (moreLeftMod && currCurrIdx == leftModIter.currentValue()) {
                        // currCurrIdx is modify; paint all existing rows as modified
                        moreLeftMod = advanceIterator(leftModIter);
                        existingRightShift = furtherShiftIndex(existingRight, existingRightShift, currResultOffset);
                        downstream.modified().writableCast().insert(existingRight);
                    } else if (rightHasModifies) {
                        modRightShift = furtherShiftIndex(modRight, modRightShift, currResultOffset);
                        downstream.modified().writableCast().insert(modRight);
                    }
                    currRightShift = furtherShiftIndex(currRight, currRightShift, currResultOffset);
                    resultRowSet.insert(currRight);
                    if (rightUpdate.shifted().nonempty()) {
                        shiftBuilder.appendShiftData(rightUpdate.shifted(), prevResultOffset, prevCardinality, currResultOffset, currCardinality);
                    } else if (currResultOffset != prevResultOffset) {
                        final long shiftDelta = currResultOffset - prevResultOffset;
                        final long lastResultIdx = prevResultOffset + prevCardinality - 1;
                        shiftBuilder.shiftRange(prevResultOffset, lastResultIdx, shiftDelta);
                    }
                }
                // Note: Only left adds remain.
                while (moreLeftCurr) {
                    final long currCurrIdx = leftCurrIter.currentValue();
                    moreLeftCurr = advanceIterator(leftCurrIter);
                    Assert.eqTrue(moreLeftAdd, "moreLeftAdd");
                    assert leftAddIter != null;
                    Assert.eq(currCurrIdx, "currCurrIdx", leftAddIter.currentValue(), "leftAddIter.currentValue()");
                    moreLeftAdd = advanceIterator(leftAddIter);
                    final long currResultIdx = currCurrIdx << currRightBits;
                    currRightShift = furtherShiftIndex(currRight, currRightShift, currResultIdx);
                    downstream.added().writableCast().insert(currRight);
                    resultRowSet.insert(currRight);
                }
                downstream.shifted = shiftBuilder.build();
            } else {
                // Explode left updates to apply to all right rows.
                assert leftUpdate != null;
                RowSet.SearchIterator iter = leftUpdate.removed().searchIterator();
                while (iter.hasNext()) {
                    final long currIdx = iter.nextLong();
                    final long currResultIdx = currIdx << currRightBits;
                    currRightShift = furtherShiftIndex(currRight, currRightShift, currResultIdx);
                    downstream.removed().writableCast().insert(currRight);
                    resultRowSet.removeRange(currResultIdx, ((currIdx + 1) << currRightBits) - 1);
                }
                downstream.shifted = expandLeftOnlyShift(leftUpdate.shifted(), crossJoinState);
                downstream.shifted().apply(resultRowSet);
                iter = leftUpdate.modified().searchIterator();
                while (iter.hasNext()) {
                    final long currIdx = iter.nextLong();
                    final long currResultIdx = currIdx << currRightBits;
                    currRightShift = furtherShiftIndex(currRight, currRightShift, currResultIdx);
                    downstream.modified().writableCast().insert(currRight);
                }
                iter = leftUpdate.added().searchIterator();
                while (iter.hasNext()) {
                    final long currIdx = iter.nextLong();
                    final long currResultIdx = currIdx << currRightBits;
                    currRightShift = furtherShiftIndex(currRight, currRightShift, currResultIdx);
                    downstream.added().writableCast().insert(currRight);
                    resultRowSet.insert(currRight);
                }
            }
        }
        result.notifyListeners(downstream);
    };
    if (leftTable.isRefreshing() && rightTable.isRefreshing()) {
        final JoinListenerRecorder leftRecorder = new JoinListenerRecorder(true, listenerDescription, leftTable, result);
        final JoinListenerRecorder rightRecorder = new JoinListenerRecorder(false, listenerDescription, rightTable, result);
        final MergedListener mergedListener = new MergedListener(Arrays.asList(leftRecorder, rightRecorder), Collections.emptyList(), listenerDescription, result) {

            @Override
            protected void process() {
                onUpdate.accept(leftRecorder.getUpdate(), rightRecorder.getUpdate());
            }
        };
        leftRecorder.setMergedListener(mergedListener);
        rightRecorder.setMergedListener(mergedListener);
        leftTable.listenForUpdates(leftRecorder);
        rightTable.listenForUpdates(rightRecorder);
        result.addParentReference(mergedListener);
    } else if (leftTable.isRefreshing() && rightTable.size() > 0) {
        leftTable.listenForUpdates(new BaseTable.ListenerImpl(listenerDescription, leftTable, result) {

            @Override
            public void onUpdate(final TableUpdate upstream) {
                onUpdate.accept(upstream, null);
            }
        });
    } else if (rightTable.isRefreshing() && leftTable.size() > 0) {
        rightTable.listenForUpdates(new BaseTable.ListenerImpl(listenerDescription, rightTable, result) {

            @Override
            public void onUpdate(final TableUpdate upstream) {
                onUpdate.accept(null, upstream);
            }
        });
    }
    // Initialize result table.
    try (final WritableRowSet currRight = rightTable.getRowSet().copy()) {
        final MutableLong currRightShift = new MutableLong();
        leftTable.getRowSet().forAllRowKeys((currIdx) -> {
            final long currResultIdx = currIdx << crossJoinState.getNumShiftBits();
            currRightShift.setValue(furtherShiftIndex(currRight, currRightShift.longValue(), currResultIdx));
            resultRowSet.insert(currRight);
        });
    }
    resultRowSet.initializePreviousValue();
    return result;
}

Example 2 with Assert

use of io.deephaven.base.verify.Assert in project deephaven-core by deephaven.

the class StreamTableAggregationTest method doOperatorTest.

/**
 * Execute a table operator ending in an aggregation.
 *
 * @param operator The operator to apply
 * @param windowed Whether the stream table RowSet should be a sliding window (if {@code true}) or zero-based (if
 *        {@code false})
 */
private void doOperatorTest(@NotNull final UnaryOperator<Table> operator, final boolean windowed) {
    final QueryTable normal = new QueryTable(RowSetFactory.empty().toTracking(), source.getColumnSourceMap());
    normal.setRefreshing(true);
    final QueryTable addOnly = (QueryTable) normal.copy();
    addOnly.setAttribute(Table.ADD_ONLY_TABLE_ATTRIBUTE, true);
    final TrackingWritableRowSet streamInternalRowSet;
    final Map<String, ? extends ColumnSource<?>> streamSources;
    if (windowed) {
        streamInternalRowSet = null;
        streamSources = source.getColumnSourceMap();
    } else {
        // Redirecting so we can present a zero-based RowSet from the stream table
        streamInternalRowSet = RowSetFactory.empty().toTracking();
        final WritableRowRedirection streamRedirections = new WrappedRowSetWritableRowRedirection(streamInternalRowSet);
        streamSources = source.getColumnSourceMap().entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, (entry -> new RedirectedColumnSource<>(streamRedirections, entry.getValue())), Assert::neverInvoked, LinkedHashMap::new));
    }
    final QueryTable stream = new QueryTable(RowSetFactory.empty().toTracking(), streamSources);
    stream.setRefreshing(true);
    stream.setAttribute(Table.STREAM_TABLE_ATTRIBUTE, true);
    TstUtils.assertTableEquals(normal, stream);
    final Table expected = operator.apply(normal);
    final Table addOnlyExpected = operator.apply(addOnly);
    final Table streamExpected = operator.apply(stream);
    TstUtils.assertTableEquals(expected, addOnlyExpected);
    TstUtils.assertTableEquals(expected, streamExpected);
    // Aggregation results are never stream tables
    TestCase.assertFalse(((BaseTable) streamExpected).isStream());
    final PrimitiveIterator.OfLong refreshSizes = LongStream.concat(LongStream.of(100, 0, 1, 2, 50, 0, 1000, 1, 0), new Random().longs(0, MAX_RANDOM_ITERATION_SIZE)).iterator();
    int step = 0;
    long usedSize = 0;
    RowSet streamLastInserted = RowSetFactory.empty();
    while (usedSize < INPUT_SIZE) {
        final long refreshSize = Math.min(INPUT_SIZE - usedSize, refreshSizes.nextLong());
        final RowSet normalStepInserted = refreshSize == 0 ? RowSetFactory.empty() : RowSetFactory.fromRange(usedSize, usedSize + refreshSize - 1);
        final RowSet streamStepInserted = streamInternalRowSet == null ? normalStepInserted.copy() : refreshSize == 0 ? RowSetFactory.empty() : RowSetFactory.fromRange(0, refreshSize - 1);
        UpdateGraphProcessor.DEFAULT.startCycleForUnitTests();
        try {
            UpdateGraphProcessor.DEFAULT.refreshUpdateSourceForUnitTests(() -> {
                if (normalStepInserted.isNonempty()) {
                    normal.getRowSet().writableCast().insert(normalStepInserted);
                    normal.notifyListeners(new TableUpdateImpl(normalStepInserted, RowSetFactory.empty(), RowSetFactory.empty(), RowSetShiftData.EMPTY, ModifiedColumnSet.EMPTY));
                }
            });
            final RowSet finalStreamLastInserted = streamLastInserted;
            UpdateGraphProcessor.DEFAULT.refreshUpdateSourceForUnitTests(() -> {
                if (streamStepInserted.isNonempty() || finalStreamLastInserted.isNonempty()) {
                    if (streamInternalRowSet != null) {
                        streamInternalRowSet.clear();
                        streamInternalRowSet.insert(normalStepInserted);
                    }
                    stream.getRowSet().writableCast().clear();
                    stream.getRowSet().writableCast().insert(streamStepInserted);
                    stream.notifyListeners(new TableUpdateImpl(streamStepInserted.copy(), finalStreamLastInserted, RowSetFactory.empty(), RowSetShiftData.EMPTY, ModifiedColumnSet.EMPTY));
                }
            });
        } finally {
            UpdateGraphProcessor.DEFAULT.completeCycleForUnitTests();
        }
        try {
            TstUtils.assertTableEquals(expected, addOnlyExpected);
            TstUtils.assertTableEquals(expected, streamExpected);
        } catch (ComparisonFailure e) {
            System.err.printf("FAILURE: step %d, previousUsedSize %d, refreshSize %d%n", step, usedSize, refreshSize);
            throw e;
        }
        ++step;
        usedSize += refreshSize;
        streamLastInserted = streamStepInserted;
    }
}

Example 3 with Assert

use of io.deephaven.base.verify.Assert in project deephaven-core by deephaven.

the class StreamTableOperationsTest method doOperatorTest.

/**
 * Execute a table operator.
 *
 * @param operator The operator to apply
 * @param windowed Whether the stream table RowSet should be a sliding window (if {@code true}) or zero-based (if
 *        {@code false})
 * @param expectStreamResult Whether the result is expected to be a stream table
 */
private void doOperatorTest(@NotNull final UnaryOperator<Table> operator, final boolean windowed, final boolean expectStreamResult) {
    final QueryTable normal = new QueryTable(RowSetFactory.empty().toTracking(), source.getColumnSourceMap());
    normal.setRefreshing(true);
    final TrackingWritableRowSet streamInternalRowSet;
    final Map<String, ? extends ColumnSource<?>> streamSources;
    if (windowed) {
        streamInternalRowSet = null;
        streamSources = source.getColumnSourceMap();
    } else {
        // Redirecting so we can present a zero-based RowSet from the stream table
        streamInternalRowSet = RowSetFactory.empty().toTracking();
        final WritableRowRedirection streamRedirections = new WrappedRowSetWritableRowRedirection(streamInternalRowSet);
        streamSources = source.getColumnSourceMap().entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, (entry -> new RedirectedColumnSource<>(streamRedirections, entry.getValue())), Assert::neverInvoked, LinkedHashMap::new));
    }
    final QueryTable stream = new QueryTable(RowSetFactory.empty().toTracking(), streamSources);
    stream.setRefreshing(true);
    stream.setAttribute(Table.STREAM_TABLE_ATTRIBUTE, true);
    TstUtils.assertTableEquals(normal, stream);
    final Table expected = operator.apply(normal);
    final Table streamExpected = operator.apply(stream);
    TstUtils.assertTableEquals(expected, streamExpected);
    TestCase.assertEquals(expectStreamResult, ((BaseTable) streamExpected).isStream());
    final PrimitiveIterator.OfLong refreshSizes = LongStream.concat(LongStream.of(100, 0, 1, 2, 50, 0, 1000, 1, 0), new Random().longs(0, MAX_RANDOM_ITERATION_SIZE)).iterator();
    int step = 0;
    long usedSize = 0;
    RowSet normalLastInserted = RowSetFactory.empty();
    RowSet streamLastInserted = RowSetFactory.empty();
    while (usedSize < INPUT_SIZE) {
        final long refreshSize = Math.min(INPUT_SIZE - usedSize, refreshSizes.nextLong());
        final RowSet normalStepInserted = refreshSize == 0 ? RowSetFactory.empty() : RowSetFactory.fromRange(usedSize, usedSize + refreshSize - 1);
        final RowSet streamStepInserted = streamInternalRowSet == null ? normalStepInserted.copy() : refreshSize == 0 ? RowSetFactory.empty() : RowSetFactory.fromRange(0, refreshSize - 1);
        UpdateGraphProcessor.DEFAULT.startCycleForUnitTests();
        try {
            final RowSet finalNormalLastInserted = normalLastInserted;
            UpdateGraphProcessor.DEFAULT.refreshUpdateSourceForUnitTests(() -> {
                if (normalStepInserted.isNonempty() || finalNormalLastInserted.isNonempty()) {
                    normal.getRowSet().writableCast().update(normalStepInserted, finalNormalLastInserted);
                    normal.notifyListeners(new TableUpdateImpl(normalStepInserted.copy(), finalNormalLastInserted, RowSetFactory.empty(), RowSetShiftData.EMPTY, ModifiedColumnSet.EMPTY));
                }
            });
            final RowSet finalStreamLastInserted = streamLastInserted;
            UpdateGraphProcessor.DEFAULT.refreshUpdateSourceForUnitTests(() -> {
                if (streamStepInserted.isNonempty() || finalStreamLastInserted.isNonempty()) {
                    if (streamInternalRowSet != null) {
                        streamInternalRowSet.clear();
                        streamInternalRowSet.insert(normalStepInserted);
                    }
                    stream.getRowSet().writableCast().clear();
                    stream.getRowSet().writableCast().insert(streamStepInserted);
                    stream.notifyListeners(new TableUpdateImpl(streamStepInserted.copy(), finalStreamLastInserted, RowSetFactory.empty(), RowSetShiftData.EMPTY, ModifiedColumnSet.EMPTY));
                }
            });
        } finally {
            UpdateGraphProcessor.DEFAULT.completeCycleForUnitTests();
        }
        try {
            TstUtils.assertTableEquals(expected, streamExpected);
        } catch (ComparisonFailure e) {
            System.err.printf("FAILURE: step %d, previousUsedSize %d, refreshSize %d%n", step, usedSize, refreshSize);
            throw e;
        }
        ++step;
        usedSize += refreshSize;
        normalLastInserted = normalStepInserted;
        streamLastInserted = streamStepInserted;
    }
}

Example 4 with Assert

use of io.deephaven.base.verify.Assert in project deephaven-core by deephaven.

the class TestPartitionAwareSourceTable method doTestRedefinition.

private void doTestRedefinition() {
    // Note: We expect redefinition to make a new CSM, but no work until we force a coalesce by asking for column
    // sources
    final List<ColumnDefinition<?>> includedColumns1 = List.of(PARTITIONING_COLUMN_DEFINITION, CHARACTER_COLUMN_DEFINITION, INTEGER_COLUMN_DEFINITION, DOUBLE_COLUMN_DEFINITION);
    final Map<Class, ColumnSource> dataTypeToColumnSource = new HashMap<>();
    includedColumns1.forEach((final ColumnDefinition columnDefinition) -> {
        final ColumnSource columnSource = mock(ColumnSource.class, "_CS_" + columnDefinition.getDataType().getSimpleName());
        dataTypeToColumnSource.put(columnDefinition.getDataType(), columnSource);
        checking(new Expectations() {

            {
                allowing(columnSource).getType();
                will(returnValue(columnDefinition.getDataType()));
                allowing(columnSource).getComponentType();
                will(returnValue(columnDefinition.getComponentType()));
                allowing(columnSource).preventsParallelism();
                will(returnValue(false));
                allowing(columnSource).isStateless();
                will(returnValue(true));
            }
        });
    });
    // Test 1: Drop a column
    // Setup the table
    checking(new Expectations() {

        {
            oneOf(componentFactory).createColumnSourceManager(with(true), with(ColumnToCodecMappings.EMPTY), with(equal(includedColumns1)));
            will(returnValue(columnSourceManager));
            oneOf(columnSourceManager).disableGrouping();
        }
    });
    final Table dropColumnsResult1 = SUT.dropColumns(BOOLEAN_COLUMN_DEFINITION.getName());
    assertIsSatisfied();
    assertTrue(dropColumnsResult1 instanceof PartitionAwareSourceTable);
    // Force a coalesce and make sure it has the right columns
    checking(new Expectations() {

        {
            oneOf(locationProvider).subscribe(with(any(TableLocationProvider.Listener.class)));
            will(new CustomAction("Supply no locations") {

                @Override
                public Object invoke(Invocation invocation) {
                    return null;
                }
            });
            oneOf(columnSourceManager).refresh();
            will(returnValue(RowSetFactory.empty()));
            oneOf(columnSourceManager).getColumnSources();
            will(returnValue(includedColumns1.stream().collect(Collectors.toMap(ColumnDefinition::getName, cd -> dataTypeToColumnSource.get(cd.getDataType()), Assert::neverInvoked, LinkedHashMap::new))));
        }
    });
    assertEquals(NUM_COLUMNS - 1, dropColumnsResult1.getColumnSources().size());
    assertIsSatisfied();
    assertNotNull(dropColumnsResult1.getColumnSource(CHARACTER_COLUMN_DEFINITION.getName()));
    assertNotNull(dropColumnsResult1.getColumnSource(INTEGER_COLUMN_DEFINITION.getName()));
    assertNotNull(dropColumnsResult1.getColumnSource(DOUBLE_COLUMN_DEFINITION.getName()));
    // Test 2: Drop another column
    // Setup the table
    final List<ColumnDefinition<?>> includedColumns2 = List.of(PARTITIONING_COLUMN_DEFINITION, INTEGER_COLUMN_DEFINITION, DOUBLE_COLUMN_DEFINITION);
    checking(new Expectations() {

        {
            oneOf(componentFactory).createColumnSourceManager(with(true), with(ColumnToCodecMappings.EMPTY), with(equal(includedColumns2)));
            will(returnValue(columnSourceManager));
            oneOf(columnSourceManager).disableGrouping();
        }
    });
    final Table dropColumnsResult2 = dropColumnsResult1.dropColumns(CHARACTER_COLUMN_DEFINITION.getName());
    assertIsSatisfied();
    assertTrue(dropColumnsResult2 instanceof PartitionAwareSourceTable);
    // Force a coalesce and make sure it has the right columns
    checking(new Expectations() {

        {
            oneOf(locationProvider).subscribe(with(any(TableLocationProvider.Listener.class)));
            will(new CustomAction("Supply no locations") {

                @Override
                public Object invoke(Invocation invocation) {
                    return null;
                }
            });
            oneOf(columnSourceManager).refresh();
            will(returnValue(RowSetFactory.empty()));
            oneOf(columnSourceManager).getColumnSources();
            will(returnValue(includedColumns2.stream().collect(Collectors.toMap(ColumnDefinition::getName, cd -> dataTypeToColumnSource.get(cd.getDataType()), Assert::neverInvoked, LinkedHashMap::new))));
        }
    });
    assertEquals(NUM_COLUMNS - 2, dropColumnsResult2.getColumnSources().size());
    assertIsSatisfied();
    assertNotNull(dropColumnsResult2.getColumnSource(INTEGER_COLUMN_DEFINITION.getName()));
    assertNotNull(dropColumnsResult2.getColumnSource(DOUBLE_COLUMN_DEFINITION.getName()));
    // Test 3: Rename a column
    // Nothing to setup for the table - the rename is deferred
    final Table renameColumnsResult1 = dropColumnsResult2.renameColumns("A=" + INTEGER_COLUMN_DEFINITION.getName());
    assertIsSatisfied();
    assertTrue(renameColumnsResult1 instanceof DeferredViewTable);
    // This will not force a coalesce, as dropColumnsResult2 is already coalesced.
    assertEquals(NUM_COLUMNS - 2, renameColumnsResult1.getColumnSources().size());
    assertIsSatisfied();
    assertNotNull(renameColumnsResult1.getColumnSource("A"));
    assertNotNull(renameColumnsResult1.getColumnSource(DOUBLE_COLUMN_DEFINITION.getName()));
    // Test 4: Use view to slice us down to one column
    // Setup the table
    final List<ColumnDefinition<?>> includedColumns3 = List.of(INTEGER_COLUMN_DEFINITION, PARTITIONING_COLUMN_DEFINITION);
    checking(new Expectations() {

        {
            oneOf(componentFactory).createColumnSourceManager(with(true), with(ColumnToCodecMappings.EMPTY), with(equal(includedColumns3)));
            will(returnValue(columnSourceManager));
            oneOf(columnSourceManager).disableGrouping();
        }
    });
    final Table viewResult1 = dropColumnsResult2.view(INTEGER_COLUMN_DEFINITION.getName());
    assertIsSatisfied();
    assertTrue(viewResult1 instanceof DeferredViewTable);
    // Force a coalesce and make sure it has the right columns
    checking(new Expectations() {

        {
            oneOf(locationProvider).subscribe(with(any(TableLocationProvider.Listener.class)));
            will(new CustomAction("Supply no locations") {

                @Override
                public Object invoke(Invocation invocation) {
                    return null;
                }
            });
            oneOf(columnSourceManager).refresh();
            will(returnValue(RowSetFactory.empty()));
            oneOf(columnSourceManager).getColumnSources();
            will(returnValue(includedColumns3.stream().collect(Collectors.toMap(ColumnDefinition::getName, cd -> dataTypeToColumnSource.get(cd.getDataType()), Assert::neverInvoked, LinkedHashMap::new))));
        }
    });
    assertEquals(NUM_COLUMNS - 4, viewResult1.getColumnSources().size());
    assertIsSatisfied();
    assertNotNull(viewResult1.getColumnSource(INTEGER_COLUMN_DEFINITION.getName()));
    // Test 5: Add a new derived column on
    // Setup the table
    final Table viewResult2 = viewResult1.updateView("SizeSquared=" + INTEGER_COLUMN_DEFINITION.getName() + '*' + INTEGER_COLUMN_DEFINITION.getName());
    assertTrue(viewResult2 instanceof DeferredViewTable);
    assertEquals(NUM_COLUMNS - 3, viewResult2.getColumnSources().size());
    assertNotNull(viewResult2.getColumnSource(INTEGER_COLUMN_DEFINITION.getName()));
    assertNotNull(viewResult2.getColumnSource("SizeSquared"));
    assertIsSatisfied();
    final Table viewResult3 = viewResult2.view("Result=SizeSquared");
    assertTrue(viewResult3 instanceof DeferredViewTable);
    assertEquals(NUM_COLUMNS - 4, viewResult3.getColumnSources().size());
    assertNotNull(viewResult3.getColumnSource("Result"));
    assertIsSatisfied();
    final Table viewResult4 = viewResult2.view("SizeSquared");
    assertTrue(viewResult4 instanceof DeferredViewTable);
    assertEquals(NUM_COLUMNS - 4, viewResult4.getColumnSources().size());
    assertNotNull(viewResult4.getColumnSource("SizeSquared"));
    assertIsSatisfied();
}

Example 5 with Assert

use of io.deephaven.base.verify.Assert in project deephaven-core by deephaven.

the class QueryTable method rollup.

@Override
public Table rollup(Collection<? extends Aggregation> aggregations, boolean includeConstituents, Selectable... groupByColumns) {
    if (isStream() && includeConstituents) {
        throw streamUnsupported("rollup with included constituents");
    }
    final SelectColumn[] gbsColumns = SelectColumn.from(groupByColumns);
    final MemoizedOperationKey rollupKey = MemoizedOperationKey.rollup(aggregations, gbsColumns, includeConstituents);
    return memoizeResult(rollupKey, () -> {
        final QueryTable baseLevel = aggNoMemo(AggregationProcessor.forRollupBase(aggregations, includeConstituents), gbsColumns);
        final Deque<SelectColumn> gbsColumnsToReaggregate = new ArrayDeque<>(Arrays.asList(gbsColumns));
        final Deque<String> nullColumnNames = new ArrayDeque<>(groupByColumns.length);
        QueryTable lastLevel = baseLevel;
        while (!gbsColumnsToReaggregate.isEmpty()) {
            nullColumnNames.addFirst(gbsColumnsToReaggregate.removeLast().getName());
            final TableDefinition lastLevelDefinition = lastLevel.getDefinition();
            final Map<String, Class<?>> nullColumns = nullColumnNames.stream().collect(Collectors.toMap(Function.identity(), ncn -> lastLevelDefinition.getColumn(ncn).getDataType(), Assert::neverInvoked, LinkedHashMap::new));
            lastLevel = lastLevel.aggNoMemo(AggregationProcessor.forRollupReaggregated(aggregations, nullColumns), gbsColumnsToReaggregate.toArray(SelectColumn.ZERO_LENGTH_SELECT_COLUMN_ARRAY));
        }
        final String[] internalColumnsToDrop = lastLevel.getDefinition().getColumnStream().map(ColumnDefinition::getName).filter(cn -> cn.endsWith(ROLLUP_COLUMN_SUFFIX)).toArray(String[]::new);
        final QueryTable finalTable = (QueryTable) lastLevel.dropColumns(internalColumnsToDrop);
        final Object reverseLookup = Require.neqNull(lastLevel.getAttribute(REVERSE_LOOKUP_ATTRIBUTE), "REVERSE_LOOKUP_ATTRIBUTE");
        finalTable.setAttribute(Table.REVERSE_LOOKUP_ATTRIBUTE, reverseLookup);
        final Table result = HierarchicalTable.createFrom(finalTable, new RollupInfo(aggregations, gbsColumns, includeConstituents ? RollupInfo.LeafType.Constituent : RollupInfo.LeafType.Normal));
        result.setAttribute(Table.HIERARCHICAL_SOURCE_TABLE_ATTRIBUTE, QueryTable.this);
        copyAttributes(result, CopyAttributeOperation.Rollup);
        maybeUpdateSortableColumns(result);
        return result;
    });
}