Examples with Table io.deephaven.engine.table.Table used on opensource projects

Example 1 with Table

use of io.deephaven.engine.table.Table in project deephaven-core by deephaven.

the class SourceTableMap method sortAndAddLocations.

private void sortAndAddLocations(@NotNull final Stream<TableLocation> locations) {
    // final value we can use to detect not-created tables
    final MutableBoolean observeCreation = new MutableBoolean(false);
    locations.sorted(Comparator.comparing(TableLocation::getKey)).forEach(tl -> {
        observeCreation.setValue(false);
        final Table previousTable = computeIfAbsent(tl.getKey(), o -> {
            observeCreation.setValue(true);
            return makeTable(tl);
        });
        if (!observeCreation.getValue()) {
            // we have a duplicate location - not allowed
            final TableLocation previousLocation = ((PartitionAwareSourceTable) previousTable).locationProvider.getTableLocation(tl.getKey());
            throw new TableDataException("Data Routing Configuration error: TableDataService elements overlap at location " + tl.toGenericString() + ". Duplicate locations are " + previousLocation.toStringDetailed() + " and " + tl.toStringDetailed());
        }
    });
}

Example 2 with Table

use of io.deephaven.engine.table.Table in project deephaven-core by deephaven.

the class PerformanceQueries method processMemory.

/**
 * A user friendly view with basic memory and GC data samples for the current engine process.
 *
 * @return a view on ProcessMemoryLog.
 */
@ScriptApi
public static Table processMemory() {
    final long maxMemoryBytes = RuntimeMemory.getInstance().getMaxMemory();
    final int maxMemoryMiB = (int) Math.ceil(maxMemoryBytes / (1024 * 1024.0));
    final Table maxMem = TableTools.newTable(TableTools.intCol("MaxMemMiB", maxMemoryMiB));
    final Table pml = TableLoggers.processMemoryLog().naturalJoin(maxMem, "");
    Table pm = pml.updateView("TotalMemMiB = (int) Math.ceil(TotalMemory / (1024 * 1024.0))", "FreeMemMiB = (int) Math.ceil(FreeMemory / (1024 * 1024.0))");
    pm = pm.view("IntervalStart = IntervalStartTime", "IntervalSeconds = IntervalDurationNanos / (1000 * 1000 * 1000.0)", "UsedMemMiB = TotalMemMiB - FreeMemMiB", "AvailMemMiB = MaxMemMiB - TotalMemMiB + FreeMemMiB", "MaxMemMiB", "AvailMemRatio = AvailMemMiB/MaxMemMiB", "GcTimeRatio = io.deephaven.engine.table.impl.util.PerformanceQueries.approxRatio(IntervalCollectionTimeNanos, IntervalDurationNanos)");
    pm = pm.formatColumns("AvailMemRatio=Decimal(`#0.0%`)", "AvailMemRatio=(AvailMemRatio < 0.05) ? PALE_RED : " + "((AvailMemRatio < 0.10) ? PALE_REDPURPLE : " + "((AvailMemRatio < 0.20) ? PALE_PURPLE : NO_FORMATTING))", "GcTimeRatio=Decimal(`#0.0%`)", "GcTimeRatio=(GcTimeRatio >= 0.75) ? PALE_RED : " + "((GcTimeRatio >= 0.50) ? PALE_REDPURPLE : " + "((GcTimeRatio > 0.05) ? PALE_PURPLE : NO_FORMATTING))", "IntervalSeconds=Decimal(`#0.000`)");
    return pm;
}

Example 3 with Table

use of io.deephaven.engine.table.Table in project deephaven-core by deephaven.

the class TableMapTest method testCrossDependencies2.

public void testCrossDependencies2() {
    UpdateGraphProcessor.DEFAULT.resetForUnitTests(false, true, 0, 2, 0, 0);
    final QueryTable sourceTable = TstUtils.testRefreshingTable(i(1, 2).toTracking(), c("USym", "aa", "bb"), c("Sentinel", 10, 20));
    final QueryTable sourceTable2 = TstUtils.testRefreshingTable(i(3, 5, 9).toTracking(), c("USym2", "aa", "bb", "dd"), c("Sentinel2", 30, 50, 90));
    final TableMap result = sourceTable.partitionBy("USym");
    final PauseHelper pauseHelper = new PauseHelper();
    QueryScope.addParam("pauseHelper", pauseHelper);
    pauseHelper.release();
    final TableMap result2 = sourceTable2.partitionBy("USym2").transformTables(t -> t.update("SlowItDown2=pauseHelper.pauseValue(2 * k)"));
    final TableMap joined = result.transformTablesWithMap(result2, (l, r) -> {
        System.out.println("Doing naturalJoin");
        return l.naturalJoin(r, "USym=USym2");
    });
    final Table merged = joined.merge();
    pauseHelper.pause();
    UpdateGraphProcessor.DEFAULT.startCycleForUnitTests();
    addToTable(sourceTable, i(5), c("USym", "dd"), c("Sentinel", 50));
    addToTable(sourceTable2, i(10), c("USym2", "dd"), c("Sentinel2", 100));
    removeRows(sourceTable2, i(9));
    System.out.println("Launching Notifications");
    sourceTable.notifyListeners(i(5), i(), i());
    sourceTable2.notifyListeners(i(10), i(9), i());
    System.out.println("Waiting for notifications");
    new Thread(() -> {
        System.out.println("Sleeping before release.");
        SleepUtil.sleep(1000);
        System.out.println("Doing release.");
        pauseHelper.release();
        System.out.println("Released.");
    }).start();
    UpdateGraphProcessor.DEFAULT.completeCycleForUnitTests();
    TableTools.showWithRowSet(merged);
}

Example 4 with Table

use of io.deephaven.engine.table.Table in project deephaven-core by deephaven.

the class TableMapTest method testTableMapSupplierListeners.

public void testTableMapSupplierListeners() {
    final QueryTable base = TstUtils.testRefreshingTable(i(0, 1, 2, 3, 4, 5).toTracking(), stringCol("Key", "Zero", "Zero", "One", "One", "One", "One"), stringCol("Color", "Red", "Blue", "Red", "Blue", "Red", "Blue"), intCol("Value", -1, 0, 1, 2, 3, 4));
    final TableMap byKey = base.partitionBy("Key");
    final TableMapSupplier supplier = new TableMapSupplier(byKey, Collections.singletonList(t -> t.where("Color=`Red`")));
    assertTableEquals(base.where("Color=`Red`"), supplier.merge());
    final Map<String, Table> listenerResults = new HashMap<>();
    supplier.addListener((key, table) -> listenerResults.put((String) key, table));
    UpdateGraphProcessor.DEFAULT.runWithinUnitTestCycle(() -> {
        final RowSet idx = i(6, 7, 8, 9);
        addToTable(base, idx, stringCol("Key", "Two", "Two", "Two", "Two"), stringCol("Color", "Red", "Blue", "Red", "Blue"), intCol("Value", 5, 6, 7, 8));
        base.notifyListeners(idx, i(), i());
    });
    UpdateGraphProcessor.DEFAULT.runWithinUnitTestCycle(() -> {
        final RowSet idx = i(10, 11, 12, 13);
        addToTable(base, idx, stringCol("Key", "Three", "Three", "Three", "Three"), stringCol("Color", "Red", "Red", "Red", "Blue"), intCol("Value", 9, 10, 11, 12));
        base.notifyListeners(idx, i(), i());
    });
    UpdateGraphProcessor.DEFAULT.runWithinUnitTestCycle(() -> {
        final RowSet idx = i(14, 15, 16, 17);
        addToTable(base, idx, stringCol("Key", "Four", "Four", "Four", "Four"), stringCol("Color", "Blue", "Blue", "Blue", "Blue"), intCol("Value", 13, 14, 15, 16));
        base.notifyListeners(idx, i(), i());
    });
    UpdateGraphProcessor.DEFAULT.runWithinUnitTestCycle(() -> {
        final RowSet idx = i(18, 19, 20, 21);
        addToTable(base, idx, stringCol("Key", "Four", "Four", "Four", "Four"), stringCol("Color", "Blue", "Blue", "Blue", "Blue"), intCol("Value", 9, 10, 11, 12));
        base.notifyListeners(idx, i(), i());
    });
    assertTableEquals(base.where("Key=`Two`", "Color=`Red`"), listenerResults.get("Two"));
    assertTableEquals(base.where("Key=`Three`", "Color=`Red`"), listenerResults.get("Three"));
    assertTableEquals(base.where("Key=`Four`", "Color=`Red`"), listenerResults.get("Four"));
}

Example 5 with Table

use of io.deephaven.engine.table.Table in project deephaven-core by deephaven.

the class TableMapTest method testAsTable.

public void testAsTable() {
    final Random random = new Random(0);
    final int size = 100;
    final TstUtils.ColumnInfo[] columnInfo;
    final QueryTable table = getTable(size, random, columnInfo = initColumnInfos(new String[] { "Sym", "intCol", "doubleCol", "Indices" }, new TstUtils.SetGenerator<>("aa", "bb", "cc", "dd"), new TstUtils.IntGenerator(0, 20), new TstUtils.DoubleGenerator(0, 100), new TstUtils.SortedLongGenerator(0, Long.MAX_VALUE - 1)));
    final Table withK = table.update("K=Indices");
    final QueryTable rightTable = getTable(size, random, initColumnInfos(new String[] { "Sym", "RightCol" }, new TstUtils.SetGenerator<>("aa", "bb", "cc", "dd"), new TstUtils.IntGenerator(100, 200)));
    final TableMap map = withK.partitionBy("Sym");
    map.populateKeys("aa", "bb", "cc", "dd");
    final Table asTable = map.asTable(false, true, false);
    final TableMap rightMap = rightTable.partitionBy("Sym");
    rightMap.populateKeys("aa", "bb", "cc", "dd");
    final Table rightAsTable = rightMap.asTable(false, true, false);
    final EvalNuggetInterface[] en = new EvalNuggetInterface[] { new EvalNugget() {

        public Table e() {
            return ((TransformableTableMap) table.update("K=Indices").partitionBy("Sym").populateKeys("aa", "bb", "cc", "dd").asTable(false, false, false).update("K2=Indices*2").select("K", "K2", "Half=doubleCol/2", "Sq=doubleCol*doubleCol", "Weight=intCol*doubleCol", "Sym")).merge().sort("K", "Sym");
        }
    }, new SizeNugget(table, asTable), new QueryTableTest.TableComparator(withK.naturalJoin(rightTable.lastBy("Sym"), "Sym").sort("K", "Sym"), asTable.naturalJoin(rightTable.lastBy("Sym"), "Sym").coalesce().sort("K", "Sym")), new QueryTableTest.TableComparator(withK.naturalJoin(rightTable.lastBy("Sym"), "Sym").sort("K", "Sym"), asTable.naturalJoin(rightAsTable.lastBy(), "Sym").coalesce().sort("K", "Sym")) };
    for (int i = 0; i < 100; i++) {
        simulateShiftAwareStep(size, random, table, columnInfo, en);
    }
}