Examples with RowIterator org.knime.core.data.RowIterator used on opensource projects

Example 16 with RowIterator

use of org.knime.core.data.RowIterator in project knime-core by knime.

the class DataContainerTest method testMemoryAlertWhileRestore.

public final void testMemoryAlertWhileRestore() throws Exception {
    DataContainer container = new DataContainer(SPEC_STR_INT_DBL, true, /* no rows in mem */
    0, false);
    int count = 100000;
    for (RowIterator it = generateRows(count); it.hasNext(); ) {
        container.addRowToTable(it.next());
    }
    container.close();
    final Buffer buffer = container.getBufferedTable().getBuffer();
    assertTrue(buffer.usesOutFile());
    buffer.restoreIntoMemory();
    RowIterator tableIterator1 = container.getTable().iterator();
    RowIterator tableIterator2 = container.getTable().iterator();
    RowIterator referenceIterator = generateRows(count);
    int i;
    for (i = 0; i < count; i++) {
        if (i == count / 2) {
            synchronized (buffer) {
                // currently it does nothing as memory alerts while restoring is not supported
                MemoryAlertSystem.getInstance().sendMemoryAlert();
            }
        }
        RowIterator pushIterator, otherIterator;
        if (i % 2 == 0) {
            pushIterator = tableIterator1;
            otherIterator = tableIterator2;
        } else {
            pushIterator = tableIterator2;
            otherIterator = tableIterator1;
        }
        DataRow pushRow = pushIterator.next();
        DataRow otherRow = otherIterator.next();
        DataRow referenceRow = referenceIterator.next();
        assertEquals(referenceRow, pushRow);
        assertEquals(referenceRow, otherRow);
    }
    assertFalse(buffer.usesOutFile());
    assertFalse(tableIterator1.hasNext());
    assertFalse(tableIterator2.hasNext());
    Thread restoreThread = new Thread(new Runnable() {

        @Override
        public void run() {
            MemoryAlertSystem.getInstance().sendMemoryAlert();
        }
    }, "Buffer restore");
    tableIterator1 = container.getTable().iterator();
    referenceIterator = generateRows(count);
    for (i = 0; i < count; i++) {
        if (i == 10) {
            restoreThread.start();
        }
        DataRow row = tableIterator1.next();
        DataRow referenceRow = referenceIterator.next();
        assertEquals(referenceRow, row);
    }
    restoreThread.join();
    assertFalse(buffer.usesOutFile());
}

Example 17 with RowIterator

use of org.knime.core.data.RowIterator in project knime-core by knime.

the class DataContainerTest method testMemoryAlertAfterClose.

public final void testMemoryAlertAfterClose() throws Exception {
    DataContainer container = new DataContainer(SPEC_STR_INT_DBL, true, Integer.MAX_VALUE, false);
    for (RowIterator it = generateRows(100000); it.hasNext(); ) {
        container.addRowToTable(it.next());
    }
    container.close();
    Buffer buffer = container.getBufferedTable().getBuffer();
    MemoryAlertSystem.getInstance().sendMemoryAlert();
    RowIterator tableIterator = container.getTable().iterator();
    for (RowIterator it = generateRows(100000); it.hasNext(); ) {
        assertEquals(it.next(), tableIterator.next());
    }
}

Example 18 with RowIterator

use of org.knime.core.data.RowIterator in project knime-core by knime.

the class DefaultTableTest method testGetRowIterator.

// testGetColumnType()
/**
 * Tests the row iterator in the table.
 *
 * @see DefaultTable#iterator()
 */
public final void testGetRowIterator() {
    /*
         * also for empty table the iterator must not be null (but atEnd()
         * immediately
         */
    DefaultTable empty = new DefaultTable(new Object[][] {}, new String[] {}, CASE_1_COLHEADER);
    RowIterator emptyIt = empty.iterator();
    assertNotNull(emptyIt);
    assertTrue(!emptyIt.hasNext());
    /*
         * create table from CASE4, get data back from row iterator and compare
         * input and output
         */
    DefaultTable case4 = new DefaultTable(CASE_4_CONTENT, CASE_4_ROWHEADER, CASE_4_COLHEADER);
    DataTableSpec spec4 = case4.getDataTableSpec();
    Vector<Object[]> content = // contains Object[] (row-wise)
    new Vector<Object[]>();
    for (DataRow row : case4) {
        assertTrue(row.getNumCells() == spec4.getNumColumns());
        Object[] rowContent = new Object[row.getNumCells()];
        for (int c = 0; c < row.getNumCells(); c++) {
            rowContent[c] = getObjectFromCell(row.getCell(c));
        }
        content.add(rowContent);
    }
    Object[][] contentAsArray = new Object[content.size()][];
    contentAsArray = content.toArray(contentAsArray);
    // contentAsArray.equals(CASE_4_CONTENT) does not work!!
    assertTrue(contentAsArray.length == CASE_4_CONTENT.length);
    for (int i = 0; i < contentAsArray.length; i++) {
        assertTrue(Arrays.equals(contentAsArray[i], CASE_4_CONTENT[i]));
    }
}

Example 19 with RowIterator

use of org.knime.core.data.RowIterator in project knime-core by knime.

the class TableSorterTest method runMemoryTest.

private void runMemoryTest(final int numRows, final int maxNumRowsPerContainer, final int maxOpenContainers) throws CanceledExecutionException {
    // Create data with fields that consume a lot memory
    DataTable inputTable = new TestData(numRows, 1);
    BufferedDataTable bdt = m_exec.createBufferedDataTable(inputTable, m_exec);
    BufferedDataTableSorter sorter = new BufferedDataTableSorter(bdt, Arrays.asList("Index"), new boolean[] { true });
    sorter.setMaxOpenContainers(maxOpenContainers);
    BufferedDataTable defaultResult = sorter.sort(m_exec);
    sorter.setMaxRows(maxNumRowsPerContainer);
    // 10MB free memory
    long currentlyUsed = MemoryAlertSystem.getUsedMemory();
    double fraction = Math.min(1, (currentlyUsed + (10 << 20)) / (double) MemoryAlertSystem.getMaximumMemory());
    MemoryAlertSystem.getInstance().setFractionUsageThreshold(fraction);
    try {
        sorter.setMemService(MemoryAlertSystem.getInstance());
        // run again with change settings
        BufferedDataTable result = sorter.sort(m_exec);
        // Check if column is sorted in ascending order
        int prevValue = Integer.MIN_VALUE;
        for (DataRow row : result) {
            int thisValue = ((IntValue) row.getCell(0)).getIntValue();
            Assert.assertTrue(thisValue >= prevValue);
        }
        // Check if it has the same results as defaultResult
        Assert.assertTrue(defaultResult.getRowCount() == result.getRowCount());
        RowIterator defaultIter = defaultResult.iterator();
        RowIterator iter = result.iterator();
        while (defaultIter.hasNext()) {
            DataRow defaultRow = defaultIter.next();
            DataRow row = iter.next();
            Assert.assertTrue(defaultRow.getKey().getString().equals(row.getKey().getString()));
            Iterator<DataCell> defaultCellIter = defaultRow.iterator();
            Iterator<DataCell> cellIter = row.iterator();
            while (defaultCellIter.hasNext()) {
                Assert.assertTrue(defaultCellIter.next().equals(cellIter.next()));
            }
        }
    } finally {
        MemoryAlertSystem.getInstance().setFractionUsageThreshold(MemoryAlertSystem.DEFAULT_USAGE_THRESHOLD);
    }
}

Example 20 with RowIterator

use of org.knime.core.data.RowIterator in project knime-core by knime.

the class Buffer method addToZipFile.

/**
 * Method that's been called from the {@link ContainerTable} to save the content. It will add zip entries to the
 * <code>zipOut</code> argument and not close the output stream when done, allowing to add additional content
 * elsewhere (for instance the <code>DataTableSpec</code>).
 *
 * @param zipOut To write to.
 * @param exec For progress/cancel
 * @throws IOException If it fails to write to a file.
 * @throws CanceledExecutionException If canceled.
 * @see org.knime.core.node.BufferedDataTable.KnowsRowCountTable #saveToFile(File, NodeSettingsWO, ExecutionMonitor)
 */
synchronized void addToZipFile(final ZipOutputStream zipOut, final ExecutionMonitor exec) throws IOException, CanceledExecutionException {
    if (m_spec == null) {
        throw new IOException("Can't save an open Buffer.");
    }
    // binary data is already deflated
    if (ZLIB_SUPPORTS_LEVEL_SWITCH_AP8083) {
        zipOut.setLevel(Deflater.NO_COMPRESSION);
    }
    zipOut.putNextEntry(new ZipEntry(ZIP_ENTRY_DATA));
    if (!usesOutFile() || m_version < IVERSION) {
        // need to use new buffer since we otherwise write properties
        // of this buffer, which prevents it from further reading (version
        // conflict) - see bug #1364
        Buffer copy = createLocalCloneForWriting();
        File tempFile = null;
        try {
            copy.initOutputWriter(new NonClosableOutputStream.Zip(zipOut));
        } catch (UnsupportedOperationException notSupported) {
            tempFile = DataContainer.createTempFile(copy.m_outputFormat.getFilenameSuffix());
            copy.initOutputWriter(tempFile);
        }
        int count = 1;
        for (RowIterator it = iterator(); it.hasNext(); ) {
            final BlobSupportDataRow row = (BlobSupportDataRow) it.next();
            final int countCurrent = count;
            exec.setProgress(count / (double) size(), () -> "Writing row " + countCurrent + " (\"" + row.getKey() + "\")");
            exec.checkCanceled();
            // make a deep copy of blobs if we have a version hop
            copy.addRow(row, m_version < IVERSION, false);
            count++;
        }
        synchronized (copy) {
            copy.closeInternal();
        }
        if (tempFile != null) {
            try (InputStream in = new FileInputStream(tempFile)) {
                IOUtils.copyLarge(in, new NonClosableOutputStream(zipOut));
            } finally {
                tempFile.delete();
            }
        }
        // in this if-statement
        if (usesOutFile()) {
            // can safely be set to null because it wrote to stream already
            copy.m_list = null;
        }
        File blobDir = m_blobDir;
        // (otherwise its blob dir will be empty
        if (m_version < IVERSION) {
            blobDir = copy.m_blobDir;
        } else {
            assert copy.m_blobDir == null;
        }
        if (blobDir != null) {
            addToZip(ZIP_ENTRY_BLOBS, zipOut, blobDir);
        }
        if (hasOwnFileStoreCells()) {
            addToZip(ZIP_ENTRY_FILESTORES, zipOut, getOwnFileStoreCellsDirectory());
        }
        zipOut.closeEntry();
        if (ZLIB_SUPPORTS_LEVEL_SWITCH_AP8083) {
            zipOut.setLevel(Deflater.DEFAULT_COMPRESSION);
        }
        zipOut.putNextEntry(new ZipEntry(ZIP_ENTRY_META));
        copy.writeMetaToFile(new NonClosableOutputStream.Zip(zipOut));
    } else {
        // does the buffering itself
        try (InputStream is = new FileInputStream(m_binFile)) {
            FileUtil.copy(is, zipOut);
        }
        if (m_blobDir != null) {
            addToZip(ZIP_ENTRY_BLOBS, zipOut, m_blobDir);
        }
        if (hasOwnFileStoreCells()) {
            addToZip(ZIP_ENTRY_FILESTORES, zipOut, getOwnFileStoreCellsDirectory());
        }
        zipOut.closeEntry();
        if (ZLIB_SUPPORTS_LEVEL_SWITCH_AP8083) {
            zipOut.setLevel(Deflater.DEFAULT_COMPRESSION);
        }
        zipOut.putNextEntry(new ZipEntry(ZIP_ENTRY_META));
        writeMetaToFile(new NonClosableOutputStream.Zip(zipOut));
    }
}