Examples with FileStore org.knime.core.data.filestore.FileStore used on opensource projects

Example 1 with FileStore

use of org.knime.core.data.filestore.FileStore in project knime-core by knime.

the class PortUtil method writeObjectToStream.

/**
 * Write the given port object into the given output stream. The output stream does not need to be buffered and is
 * not closed after calling this method.
 *
 * @param po any port object
 * @param output any output stream, does not need to be buffered
 * @param exec execution context for reporting progress and checking for cancelation
 * @throws IOException if an I/O error occurs while serializing the port object
 * @throws CanceledExecutionException if the user canceled the operation
 */
public static void writeObjectToStream(final PortObject po, final OutputStream output, final ExecutionMonitor exec) throws IOException, CanceledExecutionException {
    final boolean originalOutputIsBuffered = ((output instanceof BufferedOutputStream) || (output instanceof ByteArrayOutputStream));
    OutputStream os = originalOutputIsBuffered ? output : new BufferedOutputStream(output);
    final ZipOutputStream zipOut = new ZipOutputStream(os);
    PortObjectSpec spec = po.getSpec();
    zipOut.putNextEntry(new ZipEntry("content.xml"));
    ModelContent toc = new ModelContent("content");
    toc.addInt("version", 1);
    toc.addString("port_spec_class", spec.getClass().getName());
    toc.addString("port_object_class", po.getClass().getName());
    NotInWorkflowWriteFileStoreHandler fileStoreHandler = null;
    if (po instanceof FileStorePortObject) {
        fileStoreHandler = NotInWorkflowWriteFileStoreHandler.create();
        ModelContentWO fileStoreModelContent = toc.addModelContent("filestores");
        fileStoreModelContent.addString("handlerUUID", fileStoreHandler.getStoreUUID().toString());
        final FileStorePortObject fileStorePO = (FileStorePortObject) po;
        FileStoreUtil.invokeFlush(fileStorePO);
        List<FileStore> fileStores = FileStoreUtil.getFileStores(fileStorePO);
        ModelContentWO fileStoreKeysModel = fileStoreModelContent.addModelContent("port_file_store_keys");
        for (int i = 0; i < fileStores.size(); i++) {
            FileStoreKey key = fileStoreHandler.translateToLocal(fileStores.get(i), fileStorePO);
            key.save(fileStoreKeysModel.addModelContent("filestore_key_" + i));
        }
    }
    toc.saveToXML(new NonClosableOutputStream.Zip(zipOut));
    zipOut.putNextEntry(new ZipEntry("objectSpec.file"));
    try (PortObjectSpecZipOutputStream specOut = getPortObjectSpecZipOutputStream(new NonClosableOutputStream.Zip(zipOut))) {
        PortObjectSpecSerializer specSer = PortTypeRegistry.getInstance().getSpecSerializer(spec.getClass()).get();
        specSer.savePortObjectSpec(spec, specOut);
    }
    // 'close' will propagate as closeEntry
    zipOut.putNextEntry(new ZipEntry("object.file"));
    try (PortObjectZipOutputStream objOut = getPortObjectZipOutputStream(new NonClosableOutputStream.Zip(zipOut))) {
        PortObjectSerializer objSer = PortTypeRegistry.getInstance().getObjectSerializer(po.getClass()).get();
        objSer.savePortObject(po, objOut, exec);
    }
    if (fileStoreHandler != null && fileStoreHandler.hasCopiedFileStores()) {
        zipOut.putNextEntry(new ZipEntry("filestores/"));
        zipOut.closeEntry();
        File baseDir = fileStoreHandler.getBaseDir();
        FileUtil.zipDir(zipOut, Arrays.asList(baseDir.listFiles()), "filestores/", FileUtil.ZIP_INCLUDEALL_FILTER, exec.createSubProgress(0.5));
    }
    zipOut.finish();
    if (!originalOutputIsBuffered) {
        os.flush();
    }
}

Example 2 with FileStore

use of org.knime.core.data.filestore.FileStore in project knime-core by knime.

the class LoopStartWritableFileStoreHandler method createFileStoreInLoopBody.

/**
 * {@inheritDoc}
 */
@Override
public synchronized FileStore createFileStoreInLoopBody(final String name) throws IOException {
    final FileStore fs = createFileStoreInternal(name, OUTER_LOOP_PATH, m_flowLoopContext.getIterationIndex());
    m_fileStoresInLoopCache.add(fs);
    return fs;
}

Example 3 with FileStore

use of org.knime.core.data.filestore.FileStore in project knime-core by knime.

the class WriteFileStoreHandler method copyFileStore.

private synchronized FileStoreKey copyFileStore(final FileStore fs, final FlushCallback flushCallback) {
    FileStoreKey key = FileStoreUtil.getFileStoreKey(fs);
    if (m_createdFileStoreKeys == null) {
        LOGGER.debug("Duplicating file store objects - file store handler id " + key.getStoreUUID() + " is unknown to " + m_fileStoreHandlerRepository.getClass().getName());
        LOGGER.debug("Dump of valid file store handlers follows, omitting further log output");
        m_fileStoreHandlerRepository.printValidFileStoreHandlersToLogDebug();
        m_createdFileStoreKeys = new LRUCache<FileStoreKey, FileStoreKey>(10000);
    }
    FileStoreKey local = m_createdFileStoreKeys.get(key);
    if (local != null) {
        return local;
    }
    FileStore newStore;
    try {
        // fixes problem with file store cell that keep things in memory until serialized:
        // notify them that a copy is taken place and that they need to flush their in memory content
        FileStoreUtil.invokeFlush(flushCallback);
        newStore = createFileStoreInternal(getNextIndex() + "_" + key.getName(), null, -1);
        FileUtil.copy(fs.getFile(), newStore.getFile());
    } catch (IOException e) {
        throw new RuntimeException("Failed copying file stores to local handler", e);
    }
    final FileStoreKey newKey = FileStoreUtil.getFileStoreKey(newStore);
    m_createdFileStoreKeys.put(key, newKey);
    return newKey;
}

Example 4 with FileStore

use of org.knime.core.data.filestore.FileStore in project knime-core by knime.

the class FileStoresInLoopCache method delete.

private void delete(final FileStoreKey key, final ILoopStartWriteFileStoreHandler handler, final MutableInteger nrFilesDeleted, final MutableInteger nrFilesFailedDelete) {
    FileStore fileStore = handler.getFileStore(key);
    File file = fileStore.getFile();
    if (file.exists() && !FileUtil.deleteRecursively(file)) {
        nrFilesFailedDelete.inc();
    }
    nrFilesDeleted.inc();
}

Example 5 with FileStore

use of org.knime.core.data.filestore.FileStore in project knime-core by knime.

the class LoopEndWriteFileStoreHandler method createFileStore.

/**
 * {@inheritDoc}
 */
@Override
public FileStore createFileStore(final String name) throws IOException {
    if (m_duplicateChecker == null) {
        throw new IllegalStateException("file store handler is not open");
    }
    m_duplicateChecker.add(name);
    FileStore fs = m_loopStartFSHandler.createFileStoreInLoopBody(name);
    m_fileStoresInLoopCache.add(fs);
    return fs;
}