Examples with FileStoreKey org.knime.core.data.filestore.FileStoreKey used on opensource projects

Example 1 with FileStoreKey

use of org.knime.core.data.filestore.FileStoreKey in project knime-core by knime.

the class PortUtil method writeObjectToStream.

/**
 * Write the given port object into the given output stream. The output stream does not need to be buffered and is
 * not closed after calling this method.
 *
 * @param po any port object
 * @param output any output stream, does not need to be buffered
 * @param exec execution context for reporting progress and checking for cancelation
 * @throws IOException if an I/O error occurs while serializing the port object
 * @throws CanceledExecutionException if the user canceled the operation
 */
public static void writeObjectToStream(final PortObject po, final OutputStream output, final ExecutionMonitor exec) throws IOException, CanceledExecutionException {
    final boolean originalOutputIsBuffered = ((output instanceof BufferedOutputStream) || (output instanceof ByteArrayOutputStream));
    OutputStream os = originalOutputIsBuffered ? output : new BufferedOutputStream(output);
    final ZipOutputStream zipOut = new ZipOutputStream(os);
    PortObjectSpec spec = po.getSpec();
    zipOut.putNextEntry(new ZipEntry("content.xml"));
    ModelContent toc = new ModelContent("content");
    toc.addInt("version", 1);
    toc.addString("port_spec_class", spec.getClass().getName());
    toc.addString("port_object_class", po.getClass().getName());
    NotInWorkflowWriteFileStoreHandler fileStoreHandler = null;
    if (po instanceof FileStorePortObject) {
        fileStoreHandler = NotInWorkflowWriteFileStoreHandler.create();
        ModelContentWO fileStoreModelContent = toc.addModelContent("filestores");
        fileStoreModelContent.addString("handlerUUID", fileStoreHandler.getStoreUUID().toString());
        final FileStorePortObject fileStorePO = (FileStorePortObject) po;
        FileStoreUtil.invokeFlush(fileStorePO);
        List<FileStore> fileStores = FileStoreUtil.getFileStores(fileStorePO);
        ModelContentWO fileStoreKeysModel = fileStoreModelContent.addModelContent("port_file_store_keys");
        for (int i = 0; i < fileStores.size(); i++) {
            FileStoreKey key = fileStoreHandler.translateToLocal(fileStores.get(i), fileStorePO);
            key.save(fileStoreKeysModel.addModelContent("filestore_key_" + i));
        }
    }
    toc.saveToXML(new NonClosableOutputStream.Zip(zipOut));
    zipOut.putNextEntry(new ZipEntry("objectSpec.file"));
    try (PortObjectSpecZipOutputStream specOut = getPortObjectSpecZipOutputStream(new NonClosableOutputStream.Zip(zipOut))) {
        PortObjectSpecSerializer specSer = PortTypeRegistry.getInstance().getSpecSerializer(spec.getClass()).get();
        specSer.savePortObjectSpec(spec, specOut);
    }
    // 'close' will propagate as closeEntry
    zipOut.putNextEntry(new ZipEntry("object.file"));
    try (PortObjectZipOutputStream objOut = getPortObjectZipOutputStream(new NonClosableOutputStream.Zip(zipOut))) {
        PortObjectSerializer objSer = PortTypeRegistry.getInstance().getObjectSerializer(po.getClass()).get();
        objSer.savePortObject(po, objOut, exec);
    }
    if (fileStoreHandler != null && fileStoreHandler.hasCopiedFileStores()) {
        zipOut.putNextEntry(new ZipEntry("filestores/"));
        zipOut.closeEntry();
        File baseDir = fileStoreHandler.getBaseDir();
        FileUtil.zipDir(zipOut, Arrays.asList(baseDir.listFiles()), "filestores/", FileUtil.ZIP_INCLUDEALL_FILTER, exec.createSubProgress(0.5));
    }
    zipOut.finish();
    if (!originalOutputIsBuffered) {
        os.flush();
    }
}

Example 2 with FileStoreKey

use of org.knime.core.data.filestore.FileStoreKey in project knime-core by knime.

the class LoopEndWriteFileStoreHandler method translateToLocal.

/**
 * {@inheritDoc}
 */
@Override
public FileStoreKey translateToLocal(final FileStore fs, final FlushCallback flushCallback) {
    final FileStoreKey result = m_loopStartFSHandler.translateToLocal(fs, flushCallback);
    // might be called after node is closed, e.g. when workflow is saved
    boolean isClosed = m_fileStoresInLoopCache == null;
    if (!isClosed && m_loopStartFSHandler.isCreatedInThisLoop(result)) {
        if (m_fsKeysToKeepLRUCache.put(result, result) == null) {
            m_fileStoresInLoopCache.add(result);
        }
    }
    return result;
}

Example 3 with FileStoreKey

use of org.knime.core.data.filestore.FileStoreKey in project knime-core by knime.

the class WriteFileStoreHandler method copyFileStore.

private synchronized FileStoreKey copyFileStore(final FileStore fs, final FlushCallback flushCallback) {
    FileStoreKey key = FileStoreUtil.getFileStoreKey(fs);
    if (m_createdFileStoreKeys == null) {
        LOGGER.debug("Duplicating file store objects - file store handler id " + key.getStoreUUID() + " is unknown to " + m_fileStoreHandlerRepository.getClass().getName());
        LOGGER.debug("Dump of valid file store handlers follows, omitting further log output");
        m_fileStoreHandlerRepository.printValidFileStoreHandlersToLogDebug();
        m_createdFileStoreKeys = new LRUCache<FileStoreKey, FileStoreKey>(10000);
    }
    FileStoreKey local = m_createdFileStoreKeys.get(key);
    if (local != null) {
        return local;
    }
    FileStore newStore;
    try {
        // fixes problem with file store cell that keep things in memory until serialized:
        // notify them that a copy is taken place and that they need to flush their in memory content
        FileStoreUtil.invokeFlush(flushCallback);
        newStore = createFileStoreInternal(getNextIndex() + "_" + key.getName(), null, -1);
        FileUtil.copy(fs.getFile(), newStore.getFile());
    } catch (IOException e) {
        throw new RuntimeException("Failed copying file stores to local handler", e);
    }
    final FileStoreKey newKey = FileStoreUtil.getFileStoreKey(newStore);
    m_createdFileStoreKeys.put(key, newKey);
    return newKey;
}

Example 4 with FileStoreKey

use of org.knime.core.data.filestore.FileStoreKey in project knime-core by knime.

the class FileStoresInLoopCache method deletableUnusedFileStores.

void deletableUnusedFileStores(final FileStoresInLoopCache endNodeCacheWithKeysToPersist, final ILoopStartWriteFileStoreHandler handler) throws CanceledExecutionException {
    MutableInteger nrFilesDeleted = new MutableInteger(0);
    MutableInteger nrFailedDeletes = new MutableInteger(0);
    CloseableRowIterator allKeysIterator = m_createdFileStoresTable.iterator();
    CloseableRowIterator endNodeKeysIterator = endNodeCacheWithKeysToPersist.getCreatedFileStoresTable().iterator();
    FileStoreKey nextLoopEndFSKey = next(endNodeKeysIterator, null);
    FileStoreKey nextAllFSKey = next(allKeysIterator, null);
    while (nextLoopEndFSKey != null) {
        if (nextAllFSKey != null) {
            final int compare = nextLoopEndFSKey.compareTo(nextAllFSKey);
            if (compare == 0) {
                nextLoopEndFSKey = next(endNodeKeysIterator, nextLoopEndFSKey);
                nextAllFSKey = next(allKeysIterator, nextAllFSKey);
            } else if (compare > 0) {
                delete(nextAllFSKey, handler, nrFilesDeleted, nrFailedDeletes);
                nextAllFSKey = next(allKeysIterator, nextAllFSKey);
            } else {
                nextLoopEndFSKey = next(endNodeKeysIterator, nextLoopEndFSKey);
            }
        } else {
            break;
        }
    }
    while (nextAllFSKey != null) {
        delete(nextAllFSKey, handler, nrFilesDeleted, nrFailedDeletes);
        nextAllFSKey = next(allKeysIterator, nextAllFSKey);
    }
    allKeysIterator.close();
    endNodeKeysIterator.close();
    if (nrFilesDeleted.intValue() > 0) {
        StringBuilder b = new StringBuilder("Deleted ");
        b.append(nrFilesDeleted.intValue()).append(" files ");
        if (nrFailedDeletes.intValue() > 0) {
            b.append("; ").append(nrFailedDeletes.intValue()).append(" of which failed");
        } else {
            b.append("successfully");
        }
        LOGGER.debug(b.toString());
    }
}

Example 5 with FileStoreKey

use of org.knime.core.data.filestore.FileStoreKey in project knime-core by knime.

the class PortUtil method readObjectFromStream.

public static PortObject readObjectFromStream(final InputStream input, final ExecutionMonitor exec) throws IOException, CanceledExecutionException {
    try (ZipInputStream in = new ZipInputStream(new BufferedInputStream(input))) {
        ZipEntry entry = in.getNextEntry();
        if (!"content.xml".equals(entry.getName())) {
            throw new IOException("Invalid stream, expected zip entry \"content.xml\", got \"" + entry.getName() + "\"");
        }
        ModelContentRO toc = ModelContent.loadFromXML(new NonClosableInputStream.Zip(in));
        String specClassName = toc.getString("port_spec_class");
        String objClassName = toc.getString("port_object_class");
        // reads "objectSpec.file"
        PortObjectSpec spec = readObjectSpec(specClassName, in);
        entry = in.getNextEntry();
        if (!"object.file".equals(entry.getName())) {
            throw new IOException("Invalid stream, expected zip entry \"object.file\", got \"" + entry.getName() + "\"");
        }
        Class<?> cl;
        try {
            cl = Class.forName(objClassName);
        } catch (ClassNotFoundException e) {
            throw new IOException("Can't load class \"" + specClassName + "\"", e);
        }
        if (!PortObject.class.isAssignableFrom(cl)) {
            throw new IOException("Class \"" + cl.getSimpleName() + "\" does not a sub-class \"" + PortObject.class.getSimpleName() + "\"");
        }
        PortObject portObject;
        try (PortObjectZipInputStream objIn = getPortObjectZipInputStream(new NonClosableInputStream.Zip(in))) {
            PortObjectSerializer<?> objSer = PortTypeRegistry.getInstance().getObjectSerializer(cl.asSubclass(PortObject.class)).get();
            portObject = objSer.loadPortObject(objIn, spec, exec);
        }
        if (portObject instanceof FileStorePortObject) {
            ModelContentRO fileStoreModelContent = toc.getModelContent("filestores");
            UUID iFileStoreHandlerUUID = UUID.fromString(fileStoreModelContent.getString("handlerUUID"));
            ModelContentRO fileStoreKeysModel = fileStoreModelContent.getModelContent("port_file_store_keys");
            List<FileStoreKey> fileStoreKeys = new ArrayList<>();
            for (String key : fileStoreKeysModel.keySet()) {
                fileStoreKeys.add(FileStoreKey.load(fileStoreKeysModel.getModelContent(key)));
            }
            NotInWorkflowWriteFileStoreHandler notInWorkflowFSHandler = new NotInWorkflowWriteFileStoreHandler(iFileStoreHandlerUUID);
            entry = in.getNextEntry();
            if (entry != null && "filestores/".equals(entry.getName())) {
                File fileStoreDir = FileUtil.createTempDir("knime_fs_" + cl.getSimpleName() + "-");
                FileUtil.unzip(in, fileStoreDir, 1);
                notInWorkflowFSHandler.setBaseDir(fileStoreDir);
            }
            FileStoreUtil.retrieveFileStoreHandlerFrom((FileStorePortObject) portObject, fileStoreKeys, notInWorkflowFSHandler.getFileStoreHandlerRepository());
        }
        return portObject;
    } catch (InvalidSettingsException ex) {
        throw new IOException("Unable to parse content.xml in port object file", ex);
    }
}