Examples with FileStorePortObject org.knime.core.data.filestore.FileStorePortObject used on opensource projects

Example 1 with FileStorePortObject

use of org.knime.core.data.filestore.FileStorePortObject in project knime-core by knime.

the class PortUtil method writeObjectToStream.

/**
 * Write the given port object into the given output stream. The output stream does not need to be buffered and is
 * not closed after calling this method.
 *
 * @param po any port object
 * @param output any output stream, does not need to be buffered
 * @param exec execution context for reporting progress and checking for cancelation
 * @throws IOException if an I/O error occurs while serializing the port object
 * @throws CanceledExecutionException if the user canceled the operation
 */
public static void writeObjectToStream(final PortObject po, final OutputStream output, final ExecutionMonitor exec) throws IOException, CanceledExecutionException {
    final boolean originalOutputIsBuffered = ((output instanceof BufferedOutputStream) || (output instanceof ByteArrayOutputStream));
    OutputStream os = originalOutputIsBuffered ? output : new BufferedOutputStream(output);
    final ZipOutputStream zipOut = new ZipOutputStream(os);
    PortObjectSpec spec = po.getSpec();
    zipOut.putNextEntry(new ZipEntry("content.xml"));
    ModelContent toc = new ModelContent("content");
    toc.addInt("version", 1);
    toc.addString("port_spec_class", spec.getClass().getName());
    toc.addString("port_object_class", po.getClass().getName());
    NotInWorkflowWriteFileStoreHandler fileStoreHandler = null;
    if (po instanceof FileStorePortObject) {
        fileStoreHandler = NotInWorkflowWriteFileStoreHandler.create();
        ModelContentWO fileStoreModelContent = toc.addModelContent("filestores");
        fileStoreModelContent.addString("handlerUUID", fileStoreHandler.getStoreUUID().toString());
        final FileStorePortObject fileStorePO = (FileStorePortObject) po;
        FileStoreUtil.invokeFlush(fileStorePO);
        List<FileStore> fileStores = FileStoreUtil.getFileStores(fileStorePO);
        ModelContentWO fileStoreKeysModel = fileStoreModelContent.addModelContent("port_file_store_keys");
        for (int i = 0; i < fileStores.size(); i++) {
            FileStoreKey key = fileStoreHandler.translateToLocal(fileStores.get(i), fileStorePO);
            key.save(fileStoreKeysModel.addModelContent("filestore_key_" + i));
        }
    }
    toc.saveToXML(new NonClosableOutputStream.Zip(zipOut));
    zipOut.putNextEntry(new ZipEntry("objectSpec.file"));
    try (PortObjectSpecZipOutputStream specOut = getPortObjectSpecZipOutputStream(new NonClosableOutputStream.Zip(zipOut))) {
        PortObjectSpecSerializer specSer = PortTypeRegistry.getInstance().getSpecSerializer(spec.getClass()).get();
        specSer.savePortObjectSpec(spec, specOut);
    }
    // 'close' will propagate as closeEntry
    zipOut.putNextEntry(new ZipEntry("object.file"));
    try (PortObjectZipOutputStream objOut = getPortObjectZipOutputStream(new NonClosableOutputStream.Zip(zipOut))) {
        PortObjectSerializer objSer = PortTypeRegistry.getInstance().getObjectSerializer(po.getClass()).get();
        objSer.savePortObject(po, objOut, exec);
    }
    if (fileStoreHandler != null && fileStoreHandler.hasCopiedFileStores()) {
        zipOut.putNextEntry(new ZipEntry("filestores/"));
        zipOut.closeEntry();
        File baseDir = fileStoreHandler.getBaseDir();
        FileUtil.zipDir(zipOut, Arrays.asList(baseDir.listFiles()), "filestores/", FileUtil.ZIP_INCLUDEALL_FILTER, exec.createSubProgress(0.5));
    }
    zipOut.finish();
    if (!originalOutputIsBuffered) {
        os.flush();
    }
}

Example 2 with FileStorePortObject

use of org.knime.core.data.filestore.FileStorePortObject in project knime-core by knime.

the class FileNodePersistor method loadPortObject.

/**
 * @param portDir
 * @param settings
 * @param exec
 * @param fileStoreHandlerRepository
 * @return
 * @throws IOException
 * @throws InvalidSettingsException
 * @throws FileNotFoundException
 * @throws CanceledExecutionException
 */
private Optional<PortObject> loadPortObject(final ReferencedFile portDir, final NodeSettingsRO settings, final ExecutionMonitor exec, final FileStoreHandlerRepository fileStoreHandlerRepository) throws IOException, InvalidSettingsException, FileNotFoundException, CanceledExecutionException {
    exec.setMessage("Loading port object");
    final String specClass = settings.getString("port_spec_class");
    final String objectClass = loadPortObjectClassName(settings);
    PortObject object = null;
    PortObjectSpec spec = null;
    if (specClass != null) {
        Class<? extends PortObjectSpec> cl = PortTypeRegistry.getInstance().getSpecClass(specClass).orElseThrow(() -> new IOException("Invalid spec class \"" + specClass + "\""));
        ReferencedFile specDirRef = new ReferencedFile(portDir, settings.getString("port_spec_location"));
        File specFile = specDirRef.getFile();
        if (!specFile.isFile()) {
            throw new IOException("Can't read spec file " + specFile.getAbsolutePath());
        }
        try (PortObjectSpecZipInputStream in = PortUtil.getPortObjectSpecZipInputStream(new BufferedInputStream(new FileInputStream(specFile)))) {
            PortObjectSpecSerializer<?> serializer = PortTypeRegistry.getInstance().getSpecSerializer(cl).get();
            spec = serializer.loadPortObjectSpec(in);
            if (spec == null) {
                throw new IOException("Serializer \"" + serializer.getClass().getName() + "\" restored null spec ");
            }
        }
    }
    if (spec != null && objectClass != null) {
        Class<? extends PortObject> cl = PortTypeRegistry.getInstance().getObjectClass(objectClass).orElseThrow(() -> new IOException("Invalid object class \"" + objectClass + "\""));
        ReferencedFile objectFileRef = new ReferencedFile(portDir, settings.getString("port_object_location"));
        File objectFile = objectFileRef.getFile();
        if (!objectFile.isFile()) {
            throw new IOException("Can't read file " + objectFile.getAbsolutePath());
        }
        // buffering both disc I/O and the gzip stream pays off
        try (PortObjectZipInputStream in = PortUtil.getPortObjectZipInputStream(new BufferedInputStream(new FileInputStream(objectFile)))) {
            PortObjectSerializer<?> serializer = PortTypeRegistry.getInstance().getObjectSerializer(cl).get();
            object = serializer.loadPortObject(in, spec, exec);
        }
        if (object instanceof FileStorePortObject) {
            File fileStoreXML = new File(objectFile.getParent(), "filestore.xml");
            final ModelContentRO fileStoreModelContent = ModelContent.loadFromXML(new FileInputStream(fileStoreXML));
            List<FileStoreKey> fileStoreKeys = new ArrayList<FileStoreKey>();
            if (getLoadVersion().isOlderThan(LoadVersion.V2100)) {
                // only one filestore in <2.10 (bug 5227)
                FileStoreKey fileStoreKey = FileStoreKey.load(fileStoreModelContent);
                fileStoreKeys.add(fileStoreKey);
            } else {
                ModelContentRO keysContent = fileStoreModelContent.getModelContent("filestore_keys");
                for (String id : keysContent.keySet()) {
                    ModelContentRO keyContent = keysContent.getModelContent(id);
                    fileStoreKeys.add(FileStoreKey.load(keyContent));
                }
            }
            FileStoreUtil.retrieveFileStoreHandlerFrom((FileStorePortObject) object, fileStoreKeys, fileStoreHandlerRepository);
        }
    }
    return Optional.ofNullable(object);
}

Example 3 with FileStorePortObject

use of org.knime.core.data.filestore.FileStorePortObject in project knime-core by knime.

the class Node method copyPortObject.

/**
 * Copies the PortObject so that the copy can be given to the node model
 * implementation (and potentially modified). The copy is carried out by
 * means of the respective serializer (via streams).
 *
 * <p> Note that this method is meant to be used by the framework only.
 * @param portObject The object to be copied.
 * @param exec For progress/cancel
 * @return The (deep) copy.
 * @throws IOException In case of exceptions while accessing the stream or
 * if the argument is an instance of {@link BufferedDataTable}.
 * @throws CanceledExecutionException If canceled.
 */
public static PortObject copyPortObject(final PortObject portObject, final ExecutionContext exec) throws IOException, CanceledExecutionException {
    if (portObject instanceof BufferedDataTable) {
        throw new IOException("Can't copy BufferedDataTable objects");
    }
    // first copy the spec, then copy the object
    final PortObjectSpec s = portObject.getSpec();
    PortObjectSpec.PortObjectSpecSerializer ser = PortTypeRegistry.getInstance().getSpecSerializer(s.getClass()).get();
    ByteArrayOutputStream byteOut = new ByteArrayOutputStream(10 * 1024);
    PortObjectSpecZipOutputStream specOut = PortUtil.getPortObjectSpecZipOutputStream(byteOut);
    specOut.setLevel(0);
    ser.savePortObjectSpec(s, specOut);
    specOut.close();
    ByteArrayInputStream byteIn = new ByteArrayInputStream(byteOut.toByteArray());
    PortObjectSpecZipInputStream specIn = PortUtil.getPortObjectSpecZipInputStream(byteIn);
    PortObjectSpec specCopy = ser.loadPortObjectSpec(specIn);
    specIn.close();
    DeferredFileOutputStream deferredOutputStream = new DeferredFileOutputStream(/* 10 MB */
    10 * 1024 * 1024, "knime-portobject-copy-", ".bin", new File(KNIMEConstants.getKNIMETempDir()));
    PortObject.PortObjectSerializer obSer = PortTypeRegistry.getInstance().getObjectSerializer(portObject.getClass()).get();
    PortObjectZipOutputStream objOut = PortUtil.getPortObjectZipOutputStream(deferredOutputStream);
    objOut.setLevel(0);
    obSer.savePortObject(portObject, objOut, exec);
    objOut.close();
    InputStream inStream;
    if (deferredOutputStream.isInMemory()) {
        inStream = new ByteArrayInputStream(deferredOutputStream.getData());
    } else {
        inStream = new BufferedInputStream(new FileInputStream(deferredOutputStream.getFile()));
    }
    PortObjectZipInputStream objIn = PortUtil.getPortObjectZipInputStream(inStream);
    PortObject result = obSer.loadPortObject(objIn, specCopy, exec);
    objIn.close();
    if (portObject instanceof FileStorePortObject) {
        FileStorePortObject sourceFSObj = (FileStorePortObject) portObject;
        FileStorePortObject resultFSObj = (FileStorePortObject) result;
        FileStoreUtil.retrieveFileStoreHandlers(sourceFSObj, resultFSObj, exec.getFileStoreHandler());
    }
    if (!deferredOutputStream.isInMemory()) {
        deferredOutputStream.getFile().delete();
    }
    return result;
}

Example 4 with FileStorePortObject

use of org.knime.core.data.filestore.FileStorePortObject in project knime-core by knime.

the class PortUtil method readObjectFromStream.

public static PortObject readObjectFromStream(final InputStream input, final ExecutionMonitor exec) throws IOException, CanceledExecutionException {
    try (ZipInputStream in = new ZipInputStream(new BufferedInputStream(input))) {
        ZipEntry entry = in.getNextEntry();
        // Check that the given input stream really specifies a port object
        if (entry == null) {
            throw new IOException("File does not specify a valid port object model");
        }
        if (!"content.xml".equals(entry.getName())) {
            throw new IOException("Invalid stream, expected zip entry \"content.xml\", got \"" + entry.getName() + "\"");
        }
        ModelContentRO toc = ModelContent.loadFromXML(new NonClosableInputStream.Zip(in));
        String specClassName = toc.getString("port_spec_class");
        String objClassName = toc.getString("port_object_class");
        // reads "objectSpec.file"
        PortObjectSpec spec = readObjectSpec(specClassName, in);
        entry = in.getNextEntry();
        if (!"object.file".equals(entry.getName())) {
            throw new IOException("Invalid stream, expected zip entry \"object.file\", got \"" + entry.getName() + "\"");
        }
        Class<? extends PortObject> cl = PortTypeRegistry.getInstance().getObjectClass(objClassName).orElseThrow(() -> new IOException("Can't load class \"" + specClassName + "\""));
        PortObject portObject;
        try (PortObjectZipInputStream objIn = getPortObjectZipInputStream(new NonClosableInputStream.Zip(in))) {
            PortObjectSerializer<?> objSer = PortTypeRegistry.getInstance().getObjectSerializer(cl.asSubclass(PortObject.class)).get();
            portObject = objSer.loadPortObject(objIn, spec, exec);
        }
        if (portObject instanceof FileStorePortObject) {
            ModelContentRO fileStoreModelContent = toc.getModelContent("filestores");
            UUID iFileStoreHandlerUUID = UUID.fromString(fileStoreModelContent.getString("handlerUUID"));
            ModelContentRO fileStoreKeysModel = fileStoreModelContent.getModelContent("port_file_store_keys");
            List<FileStoreKey> fileStoreKeys = new ArrayList<>();
            for (String key : fileStoreKeysModel.keySet()) {
                fileStoreKeys.add(FileStoreKey.load(fileStoreKeysModel.getModelContent(key)));
            }
            NotInWorkflowWriteFileStoreHandler notInWorkflowFSHandler = new NotInWorkflowWriteFileStoreHandler(iFileStoreHandlerUUID);
            entry = in.getNextEntry();
            if (entry != null && "filestores/".equals(entry.getName())) {
                File fileStoreDir = FileUtil.createTempDir("knime_fs_" + cl.getSimpleName() + "-");
                FileUtil.unzip(in, fileStoreDir, 1);
                notInWorkflowFSHandler.setBaseDir(fileStoreDir);
            }
            FileStoreUtil.retrieveFileStoreHandlerFrom((FileStorePortObject) portObject, fileStoreKeys, notInWorkflowFSHandler.getDataRepository());
        }
        return portObject;
    } catch (InvalidSettingsException ex) {
        throw new IOException("Unable to parse content.xml in port object file", ex);
    }
}

Example 5 with FileStorePortObject

use of org.knime.core.data.filestore.FileStorePortObject in project knime-core by knime.

the class FileNodePersistor method loadPortObject.

/**
 * @param portDir
 * @param settings
 * @param exec
 * @param dataRepository
 * @return
 * @throws IOException
 * @throws InvalidSettingsException
 * @throws FileNotFoundException
 * @throws CanceledExecutionException
 */
private Optional<PortObject> loadPortObject(final ReferencedFile portDir, final NodeSettingsRO settings, final ExecutionMonitor exec, final WorkflowDataRepository dataRepository) throws IOException, InvalidSettingsException, FileNotFoundException, CanceledExecutionException {
    exec.setMessage("Loading port object");
    final String specClass = settings.getString("port_spec_class");
    final String objectClass = loadPortObjectClassName(settings);
    PortObject object = null;
    PortObjectSpec spec = null;
    if (specClass != null) {
        Class<? extends PortObjectSpec> cl = PortTypeRegistry.getInstance().getSpecClass(specClass).orElseThrow(() -> new IOException("Invalid spec class \"" + specClass + "\""));
        ReferencedFile specDirRef = new ReferencedFile(portDir, settings.getString("port_spec_location"));
        File specFile = specDirRef.getFile();
        if (!specFile.isFile()) {
            throw new IOException("Can't read spec file " + specFile.getAbsolutePath());
        }
        try (PortObjectSpecZipInputStream in = PortUtil.getPortObjectSpecZipInputStream(new BufferedInputStream(new FileInputStream(specFile)))) {
            PortObjectSpecSerializer<?> serializer = PortTypeRegistry.getInstance().getSpecSerializer(cl).get();
            spec = serializer.loadPortObjectSpec(in);
            if (spec == null) {
                throw new IOException("Serializer \"" + serializer.getClass().getName() + "\" restored null spec ");
            }
        }
    }
    if (spec != null && objectClass != null) {
        Class<? extends PortObject> cl = PortTypeRegistry.getInstance().getObjectClass(objectClass).orElseThrow(() -> new IOException("Invalid object class \"" + objectClass + "\""));
        ReferencedFile objectFileRef = new ReferencedFile(portDir, settings.getString("port_object_location"));
        File objectFile = objectFileRef.getFile();
        if (!objectFile.isFile()) {
            throw new IOException("Can't read file " + objectFile.getAbsolutePath());
        }
        // buffering both disc I/O and the gzip stream pays off
        try (PortObjectZipInputStream in = PortUtil.getPortObjectZipInputStream(new BufferedInputStream(new FileInputStream(objectFile)))) {
            PortObjectSerializer<?> serializer = PortTypeRegistry.getInstance().getObjectSerializer(cl).get();
            object = serializer.loadPortObject(in, spec, exec);
        }
        if (object instanceof FileStorePortObject) {
            File fileStoreXML = new File(objectFile.getParent(), "filestore.xml");
            final ModelContentRO fileStoreModelContent = ModelContent.loadFromXML(new FileInputStream(fileStoreXML));
            List<FileStoreKey> fileStoreKeys = new ArrayList<FileStoreKey>();
            if (getLoadVersion().isOlderThan(LoadVersion.V2100)) {
                // only one filestore in <2.10 (bug 5227)
                FileStoreKey fileStoreKey = FileStoreKey.load(fileStoreModelContent);
                fileStoreKeys.add(fileStoreKey);
            } else {
                ModelContentRO keysContent = fileStoreModelContent.getModelContent("filestore_keys");
                for (String id : keysContent.keySet()) {
                    ModelContentRO keyContent = keysContent.getModelContent(id);
                    fileStoreKeys.add(FileStoreKey.load(keyContent));
                }
            }
            FileStoreUtil.retrieveFileStoreHandlerFrom((FileStorePortObject) object, fileStoreKeys, dataRepository);
        }
    }
    return Optional.ofNullable(object);
}