Examples with FileStoreKey org.knime.core.data.filestore.FileStoreKey used on opensource projects

Example 6 with FileStoreKey

use of org.knime.core.data.filestore.FileStoreKey in project knime-core by knime.

the class DefaultTableStoreWriter method writeDataCell.

/**
 * Writes a data cell to the outStream.
 *
 * @param cell The cell to write.
 * @param outStream To write to.
 * @throws IOException If stream corruption happens.
 */
void writeDataCell(final DataCell cell, final DCObjectOutputVersion2 outStream) throws IOException {
    if (cell == DataType.getMissingCell()) {
        // only write 'missing' byte if that's the singleton missing cell;
        // missing cells with error cause are handled like ordinary cells below (via serializer)
        outStream.writeControlByte(BYTE_TYPE_MISSING);
        return;
    }
    boolean isBlob = cell instanceof BlobWrapperDataCell;
    CellClassInfo cellClass = isBlob ? ((BlobWrapperDataCell) cell).getBlobClassInfo() : CellClassInfo.get(cell);
    DataCellSerializer<DataCell> ser = getSerializerForDataCell(cellClass);
    Byte identifier = m_typeShortCuts.get(cellClass);
    FileStoreKey fileStoreKey = super.getFileStoreKeyAndFlush(cell);
    final boolean isJavaSerializationOrBlob = ser == null && !isBlob;
    if (isJavaSerializationOrBlob) {
        outStream.writeControlByte(BYTE_TYPE_SERIALIZATION);
    }
    outStream.writeControlByte(identifier);
    if (fileStoreKey != null) {
        outStream.writeFileStoreKey(fileStoreKey);
    }
    // DataCell is datacell-serializable
    if (!isJavaSerializationOrBlob) {
        if (isBlob) {
            BlobWrapperDataCell bc = (BlobWrapperDataCell) cell;
            outStream.writeBlobAddress(bc.getAddress());
        } else {
            outStream.writeDataCellPerKNIMESerializer(ser, cell);
        }
    } else {
        outStream.writeDataCellPerJavaSerialization(cell);
    }
}

Example 7 with FileStoreKey

use of org.knime.core.data.filestore.FileStoreKey in project knime-core by knime.

the class FileNodePersistor method savePortObject.

private static void savePortObject(final PortObjectSpec spec, final PortObject object, final File portDir, final NodeSettingsWO settings, final ExecutionMonitor exec) throws IOException, FileNotFoundException, CanceledExecutionException {
    settings.addString("port_spec_class", spec.getClass().getName());
    settings.addString("port_object_class", object.getClass().getName());
    String specDirName = "spec";
    String specFileName = "spec.zip";
    String specPath = specDirName + "/" + specFileName;
    File specDir = createDirectory(new File(portDir, specDirName));
    File specFile = new File(specDir, specFileName);
    settings.addString("port_spec_location", specPath);
    try (PortObjectSpecZipOutputStream out = PortUtil.getPortObjectSpecZipOutputStream(new BufferedOutputStream(new FileOutputStream(specFile)))) {
        PortObjectSpecSerializer serializer = PortTypeRegistry.getInstance().getSpecSerializer(spec.getClass()).get();
        serializer.savePortObjectSpec(spec, out);
    }
    String objectDirName = null;
    objectDirName = "object";
    File objectDir = createDirectory(new File(portDir, objectDirName));
    String objectPath;
    String objectFileName = "portobject.zip";
    objectPath = objectDirName + "/" + objectFileName;
    settings.addString("port_object_location", objectPath);
    File file = new File(objectDir, objectFileName);
    try (PortObjectZipOutputStream out = PortUtil.getPortObjectZipOutputStream(new BufferedOutputStream(new FileOutputStream(file)))) {
        PortObjectSerializer serializer = PortTypeRegistry.getInstance().getObjectSerializer(object.getClass()).get();
        serializer.savePortObject(object, out, exec);
        if (object instanceof FileStorePortObject) {
            List<FileStoreKey> fileStoreKeys = FileStoreUtil.getFileStores((FileStorePortObject) object).stream().map(FileStoreUtil::getFileStoreKey).collect(Collectors.toList());
            File fileStoreXML = new File(objectDir, "filestore.xml");
            final ModelContent fileStoreModelContent = new ModelContent("filestore");
            ModelContentWO keysContent = fileStoreModelContent.addModelContent("filestore_keys");
            for (int i = 0; i < fileStoreKeys.size(); i++) {
                FileStoreKey key = fileStoreKeys.get(i);
                ModelContentWO keyContent = keysContent.addModelContent("filestore_key_" + i);
                key.save(keyContent);
            }
            fileStoreModelContent.saveToXML(new FileOutputStream(fileStoreXML));
        }
    }
}

Example 8 with FileStoreKey

use of org.knime.core.data.filestore.FileStoreKey in project knime-core by knime.

the class FileNodePersistor method loadPortObject.

/**
 * @param portDir
 * @param settings
 * @param exec
 * @param fileStoreHandlerRepository
 * @return
 * @throws IOException
 * @throws InvalidSettingsException
 * @throws FileNotFoundException
 * @throws CanceledExecutionException
 */
private Optional<PortObject> loadPortObject(final ReferencedFile portDir, final NodeSettingsRO settings, final ExecutionMonitor exec, final FileStoreHandlerRepository fileStoreHandlerRepository) throws IOException, InvalidSettingsException, FileNotFoundException, CanceledExecutionException {
    exec.setMessage("Loading port object");
    final String specClass = settings.getString("port_spec_class");
    final String objectClass = loadPortObjectClassName(settings);
    PortObject object = null;
    PortObjectSpec spec = null;
    if (specClass != null) {
        Class<? extends PortObjectSpec> cl = PortTypeRegistry.getInstance().getSpecClass(specClass).orElseThrow(() -> new IOException("Invalid spec class \"" + specClass + "\""));
        ReferencedFile specDirRef = new ReferencedFile(portDir, settings.getString("port_spec_location"));
        File specFile = specDirRef.getFile();
        if (!specFile.isFile()) {
            throw new IOException("Can't read spec file " + specFile.getAbsolutePath());
        }
        try (PortObjectSpecZipInputStream in = PortUtil.getPortObjectSpecZipInputStream(new BufferedInputStream(new FileInputStream(specFile)))) {
            PortObjectSpecSerializer<?> serializer = PortTypeRegistry.getInstance().getSpecSerializer(cl).get();
            spec = serializer.loadPortObjectSpec(in);
            if (spec == null) {
                throw new IOException("Serializer \"" + serializer.getClass().getName() + "\" restored null spec ");
            }
        }
    }
    if (spec != null && objectClass != null) {
        Class<? extends PortObject> cl = PortTypeRegistry.getInstance().getObjectClass(objectClass).orElseThrow(() -> new IOException("Invalid object class \"" + objectClass + "\""));
        ReferencedFile objectFileRef = new ReferencedFile(portDir, settings.getString("port_object_location"));
        File objectFile = objectFileRef.getFile();
        if (!objectFile.isFile()) {
            throw new IOException("Can't read file " + objectFile.getAbsolutePath());
        }
        // buffering both disc I/O and the gzip stream pays off
        try (PortObjectZipInputStream in = PortUtil.getPortObjectZipInputStream(new BufferedInputStream(new FileInputStream(objectFile)))) {
            PortObjectSerializer<?> serializer = PortTypeRegistry.getInstance().getObjectSerializer(cl).get();
            object = serializer.loadPortObject(in, spec, exec);
        }
        if (object instanceof FileStorePortObject) {
            File fileStoreXML = new File(objectFile.getParent(), "filestore.xml");
            final ModelContentRO fileStoreModelContent = ModelContent.loadFromXML(new FileInputStream(fileStoreXML));
            List<FileStoreKey> fileStoreKeys = new ArrayList<FileStoreKey>();
            if (getLoadVersion().isOlderThan(LoadVersion.V2100)) {
                // only one filestore in <2.10 (bug 5227)
                FileStoreKey fileStoreKey = FileStoreKey.load(fileStoreModelContent);
                fileStoreKeys.add(fileStoreKey);
            } else {
                ModelContentRO keysContent = fileStoreModelContent.getModelContent("filestore_keys");
                for (String id : keysContent.keySet()) {
                    ModelContentRO keyContent = keysContent.getModelContent(id);
                    fileStoreKeys.add(FileStoreKey.load(keyContent));
                }
            }
            FileStoreUtil.retrieveFileStoreHandlerFrom((FileStorePortObject) object, fileStoreKeys, fileStoreHandlerRepository);
        }
    }
    return Optional.ofNullable(object);
}

Example 9 with FileStoreKey

use of org.knime.core.data.filestore.FileStoreKey in project knime-core by knime.

the class WriteFileStoreHandler method createFileStoreInternal.

FileStore createFileStoreInternal(final String name, final byte[] nestedLoopPath, final int iterationIndex) throws IOException {
    assert Thread.holdsLock(this);
    CheckUtils.checkArgumentNotNull(name, "Argument must not be null.");
    if (name.startsWith(".")) {
        throw new IOException("Name must not start with a dot: \"" + name + "\"");
    }
    if (name.contains("/") || name.contains("\\")) {
        throw new IOException("Invalid file name, must not contain (back) slash: \"" + name + "\"");
    }
    FileStoreKey key = new FileStoreKey(m_storeUUID, m_nextIndex, nestedLoopPath, iterationIndex, name);
    ensureInitBaseDirectory();
    if (m_nextIndex > MAX_NR_FILES) {
        throw new IOException("Maximum number of files stores reached: " + MAX_NR_FILES);
    }
    getParentDir(m_nextIndex, true);
    m_nextIndex++;
    FileStore fs = FileStoreUtil.createFileStore(this, key);
    return fs;
}

Example 10 with FileStoreKey

use of org.knime.core.data.filestore.FileStoreKey in project knime-core by knime.

the class AbstractTableStoreWriter method getFileStoreKeyAndFlush.

/**
 * @param cell
 * @return
 * @throws IOException
 */
public FileStoreKey getFileStoreKeyAndFlush(final DataCell cell) throws IOException {
    FileStoreKey fileStoreKey = null;
    if (cell instanceof FileStoreCell) {
        final FileStoreCell fsCell = (FileStoreCell) cell;
        FileStore fileStore = FileStoreUtil.getFileStore(fsCell);
        // TODO is the 'else' case realistic?
        if (getFileStoreHandler() instanceof IWriteFileStoreHandler) {
            fileStoreKey = getFileStoreHandler().translateToLocal(fileStore, fsCell);
        } else {
            // handler is not an IWriteFileStoreHandler but the buffer still contains file stores:
            // the flow is part of a workflow and all file stores were already properly handled
            // (this buffer is restored from disc - and then a memory alert forces the data back onto disc)
            fileStoreKey = FileStoreUtil.getFileStoreKey(fileStore);
        }
        FileStoreUtil.invokeFlush(fsCell);
    }
    return fileStoreKey;
}