Examples with ReferencedFile org.knime.core.internal.ReferencedFile used on opensource projects

Example 26 with ReferencedFile

use of org.knime.core.internal.ReferencedFile in project knime-core by knime.

the class FileNodePersistor method saveInternalHeldTables.

private static void saveInternalHeldTables(final Node node, final ReferencedFile nodeDirRef, final NodeSettingsWO settings, final Set<Integer> savedTableIDs, final ExecutionMonitor exec, final boolean saveData) throws IOException, CanceledExecutionException {
    PortObject[] internalObjects = node.getInternalHeldPortObjects();
    if (internalObjects == null || !saveData) {
        return;
    }
    final int internalTblsCount = internalObjects.length;
    NodeSettingsWO subSettings = settings.addNodeSettings("internalObjects");
    String subDirName = INTERNAL_TABLE_FOLDER_PREFIX;
    ReferencedFile subDirFile = new ReferencedFile(nodeDirRef, subDirName);
    subSettings.addString("location", subDirName);
    NodeSettingsWO portSettings = subSettings.addNodeSettings("content");
    FileUtil.deleteRecursively(subDirFile.getFile());
    subDirFile.getFile().mkdirs();
    exec.setMessage("Saving internally held objects");
    for (int i = 0; i < internalTblsCount; i++) {
        PortObject t = internalObjects[i];
        String objName = "object_" + i;
        ExecutionMonitor subProgress = exec.createSubProgress(1.0 / internalTblsCount);
        NodeSettingsWO singlePortSetting = portSettings.addNodeSettings(objName);
        ReferencedFile portDirRef = new ReferencedFile(subDirFile, objName);
        File portDir = portDirRef.getFile();
        singlePortSetting.addInt("index", i);
        if (t == null) {
            singlePortSetting.addString("type", "null");
        } else if (t instanceof BufferedDataTable) {
            BufferedDataTable table = (BufferedDataTable) t;
            saveBufferedDataTable(table, savedTableIDs, createDirectory(portDir), exec);
            singlePortSetting.addString("type", "table");
            singlePortSetting.addString("table_dir_location", objName);
        } else {
            // other type of port object - object might be in output
            int outputPortIndex = IntStream.range(0, node.getNrOutPorts()).filter(p -> node.getOutputObject(p) == t).findFirst().orElse(-1);
            if (outputPortIndex >= 0) {
                singlePortSetting.addString("type", "referenced_output");
                singlePortSetting.addInt("outport", outputPortIndex);
            } else {
                singlePortSetting.addString("type", "non-table");
                singlePortSetting.addString("port_dir_location", objName);
                savePortObject(t.getSpec(), t, createDirectory(portDir), singlePortSetting, exec);
            }
        }
        subProgress.setProgress(1.0);
    }
}

Example 27 with ReferencedFile

use of org.knime.core.internal.ReferencedFile in project knime-core by knime.

the class FileNodePersistor method loadBufferedDataTable.

private BufferedDataTable loadBufferedDataTable(final Node node, final NodeSettingsRO settings, final ExecutionMonitor execMon, final Map<Integer, BufferedDataTable> loadTblRep, final int index, final HashMap<Integer, ContainerTable> tblRep, final FileStoreHandlerRepository fileStoreHandlerRepository) throws InvalidSettingsException, IOException, CanceledExecutionException {
    // in 1.1.x and before the settings.xml contained the location
    // of the data table specs file (spec_0.xml, e.g.). From 1.2.0 on,
    // the spec is saved in data/data_0/spec.xml
    boolean isVersion11x = settings.containsKey(CFG_SPEC_FILES);
    ExecutionMonitor execSubData = execMon.createSubProgress(0.25);
    ReferencedFile nodeDirectory = getNodeDirectory();
    if (isVersion11x) {
        /* In version 1.1.x the data was stored in a different way. The
             * data.xml that is now contained in the data/data_x/ directory was
             * aggregated in a data.xml file directly in the m_nodeDir. Also the
             * spec was located at a different location.
             */
        String dataConfigFileName = settings.getString(CFG_DATA_FILE);
        File nodeDir = nodeDirectory.getFile();
        // dataConfigFile = data.xml in node dir
        File dataConfigFile = new File(nodeDir, dataConfigFileName);
        NodeSettingsRO dataSettings = NodeSettings.loadFromXML(new BufferedInputStream(new FileInputStream(dataConfigFile)));
        String dataPath = dataSettings.getString(CFG_DATA_FILE_DIR);
        // dataDir = /data
        ReferencedFile dataDirRef = new ReferencedFile(nodeDirectory, dataPath);
        // note: we do not check for existence here - in some cases
        // this directory may not exist (when exported and empty
        // directories are pruned)
        NodeSettingsRO portSettings = dataSettings.getNodeSettings(CFG_OUTPUT_PREFIX + index);
        String dataName = portSettings.getString(CFG_DATA_FILE_DIR);
        // dir = /data/data_i
        ReferencedFile dirRef = new ReferencedFile(dataDirRef, dataName);
        BufferedDataTable t = BufferedDataTable.loadFromFile(dirRef, portSettings, execSubData, loadTblRep, // no blobs or file stores in 1.1.x
        new HashMap<Integer, ContainerTable>(), new WorkflowFileStoreHandlerRepository());
        t.setOwnerRecursively(node);
        return t;
    } else {
        NodeSettingsRO dataSettings = settings.getNodeSettings(CFG_DATA_FILE);
        String dataDirStr = dataSettings.getString(CFG_DATA_FILE_DIR);
        ReferencedFile dataDirRef = new ReferencedFile(nodeDirectory, dataDirStr);
        NodeSettingsRO portSettings = dataSettings.getNodeSettings(CFG_OUTPUT_PREFIX + index);
        String dataName = portSettings.getString(CFG_DATA_FILE_DIR);
        ReferencedFile dirRef = new ReferencedFile(dataDirRef, dataName);
        File dir = dirRef.getFile();
        readDirectory(dir);
        BufferedDataTable t = BufferedDataTable.loadFromFile(dirRef, /* ignored in 1.2.0+ */
        null, execMon, loadTblRep, tblRep, fileStoreHandlerRepository);
        t.setOwnerRecursively(node);
        return t;
    }
}

Example 28 with ReferencedFile

use of org.knime.core.internal.ReferencedFile in project knime-core by knime.

the class FileNodePersistor method save.

/**
 * Saves the node, node settings, and all internal structures, spec, data, and models, to the given node directory
 * (located at the node file).
 *
 * @param nnc ...
 * @param settings ...
 * @param execMon Used to report progress during saving.
 * @param nodeDirRef Directory associated with node - will create internals folder in it
 * @param isSaveData  ...
 * @throws IOException If the node file can't be found or read.
 * @throws CanceledExecutionException If the saving has been canceled.
 * @since 2.9
 */
public static void save(final NativeNodeContainer nnc, final NodeSettingsWO settings, final ExecutionMonitor execMon, final ReferencedFile nodeDirRef, final boolean isSaveData) throws IOException, CanceledExecutionException {
    final Node node = nnc.getNode();
    saveCustomName(node, settings);
    saveHasContent(node, settings);
    saveIsInactive(node, settings);
    ReferencedFile nodeInternDirRef = getNodeInternDirectory(nodeDirRef);
    File nodeInternDir = nodeInternDirRef.getFile();
    if (nodeInternDir.exists()) {
        FileUtil.deleteRecursively(nodeInternDir);
    }
    ExecutionMonitor internalMon = execMon.createSilentSubProgress(0.2);
    ExecutionMonitor portMon = execMon.createSilentSubProgress(0.5);
    ExecutionMonitor intTblsMon = execMon.createSilentSubProgress(0.1);
    ExecutionMonitor fileStoreMon = execMon.createSilentSubProgress(0.1);
    execMon.setMessage("Internals");
    boolean isSaveInternals = isSaveData;
    if (isSaveInternals) {
        // do not save data for inactive nodes
        isSaveInternals = node.isInactiveBranchConsumer() || !node.isInactive();
    }
    if (isSaveInternals) {
        NodeContext.pushContext(nnc);
        try {
            saveNodeInternDirectory(node, nodeInternDir, settings, internalMon);
        } finally {
            NodeContext.removeLastContext();
        }
    }
    internalMon.setProgress(1.0);
    /* A hash set of all tables that originate from the corresponding node
         * (owner ID of the table equals NodeID), which have already been saved.
         * It is used to avoid multiple saves of the same table, e.g. when one
         * table is returned in multiple outputs or if an output table is used
         * as "internal" held table. See bug 2117. */
    final Set<Integer> savedTableIDs = new HashSet<Integer>();
    execMon.setMessage("Ports");
    savePorts(node, nodeDirRef, settings, savedTableIDs, portMon, isSaveData);
    portMon.setProgress(1.0);
    execMon.setMessage("Internal Tables");
    saveInternalHeldTables(node, nodeDirRef, settings, savedTableIDs, internalMon, isSaveData);
    intTblsMon.setProgress(1.0);
    // save them last as now all tables have been saved (all cells ran through persistor) and all
    // FileStore#getFile() have been called and saved
    execMon.setMessage("File Store Objects");
    saveFileStoreObjects(node, nodeDirRef, settings, fileStoreMon, isSaveData);
    fileStoreMon.setProgress(1.0);
    execMon.setProgress(1.0);
}

Example 29 with ReferencedFile

use of org.knime.core.internal.ReferencedFile in project knime-core by knime.

the class FileNodePersistor method saveFileStoreObjects.

/**
 * @param node
 * @param nodeDirRef
 * @param settings
 * @param fileStoreMon
 * @param isSaveData
 * @throws IOException
 */
private static void saveFileStoreObjects(final Node node, final ReferencedFile nodeDirRef, final NodeSettingsWO settings, final ExecutionMonitor fileStoreMon, final boolean isSaveData) throws IOException {
    NodeSettingsWO fsSettings = settings.addNodeSettings("filestores");
    IFileStoreHandler fileStoreHandler = node.getFileStoreHandler();
    String uuidS;
    String dirNameInFlow;
    if (isSaveData && fileStoreHandler instanceof WriteFileStoreHandler) {
        final WriteFileStoreHandler defFileStoreHandler = (WriteFileStoreHandler) fileStoreHandler;
        File baseDir = defFileStoreHandler.getBaseDir();
        dirNameInFlow = baseDir == null ? null : FILESTORE_FOLDER_PREFIX;
        if (dirNameInFlow != null) {
            File saveLocation = new File(nodeDirRef.getFile(), dirNameInFlow);
            FileUtil.copyDir(baseDir, saveLocation);
        }
        uuidS = defFileStoreHandler.getStoreUUID().toString();
    } else {
        uuidS = null;
        dirNameInFlow = null;
    }
    fsSettings.addString("file_store_location", dirNameInFlow);
    fsSettings.addString("file_store_id", uuidS);
}

Example 30 with ReferencedFile

use of org.knime.core.internal.ReferencedFile in project knime-core by knime.

the class FileNodePersistor method loadInternalHeldObjects.

/**
 * New with {@link LoadVersion#V2100}, supports {@link org.knime.core.node.port.PortObjectHolder}.
 */
void loadInternalHeldObjects(final Node node, final ExecutionMonitor execMon, final NodeSettingsRO settings, final Map<Integer, BufferedDataTable> loadTblRep, final HashMap<Integer, ContainerTable> tblRep, final FileStoreHandlerRepository fileStoreHandlerRepository) throws IOException, InvalidSettingsException, CanceledExecutionException {
    assert !getLoadVersion().isOlderThan(FileWorkflowPersistor.LoadVersion.V2100);
    if (!settings.containsKey("internalObjects")) {
        return;
    }
    NodeSettingsRO subSettings = settings.getNodeSettings("internalObjects");
    String subDirName = subSettings.getString("location");
    ReferencedFile subDirFile = new ReferencedFile(getNodeDirectory(), subDirName);
    NodeSettingsRO portSettings = subSettings.getNodeSettings("content");
    Set<String> keySet = portSettings.keySet();
    PortObject[] result = new PortObject[keySet.size()];
    for (String s : keySet) {
        ExecutionMonitor subProgress = execMon.createSubProgress(1.0 / result.length);
        NodeSettingsRO singlePortSetting = portSettings.getNodeSettings(s);
        int index = singlePortSetting.getInt("index");
        if (index < 0 || index >= result.length) {
            throw new InvalidSettingsException("Invalid index: " + index);
        }
        String type = singlePortSetting.getString("type");
        PortObject object = null;
        if ("null".equals(type)) {
        // object stays null
        } else if ("table".equals(type)) {
            String location = singlePortSetting.getString("table_dir_location");
            if (location != null) {
                ReferencedFile portDirRef = new ReferencedFile(subDirFile, location);
                readDirectory(portDirRef.getFile());
                BufferedDataTable t = loadBufferedDataTable(portDirRef, subProgress, loadTblRep, tblRep, fileStoreHandlerRepository);
                t.setOwnerRecursively(node);
                object = t;
            }
        } else if ("referenced_output".equals(type)) {
            int outputPortIndex = singlePortSetting.getInt("outport");
            CheckUtils.checkSetting(outputPortIndex >= 0, "Port index must not < 0: $d", outputPortIndex);
            object = getPortObject(outputPortIndex);
        } else if ("non-table".equals(type)) {
            String location = singlePortSetting.getString("port_dir_location");
            ReferencedFile portDirRef = new ReferencedFile(subDirFile, location);
            readDirectory(portDirRef.getFile());
            object = loadPortObject(portDirRef, singlePortSetting, subProgress, fileStoreHandlerRepository).orElseThrow(() -> new IOException("Settings do not reference internal held port object"));
        } else {
            CheckUtils.checkSetting(false, "Unknown object reference %s", type);
        }
        result[index] = object;
        subProgress.setProgress(1.0);
    }
    setInternalHeldPortObjects(result);
}