Search in sources :

Example 1 with StringUtils.removeStart

use of org.apache.commons.lang3.StringUtils.removeStart in project knime-core by knime.

the class WorkflowManager method getExternalNodeDataHandles.

/**
 * Implementation of {@link #getInputNodes()} and {@link #getExternalOutputs()}.
 *
 * @param nodeModelClass either {@link InputNode}.class or {@link OutputNode}.class.
 * @param retriever resolves the data object from the input or output.
 * @return List of nodes with their parameter names unified.
 */
// package scope for tests
<T> List<ExternalNodeDataHandle> getExternalNodeDataHandles(final Class<T> nodeModelClass, final Function<T, ExternalNodeData> retriever) {
    List<ExternalNodeDataHandle> result = new ArrayList<>();
    try (WorkflowLock lock = lock()) {
        Map<NodeID, T> externalNodeDataMap = findNodes(nodeModelClass, NodeModelFilter.all(), true, true);
        // get all "parameter names" from all nodes in the workflow in order to see if there are name conflicts;
        // occurrence map like {"string-input" -> 1, "foo" -> 1, "bar" -> 3}
        Map<String, Long> parameterNamesToCountMap = externalNodeDataMap.values().stream().map(retriever).map(extNodeData -> extNodeData.getID()).collect(Collectors.groupingBy(Function.identity(), Collectors.counting()));
        parameterNamesToCountMap.values().removeAll(Collections.singleton(Long.valueOf(1L)));
        Set<String> nonUniqueParameterNames = parameterNamesToCountMap.keySet();
        if (!nonUniqueParameterNames.isEmpty()) {
            LOGGER.warnWithFormat("Workflow contains nodes with duplicate parameter name " + "(will be made unique by appending node IDs): %s", ConvenienceMethods.getShortStringFrom(nonUniqueParameterNames, 5));
        }
        // create result list, make keys unique where needed but also retain the fully qualified parameter name
        for (Map.Entry<NodeID, T> entry : externalNodeDataMap.entrySet()) {
            NodeID nodeID = entry.getKey();
            ExternalNodeData nodeData = retriever.apply(entry.getValue());
            String parameterNameShort = nodeData.getID();
            // this ID = 0:3, nodeID = 0:3:5:0:2:2 => nodeIDRelativePath = 5:0:2:2
            String nodeIDRelativePath = StringUtils.removeStart(nodeID.toString(), getID().toString() + ":");
            // nodeIDRelativePath = 5:0:2:2 --> 5:2:2 (':0' are internals of a subnode)
            nodeIDRelativePath = StringUtils.remove(nodeIDRelativePath, ":0");
            String parameterNameFullyQualified = (parameterNameShort.isEmpty() ? "" : parameterNameShort + "-") + nodeIDRelativePath;
            if (nonUniqueParameterNames.contains(parameterNameShort)) {
                parameterNameShort = parameterNameFullyQualified;
            }
            result.add(new ExternalNodeDataHandle(parameterNameShort, parameterNameFullyQualified, (NativeNodeContainer) findNodeContainer(nodeID), nodeData));
        }
        return result;
    }
}
Also used : Arrays(java.util.Arrays) NodeSettingsRO(org.knime.core.node.NodeSettingsRO) EXECUTING(org.knime.core.node.workflow.InternalNodeContainerState.EXECUTING) ReferencedFile(org.knime.core.internal.ReferencedFile) CanceledExecutionException(org.knime.core.node.CanceledExecutionException) IWriteFileStoreHandler(org.knime.core.data.filestore.internal.IWriteFileStoreHandler) InteractiveNode(org.knime.core.node.interactive.InteractiveNode) CoreException(org.eclipse.core.runtime.CoreException) StringUtils(org.apache.commons.lang3.StringUtils) AbstractQuickFormValueInConfiguration(org.knime.core.quickform.AbstractQuickFormValueInConfiguration) NodeView(org.knime.core.node.NodeView) NodeContainerExecutionStatus(org.knime.core.node.workflow.execresult.NodeContainerExecutionStatus) Node(org.knime.core.node.Node) Vector(java.util.Vector) Matcher(java.util.regex.Matcher) WorkflowFileStoreHandlerRepository(org.knime.core.data.filestore.internal.WorkflowFileStoreHandlerRepository) Map(java.util.Map) FileStoreHandlerRepository(org.knime.core.data.filestore.internal.FileStoreHandlerRepository) LockFailedException(org.knime.core.util.LockFailedException) ExecutionMonitor(org.knime.core.node.ExecutionMonitor) EXECUTINGREMOTELY(org.knime.core.node.workflow.InternalNodeContainerState.EXECUTINGREMOTELY) Set(java.util.Set) ResolverUtil(org.knime.core.util.pathresolve.ResolverUtil) Executors(java.util.concurrent.Executors) POSTEXECUTE(org.knime.core.node.workflow.InternalNodeContainerState.POSTEXECUTE) InputNode(org.knime.core.node.dialog.InputNode) ArrayBlockingQueue(java.util.concurrent.ArrayBlockingQueue) Stream(java.util.stream.Stream) CONFIGURED(org.knime.core.node.workflow.InternalNodeContainerState.CONFIGURED) NoSuchAlgorithmException(java.security.NoSuchAlgorithmException) ThreadNodeExecutionJobManager(org.knime.core.node.exec.ThreadNodeExecutionJobManager) CollapseIntoMetaNodeResult(org.knime.core.node.workflow.action.CollapseIntoMetaNodeResult) CopyOnWriteArrayList(java.util.concurrent.CopyOnWriteArrayList) VirtualParallelizedChunkPortObjectInNodeModel(org.knime.core.node.workflow.virtual.parchunk.VirtualParallelizedChunkPortObjectInNodeModel) FlowVariablePortObject(org.knime.core.node.port.flowvariable.FlowVariablePortObject) VMFileLocker(org.knime.core.util.VMFileLocker) RestoredFlowLoopContext(org.knime.core.node.workflow.FlowLoopContext.RestoredFlowLoopContext) SplitType(org.knime.core.node.workflow.NodeContainer.NodeContainerSettings.SplitType) AbstractQuickFormConfiguration(org.knime.core.quickform.AbstractQuickFormConfiguration) EXECUTED_QUEUED(org.knime.core.node.workflow.InternalNodeContainerState.EXECUTED_QUEUED) ArrayList(java.util.ArrayList) LinkedHashMap(java.util.LinkedHashMap) NotConfigurableException(org.knime.core.node.NotConfigurableException) UpdateStatus(org.knime.core.node.workflow.MetaNodeTemplateInformation.UpdateStatus) IConfigurationElement(org.eclipse.core.runtime.IConfigurationElement) WorkflowExecutionResult(org.knime.core.node.workflow.execresult.WorkflowExecutionResult) DialogNode(org.knime.core.node.dialog.DialogNode) LinkedHashSet(java.util.LinkedHashSet) VirtualParallelizedChunkNodeInput(org.knime.core.node.workflow.virtual.parchunk.VirtualParallelizedChunkNodeInput) KNIMEConstants(org.knime.core.node.KNIMEConstants) LoadVersion(org.knime.core.node.workflow.FileWorkflowPersistor.LoadVersion) QuickFormInputNode(org.knime.core.quickform.in.QuickFormInputNode) Executor(java.util.concurrent.Executor) PREEXECUTE(org.knime.core.node.workflow.InternalNodeContainerState.PREEXECUTE) InteractiveView(org.knime.core.node.interactive.InteractiveView) FileOutputStream(java.io.FileOutputStream) PortObjectSpec(org.knime.core.node.port.PortObjectSpec) IOException(java.io.IOException) FileUtils(org.apache.commons.io.FileUtils) File(java.io.File) ViewContent(org.knime.core.node.interactive.ViewContent) UNCONFIGURED_MARKEDFOREXEC(org.knime.core.node.workflow.InternalNodeContainerState.UNCONFIGURED_MARKEDFOREXEC) OutputNode(org.knime.core.node.dialog.OutputNode) Platform(org.eclipse.core.runtime.Platform) LoopStatus(org.knime.core.node.workflow.NativeNodeContainer.LoopStatus) FileUtil(org.knime.core.util.FileUtil) CONFIGURED_QUEUED(org.knime.core.node.workflow.InternalNodeContainerState.CONFIGURED_QUEUED) NodeType(org.knime.core.node.NodeFactory.NodeType) ExpandSubnodeResult(org.knime.core.node.workflow.action.ExpandSubnodeResult) ListIterator(java.util.ListIterator) InvalidSettingsException(org.knime.core.node.InvalidSettingsException) URL(java.net.URL) Date(java.util.Date) URISyntaxException(java.net.URISyntaxException) WorkflowLoadResult(org.knime.core.node.workflow.WorkflowPersistor.WorkflowLoadResult) ReexecutionCallback(org.knime.core.node.interactive.ReexecutionCallback) MetaNodeLinkUpdateResult(org.knime.core.node.workflow.WorkflowPersistor.MetaNodeLinkUpdateResult) FileFilterUtils(org.apache.commons.io.filefilter.FileFilterUtils) NodeDialogPane(org.knime.core.node.NodeDialogPane) VirtualParallelizedChunkPortObjectInNodeFactory(org.knime.core.node.workflow.virtual.parchunk.VirtualParallelizedChunkPortObjectInNodeFactory) IExtensionRegistry(org.eclipse.core.runtime.IExtensionRegistry) IExtensionPoint(org.eclipse.core.runtime.IExtensionPoint) AbstractNodeView(org.knime.core.node.AbstractNodeView) ConvenienceMethods(org.knime.core.node.util.ConvenienceMethods) URI(java.net.URI) ThreadFactory(java.util.concurrent.ThreadFactory) ContainerTable(org.knime.core.data.container.ContainerTable) MetaNodeToSubNodeResult(org.knime.core.node.workflow.action.MetaNodeToSubNodeResult) PortType(org.knime.core.node.port.PortType) NodeAndInports(org.knime.core.node.workflow.Workflow.NodeAndInports) CONFIGURED_MARKEDFOREXEC(org.knime.core.node.workflow.InternalNodeContainerState.CONFIGURED_MARKEDFOREXEC) Collection(java.util.Collection) CredentialsNode(org.knime.core.node.workflow.CredentialsStore.CredentialsNode) NodeModel(org.knime.core.node.NodeModel) Collectors(java.util.stream.Collectors) Objects(java.util.Objects) List(java.util.List) BufferedDataTable(org.knime.core.node.BufferedDataTable) WorkflowPortTemplate(org.knime.core.node.workflow.WorkflowPersistor.WorkflowPortTemplate) LoadResultEntryType(org.knime.core.node.workflow.WorkflowPersistor.LoadResultEntry.LoadResultEntryType) NodeContainerTemplateLinkUpdateResult(org.knime.core.node.workflow.WorkflowPersistor.NodeContainerTemplateLinkUpdateResult) HiLiteHandler(org.knime.core.node.property.hilite.HiLiteHandler) Entry(java.util.Map.Entry) Optional(java.util.Optional) CheckUtils(org.knime.core.node.util.CheckUtils) CheckUtils.checkState(org.knime.core.node.util.CheckUtils.checkState) PortObject(org.knime.core.node.port.PortObject) NodeContainerExecutionResult(org.knime.core.node.workflow.execresult.NodeContainerExecutionResult) IntStream(java.util.stream.IntStream) IOCase(org.apache.commons.io.IOCase) ThreadPoolExecutor(java.util.concurrent.ThreadPoolExecutor) NodeExecutionJobManagerPool(org.knime.core.node.util.NodeExecutionJobManagerPool) SingleNodeContainerSettings(org.knime.core.node.workflow.SingleNodeContainer.SingleNodeContainerSettings) EXECUTED_MARKEDFOREXEC(org.knime.core.node.workflow.InternalNodeContainerState.EXECUTED_MARKEDFOREXEC) HashMap(java.util.HashMap) NodeSettings(org.knime.core.node.NodeSettings) ConnectionType(org.knime.core.node.workflow.ConnectionContainer.ConnectionType) Function(java.util.function.Function) Pair(org.knime.core.util.Pair) HashSet(java.util.HashSet) IDLE(org.knime.core.node.workflow.InternalNodeContainerState.IDLE) ExternalNodeData(org.knime.core.node.dialog.ExternalNodeData) ConnectionContainerTemplate(org.knime.core.node.workflow.WorkflowPersistor.ConnectionContainerTemplate) NodeLogger(org.knime.core.node.NodeLogger) NodeCreationContext(org.knime.core.node.NodeCreationContext) EXECUTED(org.knime.core.node.workflow.InternalNodeContainerState.EXECUTED) LoadResult(org.knime.core.node.workflow.WorkflowPersistor.LoadResult) LinkedList(java.util.LinkedList) Role(org.knime.core.node.workflow.MetaNodeTemplateInformation.Role) InteractiveWebViewsResult(org.knime.core.node.workflow.action.InteractiveWebViewsResult) SubNodeToMetaNodeResult(org.knime.core.node.workflow.action.SubNodeToMetaNodeResult) OutputStream(java.io.OutputStream) NodeFactory(org.knime.core.node.NodeFactory) Iterator(java.util.Iterator) ReentrantLock(java.util.concurrent.locks.ReentrantLock) TemplateType(org.knime.core.node.workflow.MetaNodeTemplateInformation.TemplateType) MetaPortInfo(org.knime.core.node.port.MetaPortInfo) VirtualParallelizedChunkPortObjectOutNodeFactory(org.knime.core.node.workflow.virtual.parchunk.VirtualParallelizedChunkPortObjectOutNodeFactory) MetaNodeDialogNode(org.knime.core.node.dialog.MetaNodeDialogNode) ParallelizedChunkContent(org.knime.core.node.workflow.virtual.parchunk.ParallelizedChunkContent) NodeSettingsWO(org.knime.core.node.NodeSettingsWO) Type(org.knime.core.node.workflow.NodeMessage.Type) TimeUnit(java.util.concurrent.TimeUnit) Condition(java.util.concurrent.locks.Condition) IEarlyStartup(org.knime.core.util.IEarlyStartup) ParallelizedChunkContentMaster(org.knime.core.node.workflow.virtual.parchunk.ParallelizedChunkContentMaster) IFileStoreHandler(org.knime.core.data.filestore.internal.IFileStoreHandler) NodeProperty(org.knime.core.node.workflow.NodePropertyChangedEvent.NodeProperty) Collections(java.util.Collections) ExternalNodeData(org.knime.core.node.dialog.ExternalNodeData) CopyOnWriteArrayList(java.util.concurrent.CopyOnWriteArrayList) ArrayList(java.util.ArrayList) Map(java.util.Map) LinkedHashMap(java.util.LinkedHashMap) HashMap(java.util.HashMap)

Example 2 with StringUtils.removeStart

use of org.apache.commons.lang3.StringUtils.removeStart in project engine by craftercms.

the class S3SiteListResolver method getSiteListFromBucketKeys.

protected Collection<String> getSiteListFromBucketKeys(String bucketName, String rootPrefix) {
    List<String> siteNames = new ArrayList<>();
    AmazonS3 client = clientBuilder.getClient();
    ListObjectsV2Request request = new ListObjectsV2Request().withBucketName(bucketName).withPrefix(rootPrefix).withDelimiter(DELIMITER);
    ListObjectsV2Result result = client.listObjectsV2(request);
    if (CollectionUtils.isNotEmpty(result.getCommonPrefixes())) {
        result.getCommonPrefixes().stream().map(prefix -> StringUtils.stripEnd(StringUtils.removeStart(prefix, rootPrefix), DELIMITER)).forEach(siteNames::add);
    }
    return siteNames;
}
Also used : SiteListResolver(org.craftercms.engine.service.context.SiteListResolver) Collection(java.util.Collection) Bucket(com.amazonaws.services.s3.model.Bucket) StringUtils(org.apache.commons.lang3.StringUtils) CollectionUtils(org.apache.commons.collections4.CollectionUtils) DELIMITER(org.craftercms.engine.store.s3.S3ContentStoreAdapter.DELIMITER) ListObjectsV2Result(com.amazonaws.services.s3.model.ListObjectsV2Result) ArrayList(java.util.ArrayList) ListObjectsV2Request(com.amazonaws.services.s3.model.ListObjectsV2Request) List(java.util.List) Matcher(java.util.regex.Matcher) SiteContextFactory(org.craftercms.engine.service.context.SiteContextFactory) AmazonS3(com.amazonaws.services.s3.AmazonS3) Required(org.springframework.beans.factory.annotation.Required) Pattern(java.util.regex.Pattern) S3ClientBuilder(org.craftercms.engine.store.s3.util.S3ClientBuilder) AmazonS3URI(com.amazonaws.services.s3.AmazonS3URI) AmazonS3(com.amazonaws.services.s3.AmazonS3) ListObjectsV2Request(com.amazonaws.services.s3.model.ListObjectsV2Request) ListObjectsV2Result(com.amazonaws.services.s3.model.ListObjectsV2Result) ArrayList(java.util.ArrayList)

Aggregations

ArrayList (java.util.ArrayList)2 Collection (java.util.Collection)2 List (java.util.List)2 Matcher (java.util.regex.Matcher)2 StringUtils (org.apache.commons.lang3.StringUtils)2 AmazonS3 (com.amazonaws.services.s3.AmazonS3)1 AmazonS3URI (com.amazonaws.services.s3.AmazonS3URI)1 Bucket (com.amazonaws.services.s3.model.Bucket)1 ListObjectsV2Request (com.amazonaws.services.s3.model.ListObjectsV2Request)1 ListObjectsV2Result (com.amazonaws.services.s3.model.ListObjectsV2Result)1 File (java.io.File)1 FileOutputStream (java.io.FileOutputStream)1 IOException (java.io.IOException)1 OutputStream (java.io.OutputStream)1 URI (java.net.URI)1 URISyntaxException (java.net.URISyntaxException)1 URL (java.net.URL)1 NoSuchAlgorithmException (java.security.NoSuchAlgorithmException)1 Arrays (java.util.Arrays)1 Collections (java.util.Collections)1