Examples with FlowletConnection co.cask.cdap.api.flow.FlowletConnection used on opensource projects

Example 1 with FlowletConnection

use of co.cask.cdap.api.flow.FlowletConnection in project cdap by caskdata.

the class DefaultFlowConfigurer method connect.

@Override
public void connect(String from, String to) {
    Preconditions.checkArgument(from != null && to != null, UserMessages.getMessage(UserErrors.INVALID_FLOWLET_NULL));
    Preconditions.checkArgument(flowlets.containsKey(from), UserMessages.getMessage(UserErrors.INVALID_FLOWLET_NAME), from);
    Preconditions.checkArgument(flowlets.containsKey(to), UserMessages.getMessage(UserErrors.INVALID_FLOWLET_NAME), to);
    connections.add(new FlowletConnection(FlowletConnection.Type.FLOWLET, from, to));
}

Example 2 with FlowletConnection

use of co.cask.cdap.api.flow.FlowletConnection in project cdap by caskdata.

the class DefaultFlowConfigurer method connectStream.

@Override
public void connectStream(@Nullable String namespace, String stream, String flowlet) {
    Preconditions.checkNotNull(stream, UserMessages.getMessage(UserErrors.INVALID_STREAM_NULL));
    Preconditions.checkNotNull(flowlet, UserMessages.getMessage(UserErrors.INVALID_FLOWLET_NULL));
    Preconditions.checkArgument(flowlets.containsKey(flowlet), UserMessages.getMessage(UserErrors.INVALID_FLOWLET_NAME), flowlet);
    connections.add(new FlowletConnection(FlowletConnection.Type.STREAM, namespace, stream, flowlet));
}

Example 3 with FlowletConnection

use of co.cask.cdap.api.flow.FlowletConnection in project cdap by caskdata.

the class ApplicationLifecycleService method deleteApp.

// deletes without performs checks that no programs are running
/**
   * Delete the specified application without performing checks that its programs are stopped.
   *
   * @param appId the id of the application to delete
   * @param spec the spec of the application to delete
   * @throws Exception
   */
private void deleteApp(final ApplicationId appId, ApplicationSpecification spec) throws Exception {
    Id.Application idApplication = appId.toId();
    // enforce ADMIN privileges on the app
    authorizationEnforcer.enforce(appId, authenticationContext.getPrincipal(), Action.ADMIN);
    // first remove all privileges on the app
    revokePrivileges(appId, spec);
    //Delete the schedules
    scheduler.deleteSchedules(appId);
    deleteMetrics(appId);
    //Delete all preferences of the application and of all its programs
    deletePreferences(appId);
    // TODO: This should be unified with the DeletedProgramHandlerStage
    for (final FlowSpecification flowSpecification : spec.getFlows().values()) {
        Id.Program flowProgramId = Id.Program.from(idApplication, ProgramType.FLOW, flowSpecification.getName());
        // Collects stream name to all group ids consuming that stream
        final Multimap<String, Long> streamGroups = HashMultimap.create();
        for (FlowletConnection connection : flowSpecification.getConnections()) {
            if (connection.getSourceType() == FlowletConnection.Type.STREAM) {
                long groupId = FlowUtils.generateConsumerGroupId(flowProgramId, connection.getTargetName());
                streamGroups.put(connection.getSourceName(), groupId);
            }
        }
        // Remove all process states and group states for each stream
        final String namespace = String.format("%s.%s", flowProgramId.getApplicationId(), flowProgramId.getId());
        impersonator.doAs(appId, new Callable<Void>() {

            // TODO: (CDAP-7326) since one worker or flow can only be ran by a single instance of APP, (also a single
            // version), should delete flow for each version
            @Override
            public Void call() throws Exception {
                for (Map.Entry<String, Collection<Long>> entry : streamGroups.asMap().entrySet()) {
                    streamConsumerFactory.dropAll(appId.getParent().stream(entry.getKey()), namespace, entry.getValue());
                }
                queueAdmin.dropAllForFlow(appId.flow(flowSpecification.getName()));
                return null;
            }
        });
    }
    ApplicationSpecification appSpec = store.getApplication(appId);
    deleteAppMetadata(appId, appSpec);
    deleteRouteConfig(appId, appSpec);
    store.deleteWorkflowStats(appId);
    store.removeApplication(appId);
    try {
        // delete the owner as it has already been determined that this is the only version of the app
        ownerAdmin.delete(appId);
    } catch (Exception e) {
        LOG.warn("Failed to delete app owner principal for application {} if one existed while deleting the " + "application.", appId);
    }
    try {
        usageRegistry.unregister(appId);
    } catch (Exception e) {
        LOG.warn("Failed to unregister usage of app: {}", appId, e);
    }
}

Example 4 with FlowletConnection

use of co.cask.cdap.api.flow.FlowletConnection in project cdap by caskdata.

the class FlowVerification method verify.

/**
   * Verifies a single {@link FlowSpecification} for a {@link co.cask.cdap.api.flow.Flow}.
   *
   * @param input to be verified
   * @return An instance of {@link VerifyResult} depending of status of verification.
   */
@Override
public VerifyResult verify(ApplicationId appId, final FlowSpecification input) {
    VerifyResult verifyResult = super.verify(appId, input);
    if (!verifyResult.isSuccess()) {
        return verifyResult;
    }
    String flowName = input.getName();
    // Check if there are no flowlets.
    if (input.getFlowlets().isEmpty()) {
        return VerifyResult.failure(Err.Flow.ATLEAST_ONE_FLOWLET, flowName);
    }
    // Check if there no connections.
    if (input.getConnections().isEmpty()) {
        return VerifyResult.failure(Err.Flow.ATLEAST_ONE_CONNECTION, flowName);
    }
    // We go through each Flowlet and verify the flowlets.
    // First collect all source flowlet names
    Set<String> sourceFlowletNames = Sets.newHashSet();
    for (FlowletConnection connection : input.getConnections()) {
        if (connection.getSourceType() == FlowletConnection.Type.FLOWLET) {
            sourceFlowletNames.add(connection.getSourceName());
        }
    }
    for (Map.Entry<String, FlowletDefinition> entry : input.getFlowlets().entrySet()) {
        FlowletDefinition defn = entry.getValue();
        String flowletName = defn.getFlowletSpec().getName();
        // Check if the Flowlet Name is an ID.
        if (!EntityId.isValidId(defn.getFlowletSpec().getName())) {
            return VerifyResult.failure(Err.NOT_AN_ID, flowName + ":" + flowletName);
        }
        // We check if all the dataset names used are ids
        for (String dataSet : defn.getDatasets()) {
            if (!EntityId.isValidDatasetId(dataSet)) {
                return VerifyResult.failure(Err.NOT_AN_ID, flowName + ":" + flowletName + ":" + dataSet);
            }
        }
        // Check if the flowlet has output, it must be appear as source flowlet in at least one connection
        if (entry.getValue().getOutputs().size() > 0 && !sourceFlowletNames.contains(flowletName)) {
            return VerifyResult.failure(Err.Flow.OUTPUT_NOT_CONNECTED, flowName, flowletName);
        }
    }
    // NOTE: We should unify the logic here and the queue spec generation, as they are doing the same thing.
    Table<QueueSpecificationGenerator.Node, String, Set<QueueSpecification>> queueSpecTable = new SimpleQueueSpecificationGenerator(appId).create(input);
    // For all connections, there should be an entry in the table.
    for (FlowletConnection connection : input.getConnections()) {
        QueueSpecificationGenerator.Node node;
        if (connection.getSourceType() == FlowletConnection.Type.FLOWLET) {
            node = new QueueSpecificationGenerator.Node(connection.getSourceType(), connection.getSourceName());
        } else {
            String sourceNamespace = connection.getSourceNamespace() == null ? appId.getNamespace() : connection.getSourceNamespace();
            node = new QueueSpecificationGenerator.Node(connection.getSourceType(), sourceNamespace, connection.getSourceName());
        }
        if (!queueSpecTable.contains(node, connection.getTargetName())) {
            return VerifyResult.failure(Err.Flow.NO_INPUT_FOR_OUTPUT, flowName, connection.getTargetName(), connection.getSourceType(), connection.getSourceName());
        }
    }
    // For each output entity, check for any unconnected output
    for (QueueSpecificationGenerator.Node node : queueSpecTable.rowKeySet()) {
        // For stream output, no need to check
        if (node.getType() == FlowletConnection.Type.STREAM) {
            continue;
        }
        // For all outputs of a flowlet, remove all the matched connected schema, if there is anything left,
        // then it's a incomplete flow connection (has output not connect to any input).
        Multimap<String, Schema> outputs = toMultimap(input.getFlowlets().get(node.getName()).getOutputs());
        for (Map.Entry<String, Set<QueueSpecification>> entry : queueSpecTable.row(node).entrySet()) {
            for (QueueSpecification queueSpec : entry.getValue()) {
                outputs.remove(queueSpec.getQueueName().getSimpleName(), queueSpec.getOutputSchema());
            }
        }
        if (!outputs.isEmpty()) {
            return VerifyResult.failure(Err.Flow.MORE_OUTPUT_NOT_ALLOWED, flowName, node.getType().toString().toLowerCase(), node.getName(), outputs);
        }
    }
    return VerifyResult.success();
}

Example 5 with FlowletConnection

use of co.cask.cdap.api.flow.FlowletConnection in project cdap by caskdata.

the class FlowSpecificationCodec method deserialize.

@Override
public FlowSpecification deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) throws JsonParseException {
    JsonObject jsonObj = json.getAsJsonObject();
    String className = jsonObj.get("className").getAsString();
    String name = jsonObj.get("name").getAsString();
    String description = jsonObj.get("description").getAsString();
    Map<String, FlowletDefinition> flowlets = deserializeMap(jsonObj.get("flowlets"), context, FlowletDefinition.class);
    List<FlowletConnection> connections = deserializeList(jsonObj.get("connections"), context, FlowletConnection.class);
    return new DefaultFlowSpecification(className, name, description, flowlets, connections);
}