Examples with PluginRequirement io.cdap.cdap.internal.pipeline.PluginRequirement used on opensource projects

Example 1 with PluginRequirement

use of io.cdap.cdap.internal.pipeline.PluginRequirement in project cdap by caskdata.

the class ProvisioningService method getUnfulfilledRequirements.

/**
 * Checks if the given provisioner fulfills all the requirements of a program.
 *
 * @param provisionerFulfilledRequirements {@link Requirements} containing requirements fulfilled by the provisioner
 * @param requirements a {@link Set} of {@link PluginRequirement}
 *
 * @return {@link Set} {@link PluginRequirement} which consists of plugin identifier and the requirements which
 * are not meet
 */
@VisibleForTesting
Set<PluginRequirement> getUnfulfilledRequirements(Capabilities provisionerFulfilledRequirements, Set<PluginRequirement> requirements) {
    // create a map of plugin name to unfulfilled requirement if there are any
    Set<PluginRequirement> unfulfilledRequirements = new HashSet<>();
    Set<String> capabilities = provisionerFulfilledRequirements.getDatasetTypes().stream().map(String::toLowerCase).collect(Collectors.toSet());
    for (PluginRequirement pluginRequirement : requirements) {
        Requirements requisites = pluginRequirement.getRequirements();
        Sets.SetView<String> unfulfilledRequirement = Sets.difference(requisites.getDatasetTypes(), capabilities);
        if (!unfulfilledRequirement.isEmpty()) {
            unfulfilledRequirements.add(new PluginRequirement(pluginRequirement.getName(), pluginRequirement.getType(), new Requirements(unfulfilledRequirement.immutableCopy())));
        }
    }
    return unfulfilledRequirements;
}

Example 2 with PluginRequirement

use of io.cdap.cdap.internal.pipeline.PluginRequirement in project cdap by caskdata.

the class ProvisioningService method provision.

/**
 * Record that a cluster will be provisioned for a program run, returning a Runnable that will actually perform
 * the cluster provisioning. This method must be run within a transaction.
 * The task returned should only be executed after the transaction that ran this method has completed.
 * Running the returned Runnable will start the actual task using an executor within this service so that it can be
 * tracked and optionally cancelled using {@link #cancelProvisionTask(ProgramRunId)}. The caller does not need to
 * submit the runnable using their own executor.
 *
 * @param provisionRequest the provision request
 * @param context context for the transaction
 * @return runnable that will actually execute the cluster provisioning
 */
public Runnable provision(ProvisionRequest provisionRequest, StructuredTableContext context) throws IOException {
    ProgramRunId programRunId = provisionRequest.getProgramRunId();
    ProgramOptions programOptions = provisionRequest.getProgramOptions();
    Map<String, String> args = programOptions.getArguments().asMap();
    String name = SystemArguments.getProfileProvisioner(args);
    Provisioner provisioner = provisionerInfo.get().provisioners.get(name);
    // any errors seen here will transition the state straight to deprovisioned since no cluster create was attempted
    if (provisioner == null) {
        runWithProgramLogging(programRunId, args, () -> LOG.error("Could not provision cluster for the run because provisioner {} does not exist.", name));
        programStateWriter.error(programRunId, new IllegalStateException("Provisioner does not exist."));
        provisionerNotifier.deprovisioned(programRunId);
        return () -> {
        };
    }
    // get plugin requirement information and check for capability to run on the provisioner
    Set<PluginRequirement> requirements = GSON.fromJson(args.get(ProgramOptionConstants.PLUGIN_REQUIREMENTS), PLUGIN_REQUIREMENT_SET_TYPE);
    if (requirements != null) {
        Set<PluginRequirement> unfulfilledRequirements = getUnfulfilledRequirements(provisioner.getCapabilities(), requirements);
        if (!unfulfilledRequirements.isEmpty()) {
            runWithProgramLogging(programRunId, args, () -> LOG.error(String.format("'%s' cannot be run using profile '%s' because the profile does not met all " + "plugin requirements. Following requirements were not meet by the listed " + "plugins: '%s'", programRunId.getProgram(), name, groupByRequirement(unfulfilledRequirements))));
            programStateWriter.error(programRunId, new IllegalArgumentException("Provisioner does not meet all the " + "requirements for the program to run."));
            provisionerNotifier.deprovisioned(programRunId);
            return () -> {
            };
        }
    }
    Map<String, String> properties = SystemArguments.getProfileProperties(args);
    ProvisioningOp provisioningOp = new ProvisioningOp(ProvisioningOp.Type.PROVISION, ProvisioningOp.Status.REQUESTING_CREATE);
    ProvisioningTaskInfo provisioningTaskInfo = new ProvisioningTaskInfo(programRunId, provisionRequest.getProgramDescriptor(), programOptions, properties, name, provisionRequest.getUser(), provisioningOp, createKeysDirectory(programRunId).toURI(), null);
    ProvisionerTable provisionerTable = new ProvisionerTable(context);
    provisionerTable.putTaskInfo(provisioningTaskInfo);
    return createProvisionTask(provisioningTaskInfo, provisioner);
}

Example 3 with PluginRequirement

use of io.cdap.cdap.internal.pipeline.PluginRequirement in project cdap by caskdata.

the class ProvisioningServiceTest method testUnfulfilledRequirements.

@Test
public void testUnfulfilledRequirements() {
    Capabilities provisionerCapabilities = new Capabilities(ImmutableSet.of(Table.TYPE));
    Set<PluginRequirement> requirements = ImmutableSet.of(new PluginRequirement("source1", "batchsource", new Requirements(ImmutableSet.of(Table.TYPE, "unicorn"))), new PluginRequirement("sink1", "batchsink", new Requirements(ImmutableSet.of(Table.TYPE, "dragon"))));
    Set<PluginRequirement> expectedUnfulfilledRequirements = ImmutableSet.of(new PluginRequirement("source1", "batchsource", new Requirements(ImmutableSet.of("unicorn"))), new PluginRequirement("sink1", "batchsink", new Requirements(ImmutableSet.of("dragon"))));
    assertRequirementFulfillment(provisionerCapabilities, requirements, expectedUnfulfilledRequirements);
    // check when there are multiple plugins with same name but different type
    requirements = ImmutableSet.of(new PluginRequirement("source1", "batchsource", new Requirements(ImmutableSet.of(Table.TYPE, "unicorn"))), new PluginRequirement("sink1", "batchsink", new Requirements(ImmutableSet.of(Table.TYPE, "dragon"))), new PluginRequirement("sink1", "anothersink", new Requirements(ImmutableSet.of(Table.TYPE, "narwhal"))));
    expectedUnfulfilledRequirements = ImmutableSet.of(new PluginRequirement("source1", "batchsource", new Requirements(ImmutableSet.of("unicorn"))), new PluginRequirement("sink1", "batchsink", new Requirements(ImmutableSet.of("dragon"))), new PluginRequirement("sink1", "anothersink", new Requirements(ImmutableSet.of("narwhal"))));
    assertRequirementFulfillment(provisionerCapabilities, requirements, expectedUnfulfilledRequirements);
    // check when provisioner does not have any specified capability
    provisionerCapabilities = Capabilities.EMPTY;
    assertRequirementFulfillment(provisionerCapabilities, requirements, requirements);
}

Example 4 with PluginRequirement

use of io.cdap.cdap.internal.pipeline.PluginRequirement in project cdap by caskdata.

the class ProvisioningServiceTest method testGroupByRequirement.

@Test
public void testGroupByRequirement() {
    Set<PluginRequirement> requirements = ImmutableSet.of(new PluginRequirement("source1", "batchsource", new Requirements(ImmutableSet.of(Table.TYPE, "unicorn"))), new PluginRequirement("sink1", "batchsink", new Requirements(ImmutableSet.of(Table.TYPE, "dragon"))), new PluginRequirement("sink1", "anothersink", new Requirements(ImmutableSet.of(Table.TYPE, "narwhal"))));
    Map<String, Set<String>> pluginGroupedByRequirement = provisioningService.groupByRequirement(requirements);
    Assert.assertEquals(4, pluginGroupedByRequirement.size());
    Assert.assertEquals(ImmutableSet.of("batchsource:source1", "batchsink:sink1", "anothersink:sink1"), pluginGroupedByRequirement.get(Table.TYPE));
    Assert.assertEquals(ImmutableSet.of("batchsource:source1"), pluginGroupedByRequirement.get("unicorn"));
    Assert.assertEquals(ImmutableSet.of("batchsink:sink1"), pluginGroupedByRequirement.get("dragon"));
    Assert.assertEquals(ImmutableSet.of("anothersink:sink1"), pluginGroupedByRequirement.get("narwhal"));
    // test empty
    pluginGroupedByRequirement = provisioningService.groupByRequirement(Collections.emptySet());
    Assert.assertTrue(pluginGroupedByRequirement.isEmpty());
}

Example 5 with PluginRequirement

use of io.cdap.cdap.internal.pipeline.PluginRequirement in project cdap by caskdata.

the class ProvisioningServiceTest method testFulfilledRequirements.

@Test
public void testFulfilledRequirements() {
    Capabilities provisionerCapabilities = new Capabilities(ImmutableSet.of(Table.TYPE));
    Set<PluginRequirement> requirements = ImmutableSet.of(new PluginRequirement("source1", "batchsource", new Requirements(Collections.emptySet())), new PluginRequirement("sink1", "batchsink", new Requirements(ImmutableSet.of(Table.TYPE))));
    // there should not be any incapability
    assertRequirementFulfillment(provisionerCapabilities, requirements, Collections.emptySet());
    provisionerCapabilities = new Capabilities(ImmutableSet.of(Table.TYPE));
    requirements = ImmutableSet.of(new PluginRequirement("source1", "batchsource", new Requirements(ImmutableSet.of(Table.TYPE))), new PluginRequirement("sink1", "batchsink", new Requirements(ImmutableSet.of(Table.TYPE))));
    // there should not be any incapability
    assertRequirementFulfillment(provisionerCapabilities, requirements, Collections.emptySet());
}