Examples with DatasetCreationSpec io.cdap.cdap.internal.dataset.DatasetCreationSpec used on opensource projects

Example 1 with DatasetCreationSpec

use of io.cdap.cdap.internal.dataset.DatasetCreationSpec in project cdap by caskdata.

the class WorkflowVerificationTest method verifyWorkflowWithLocalDatasetSpecification.

private void verifyWorkflowWithLocalDatasetSpecification(ApplicationSpecification appSpec) {
    WorkflowSpecification spec = appSpec.getWorkflows().get("WorkflowWithLocalDatasets");
    List<WorkflowNode> nodes = spec.getNodes();
    Assert.assertTrue(nodes.size() == 2);
    WorkflowNode node = nodes.get(0);
    Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
    WorkflowActionNode actionNode = (WorkflowActionNode) node;
    Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.MAPREDUCE, "MR1")));
    node = nodes.get(1);
    Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
    actionNode = (WorkflowActionNode) node;
    Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.SPARK, "SP1")));
    Map<String, DatasetCreationSpec> localDatasetSpecs = spec.getLocalDatasetSpecs();
    Assert.assertEquals(5, localDatasetSpecs.size());
    DatasetCreationSpec datasetCreationSpec = localDatasetSpecs.get("mytable");
    Assert.assertEquals(Table.class.getName(), datasetCreationSpec.getTypeName());
    Assert.assertEquals(0, datasetCreationSpec.getProperties().getProperties().size());
    datasetCreationSpec = localDatasetSpecs.get("myfile");
    Assert.assertEquals(FileSet.class.getName(), datasetCreationSpec.getTypeName());
    Assert.assertEquals(0, datasetCreationSpec.getProperties().getProperties().size());
    datasetCreationSpec = localDatasetSpecs.get("myfile_with_properties");
    Assert.assertEquals(FileSet.class.getName(), datasetCreationSpec.getTypeName());
    Assert.assertEquals("prop_value", datasetCreationSpec.getProperties().getProperties().get("prop_key"));
    datasetCreationSpec = localDatasetSpecs.get("mytablefromtype");
    Assert.assertEquals(Table.class.getName(), datasetCreationSpec.getTypeName());
    Assert.assertEquals(0, datasetCreationSpec.getProperties().getProperties().size());
    datasetCreationSpec = localDatasetSpecs.get("myfilefromtype");
    Assert.assertEquals(FileSet.class.getName(), datasetCreationSpec.getTypeName());
    Assert.assertEquals("another_prop_value", datasetCreationSpec.getProperties().getProperties().get("another_prop_key"));
    // Check if the application specification has correct modules
    Map<String, String> datasetModules = appSpec.getDatasetModules();
    Assert.assertEquals(2, datasetModules.size());
    Assert.assertTrue(datasetModules.containsKey(FileSet.class.getName()));
    Assert.assertTrue(datasetModules.containsKey(Table.class.getName()));
}

Example 2 with DatasetCreationSpec

use of io.cdap.cdap.internal.dataset.DatasetCreationSpec in project cdap by caskdata.

the class ApplicationVerificationStage method verifyData.

private void verifyData(ApplicationId appId, ApplicationSpecification specification, @Nullable KerberosPrincipalId ownerPrincipal) throws Exception {
    // NOTE: no special restrictions on dataset module names, etc
    VerifyResult result;
    for (DatasetCreationSpec dataSetCreateSpec : specification.getDatasets().values()) {
        result = getVerifier(DatasetCreationSpec.class).verify(appId, dataSetCreateSpec);
        if (!result.isSuccess()) {
            throw new RuntimeException(result.getMessage());
        }
        String dsName = dataSetCreateSpec.getInstanceName();
        final DatasetId datasetInstanceId = appId.getParent().dataset(dsName);
        // get the authorizing user
        String authorizingUser = AuthorizationUtil.getAppAuthorizingUser(ownerAdmin, authenticationContext, appId, ownerPrincipal);
        DatasetSpecification existingSpec = AuthorizationUtil.authorizeAs(authorizingUser, new Callable<DatasetSpecification>() {

            @Override
            public DatasetSpecification call() throws Exception {
                return dsFramework.getDatasetSpec(datasetInstanceId);
            }
        });
        if (existingSpec != null && !existingSpec.getType().equals(dataSetCreateSpec.getTypeName())) {
            // New app trying to deploy an dataset with same instanceName but different Type than that of existing.
            throw new DataSetException(String.format("Cannot Deploy Dataset : %s with Type : %s : Dataset with different Type Already Exists", dsName, dataSetCreateSpec.getTypeName()));
        }
        // if the dataset existed verify its owner is same.
        if (existingSpec != null) {
            verifyOwner(datasetInstanceId, ownerPrincipal);
        }
    }
}

Example 3 with DatasetCreationSpec

use of io.cdap.cdap.internal.dataset.DatasetCreationSpec in project cdap by caskdata.

the class WorkflowDriver method createLocalDatasets.

private void createLocalDatasets() throws IOException, DatasetManagementException {
    final KerberosPrincipalId principalId = ProgramRunners.getApplicationPrincipal(programOptions);
    for (final Map.Entry<String, String> entry : datasetFramework.getDatasetNameMapping().entrySet()) {
        final String localInstanceName = entry.getValue();
        final DatasetId instanceId = new DatasetId(workflowRunId.getNamespace(), localInstanceName);
        final DatasetCreationSpec instanceSpec = workflowSpec.getLocalDatasetSpecs().get(entry.getKey());
        LOG.debug("Adding Workflow local dataset instance: {}", localInstanceName);
        try {
            Retries.callWithRetries(new Retries.Callable<Void, Exception>() {

                @Override
                public Void call() throws Exception {
                    DatasetProperties properties = addLocalDatasetProperty(instanceSpec.getProperties(), keepLocal(entry.getKey()));
                    // we have to do this check since addInstance method can only be used when app impersonation is enabled
                    if (principalId != null) {
                        datasetFramework.addInstance(instanceSpec.getTypeName(), instanceId, properties, principalId);
                    } else {
                        datasetFramework.addInstance(instanceSpec.getTypeName(), instanceId, properties);
                    }
                    return null;
                }
            }, RetryStrategies.fixDelay(Constants.Retry.LOCAL_DATASET_OPERATION_RETRY_DELAY_SECONDS, TimeUnit.SECONDS));
        } catch (IOException | DatasetManagementException e) {
            throw e;
        } catch (Exception e) {
            // this should never happen
            throw new IllegalStateException(e);
        }
    }
}

Example 4 with DatasetCreationSpec

use of io.cdap.cdap.internal.dataset.DatasetCreationSpec in project cdap by caskdata.

the class DefaultDatasetConfigurer method createDataset.

@Override
public void createDataset(String datasetInstanceName, String typeName, DatasetProperties properties) {
    checkArgument(datasetInstanceName != null, "Dataset instance name cannot be null.");
    checkArgument(typeName != null, "Dataset type name cannot be null.");
    checkArgument(properties != null, "Instance properties name cannot be null.");
    DatasetCreationSpec spec = new DatasetCreationSpec(datasetInstanceName, typeName, properties);
    DatasetCreationSpec existingSpec = datasetSpecs.get(datasetInstanceName);
    if (existingSpec != null && !existingSpec.equals(spec)) {
        throw new IllegalArgumentException(String.format("DatasetInstance '%s' was added multiple times with " + "different specifications. Please resolve the conflict so that there is only one specification for " + "the dataset instance.", datasetInstanceName));
    }
    datasetSpecs.put(datasetInstanceName, spec);
}

Example 5 with DatasetCreationSpec

use of io.cdap.cdap.internal.dataset.DatasetCreationSpec in project cdap by caskdata.

the class WorkflowHttpHandler method deleteWorkflowLocalDatasets.

@DELETE
@Path("/apps/{app-id}/workflows/{workflow-id}/runs/{run-id}/localdatasets")
public void deleteWorkflowLocalDatasets(HttpRequest request, HttpResponder responder, @PathParam("namespace-id") String namespaceId, @PathParam("app-id") String applicationId, @PathParam("workflow-id") String workflowId, @PathParam("run-id") String runId) throws NotFoundException {
    WorkflowSpecification workflowSpec = getWorkflowSpecForValidRun(namespaceId, applicationId, workflowId, runId);
    Set<String> errorOnDelete = new HashSet<>();
    for (Map.Entry<String, DatasetCreationSpec> localDatasetEntry : workflowSpec.getLocalDatasetSpecs().entrySet()) {
        String mappedDatasetName = localDatasetEntry.getKey() + "." + runId;
        // try best to delete the local datasets.
        try {
            datasetFramework.deleteInstance(new DatasetId(namespaceId, mappedDatasetName));
        } catch (InstanceNotFoundException e) {
        // Dataset instance is already deleted. so its no-op.
        } catch (Throwable t) {
            errorOnDelete.add(mappedDatasetName);
            LOG.error("Failed to delete the Workflow local dataset {}. Reason - {}", mappedDatasetName, t.getMessage());
        }
    }
    if (errorOnDelete.isEmpty()) {
        responder.sendStatus(HttpResponseStatus.OK);
        return;
    }
    String errorMessage = "Failed to delete Workflow local datasets - " + Joiner.on(",").join(errorOnDelete);
    throw new RuntimeException(errorMessage);
}