Examples with DatasetCreationSpec io.cdap.cdap.internal.dataset.DatasetCreationSpec used on opensource projects

Example 6 with DatasetCreationSpec

use of io.cdap.cdap.internal.dataset.DatasetCreationSpec in project cdap by caskdata.

the class WorkflowSpecificationCodec method deserialize.

@Override
public WorkflowSpecification deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) throws JsonParseException {
    JsonObject jsonObj = json.getAsJsonObject();
    String className = jsonObj.get("className").getAsString();
    String name = jsonObj.get("name").getAsString();
    String description = jsonObj.get("description").getAsString();
    Map<String, Plugin> plugins = deserializeMap(jsonObj.get("plugins"), context, Plugin.class);
    Map<String, String> properties = deserializeMap(jsonObj.get("properties"), context, String.class);
    List<WorkflowNode> nodes = deserializeList(jsonObj.get("nodes"), context, WorkflowNode.class);
    Map<String, DatasetCreationSpec> localDatasetSpec = deserializeMap(jsonObj.get("localDatasetSpecs"), context, DatasetCreationSpec.class);
    return new WorkflowSpecification(className, name, description, properties, nodes, localDatasetSpec, plugins);
}

Example 7 with DatasetCreationSpec

use of io.cdap.cdap.internal.dataset.DatasetCreationSpec in project cdap by caskdata.

the class ApplicationDetail method fromSpec.

public static ApplicationDetail fromSpec(ApplicationSpecification spec, @Nullable String ownerPrincipal) {
    List<ProgramRecord> programs = new ArrayList<>();
    for (ProgramSpecification programSpec : spec.getMapReduce().values()) {
        programs.add(new ProgramRecord(ProgramType.MAPREDUCE, spec.getName(), programSpec.getName(), programSpec.getDescription()));
    }
    for (ProgramSpecification programSpec : spec.getServices().values()) {
        programs.add(new ProgramRecord(ProgramType.SERVICE, spec.getName(), programSpec.getName(), programSpec.getDescription()));
    }
    for (ProgramSpecification programSpec : spec.getSpark().values()) {
        programs.add(new ProgramRecord(ProgramType.SPARK, spec.getName(), programSpec.getName(), programSpec.getDescription()));
    }
    for (ProgramSpecification programSpec : spec.getWorkers().values()) {
        programs.add(new ProgramRecord(ProgramType.WORKER, spec.getName(), programSpec.getName(), programSpec.getDescription()));
    }
    for (ProgramSpecification programSpec : spec.getWorkflows().values()) {
        programs.add(new ProgramRecord(ProgramType.WORKFLOW, spec.getName(), programSpec.getName(), programSpec.getDescription()));
    }
    List<DatasetDetail> datasets = new ArrayList<>();
    for (DatasetCreationSpec datasetSpec : spec.getDatasets().values()) {
        datasets.add(new DatasetDetail(datasetSpec.getInstanceName(), datasetSpec.getTypeName()));
    }
    List<PluginDetail> plugins = new ArrayList<>();
    for (Map.Entry<String, Plugin> pluginEnty : spec.getPlugins().entrySet()) {
        plugins.add(new PluginDetail(pluginEnty.getKey(), pluginEnty.getValue().getPluginClass().getName(), pluginEnty.getValue().getPluginClass().getType()));
    }
    // this is only required if there are old apps lying around that failed to get upgrading during
    // the upgrade to v3.2 for some reason. In those cases artifact id will be null until they re-deploy the app.
    // in the meantime, we don't want this api call to null pointer exception.
    ArtifactSummary summary = spec.getArtifactId() == null ? new ArtifactSummary(spec.getName(), null) : ArtifactSummary.from(spec.getArtifactId());
    return new ApplicationDetail(spec.getName(), spec.getAppVersion(), spec.getDescription(), spec.getConfiguration(), datasets, programs, plugins, summary, ownerPrincipal);
}

Example 8 with DatasetCreationSpec

use of io.cdap.cdap.internal.dataset.DatasetCreationSpec in project cdap by cdapio.

the class ApplicationVerificationStage method verifyData.

private void verifyData(ApplicationId appId, ApplicationSpecification specification, @Nullable KerberosPrincipalId ownerPrincipal) throws Exception {
    // NOTE: no special restrictions on dataset module names, etc
    VerifyResult result;
    for (DatasetCreationSpec dataSetCreateSpec : specification.getDatasets().values()) {
        result = getVerifier(DatasetCreationSpec.class).verify(appId, dataSetCreateSpec);
        if (!result.isSuccess()) {
            throw new RuntimeException(result.getMessage());
        }
        String dsName = dataSetCreateSpec.getInstanceName();
        final DatasetId datasetInstanceId = appId.getParent().dataset(dsName);
        // get the authorizing user
        String authorizingUser = AuthorizationUtil.getAppAuthorizingUser(ownerAdmin, authenticationContext, appId, ownerPrincipal);
        DatasetSpecification existingSpec = AuthorizationUtil.authorizeAs(authorizingUser, new Callable<DatasetSpecification>() {

            @Override
            public DatasetSpecification call() throws Exception {
                return dsFramework.getDatasetSpec(datasetInstanceId);
            }
        });
        if (existingSpec != null && !existingSpec.getType().equals(dataSetCreateSpec.getTypeName())) {
            // New app trying to deploy an dataset with same instanceName but different Type than that of existing.
            throw new DataSetException(String.format("Cannot Deploy Dataset : %s with Type : %s : Dataset with different Type Already Exists", dsName, dataSetCreateSpec.getTypeName()));
        }
        // if the dataset existed verify its owner is same.
        if (existingSpec != null) {
            verifyOwner(datasetInstanceId, ownerPrincipal);
        }
    }
}

Example 9 with DatasetCreationSpec

use of io.cdap.cdap.internal.dataset.DatasetCreationSpec in project cdap by cdapio.

the class ApplicationDetail method fromSpec.

public static ApplicationDetail fromSpec(ApplicationSpecification spec, @Nullable String ownerPrincipal) {
    List<ProgramRecord> programs = new ArrayList<>();
    for (ProgramSpecification programSpec : spec.getMapReduce().values()) {
        programs.add(new ProgramRecord(ProgramType.MAPREDUCE, spec.getName(), programSpec.getName(), programSpec.getDescription()));
    }
    for (ProgramSpecification programSpec : spec.getServices().values()) {
        programs.add(new ProgramRecord(ProgramType.SERVICE, spec.getName(), programSpec.getName(), programSpec.getDescription()));
    }
    for (ProgramSpecification programSpec : spec.getSpark().values()) {
        programs.add(new ProgramRecord(ProgramType.SPARK, spec.getName(), programSpec.getName(), programSpec.getDescription()));
    }
    for (ProgramSpecification programSpec : spec.getWorkers().values()) {
        programs.add(new ProgramRecord(ProgramType.WORKER, spec.getName(), programSpec.getName(), programSpec.getDescription()));
    }
    for (ProgramSpecification programSpec : spec.getWorkflows().values()) {
        programs.add(new ProgramRecord(ProgramType.WORKFLOW, spec.getName(), programSpec.getName(), programSpec.getDescription()));
    }
    List<DatasetDetail> datasets = new ArrayList<>();
    for (DatasetCreationSpec datasetSpec : spec.getDatasets().values()) {
        datasets.add(new DatasetDetail(datasetSpec.getInstanceName(), datasetSpec.getTypeName()));
    }
    List<PluginDetail> plugins = new ArrayList<>();
    for (Map.Entry<String, Plugin> pluginEnty : spec.getPlugins().entrySet()) {
        plugins.add(new PluginDetail(pluginEnty.getKey(), pluginEnty.getValue().getPluginClass().getName(), pluginEnty.getValue().getPluginClass().getType()));
    }
    // this is only required if there are old apps lying around that failed to get upgrading during
    // the upgrade to v3.2 for some reason. In those cases artifact id will be null until they re-deploy the app.
    // in the meantime, we don't want this api call to null pointer exception.
    ArtifactSummary summary = spec.getArtifactId() == null ? new ArtifactSummary(spec.getName(), null) : ArtifactSummary.from(spec.getArtifactId());
    return new ApplicationDetail(spec.getName(), spec.getAppVersion(), spec.getDescription(), spec.getConfiguration(), datasets, programs, plugins, summary, ownerPrincipal);
}

Example 10 with DatasetCreationSpec

use of io.cdap.cdap.internal.dataset.DatasetCreationSpec in project cdap by cdapio.

the class WorkflowSpecificationCodec method deserialize.

@Override
public WorkflowSpecification deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) throws JsonParseException {
    JsonObject jsonObj = json.getAsJsonObject();
    String className = jsonObj.get("className").getAsString();
    String name = jsonObj.get("name").getAsString();
    String description = jsonObj.get("description").getAsString();
    Map<String, Plugin> plugins = deserializeMap(jsonObj.get("plugins"), context, Plugin.class);
    Map<String, String> properties = deserializeMap(jsonObj.get("properties"), context, String.class);
    List<WorkflowNode> nodes = deserializeList(jsonObj.get("nodes"), context, WorkflowNode.class);
    Map<String, DatasetCreationSpec> localDatasetSpec = deserializeMap(jsonObj.get("localDatasetSpecs"), context, DatasetCreationSpec.class);
    return new WorkflowSpecification(className, name, description, properties, nodes, localDatasetSpec, plugins);
}