Examples with DatasetSpecificationSummary co.cask.cdap.proto.DatasetSpecificationSummary used on opensource projects

Example 1 with DatasetSpecificationSummary

use of co.cask.cdap.proto.DatasetSpecificationSummary in project cdap by caskdata.

the class WorkflowHttpHandler method getWorkflowLocalDatasets.

@GET
@Path("/apps/{app-id}/workflows/{workflow-id}/runs/{run-id}/localdatasets")
public void getWorkflowLocalDatasets(HttpRequest request, HttpResponder responder, @PathParam("namespace-id") String namespaceId, @PathParam("app-id") String applicationId, @PathParam("workflow-id") String workflowId, @PathParam("run-id") String runId) throws NotFoundException, DatasetManagementException {
    WorkflowSpecification workflowSpec = getWorkflowSpecForValidRun(namespaceId, applicationId, workflowId, runId);
    Map<String, DatasetSpecificationSummary> localDatasetSummaries = new HashMap<>();
    for (Map.Entry<String, DatasetCreationSpec> localDatasetEntry : workflowSpec.getLocalDatasetSpecs().entrySet()) {
        String mappedDatasetName = localDatasetEntry.getKey() + "." + runId;
        String datasetType = localDatasetEntry.getValue().getTypeName();
        Map<String, String> datasetProperties = localDatasetEntry.getValue().getProperties().getProperties();
        if (datasetFramework.hasInstance(new DatasetId(namespaceId, mappedDatasetName))) {
            localDatasetSummaries.put(localDatasetEntry.getKey(), new DatasetSpecificationSummary(mappedDatasetName, datasetType, datasetProperties));
        }
    }
    responder.sendJson(HttpResponseStatus.OK, localDatasetSummaries);
}

Example 2 with DatasetSpecificationSummary

use of co.cask.cdap.proto.DatasetSpecificationSummary in project cdap by caskdata.

the class ConversionHelpers method spec2Summary.

static Collection<DatasetSpecificationSummary> spec2Summary(Collection<DatasetSpecification> specs) {
    List<DatasetSpecificationSummary> datasetSummaries = Lists.newArrayList();
    for (DatasetSpecification spec : specs) {
        // HBaseQueueAdmin through DatasetFramework.
        if (QueueConstants.STATE_STORE_NAME.equals(spec.getName())) {
            continue;
        }
        spec = DatasetsUtil.fixOriginalProperties(spec);
        datasetSummaries.add(new DatasetSpecificationSummary(spec.getName(), spec.getType(), spec.getDescription(), spec.getOriginalProperties()));
    }
    return datasetSummaries;
}

Example 3 with DatasetSpecificationSummary

use of co.cask.cdap.proto.DatasetSpecificationSummary in project cdap by caskdata.

the class LevelDBDatasetMetricsReporter method report.

private void report(Map<TableId, LevelDBTableService.TableStats> datasetStat) throws DatasetManagementException {
    for (Map.Entry<TableId, LevelDBTableService.TableStats> statEntry : datasetStat.entrySet()) {
        String namespace = statEntry.getKey().getNamespace();
        // emit metrics for only user datasets, tables in system namespace are ignored
        if (NamespaceId.SYSTEM.getNamespace().equals(namespace)) {
            continue;
        }
        String tableName = statEntry.getKey().getTableName();
        Collection<DatasetSpecificationSummary> instances = dsFramework.getInstances(new NamespaceId(namespace));
        for (DatasetSpecificationSummary spec : instances) {
            DatasetSpecification specification = dsFramework.getDatasetSpec(new DatasetId(namespace, spec.getName()));
            if (specification.isParent(tableName)) {
                MetricsContext collector = metricsService.getContext(ImmutableMap.of(Constants.Metrics.Tag.NAMESPACE, namespace, Constants.Metrics.Tag.DATASET, spec.getName()));
                int sizeInMb = (int) (statEntry.getValue().getDiskSizeBytes() / BYTES_IN_MB);
                collector.gauge("dataset.size.mb", sizeInMb);
                break;
            }
        }
    }
}

Example 4 with DatasetSpecificationSummary

use of co.cask.cdap.proto.DatasetSpecificationSummary in project cdap by caskdata.

the class ExistingEntitySystemMetadataWriter method writeSystemMetadataForDatasets.

private void writeSystemMetadataForDatasets(NamespaceId namespace, DatasetFramework dsFramework) throws DatasetManagementException, IOException, NamespaceNotFoundException {
    SystemDatasetInstantiatorFactory systemDatasetInstantiatorFactory = new SystemDatasetInstantiatorFactory(locationFactory, dsFramework, cConf);
    try (SystemDatasetInstantiator systemDatasetInstantiator = systemDatasetInstantiatorFactory.create()) {
        for (DatasetSpecificationSummary summary : dsFramework.getInstances(namespace)) {
            final DatasetId dsInstance = namespace.dataset(summary.getName());
            DatasetProperties dsProperties = DatasetProperties.of(summary.getProperties());
            String dsType = summary.getType();
            Dataset dataset = null;
            try {
                try {
                    dataset = impersonator.doAs(dsInstance, new Callable<Dataset>() {

                        @Override
                        public Dataset call() throws Exception {
                            return systemDatasetInstantiator.getDataset(dsInstance);
                        }
                    });
                } catch (Exception e) {
                    LOG.warn("Exception while instantiating dataset {}", dsInstance, e);
                }
                SystemMetadataWriter writer = new DatasetSystemMetadataWriter(metadataStore, dsInstance, dsProperties, dataset, dsType, summary.getDescription());
                writer.write();
            } finally {
                if (dataset != null) {
                    dataset.close();
                }
            }
        }
    }
}

Example 5 with DatasetSpecificationSummary

use of co.cask.cdap.proto.DatasetSpecificationSummary in project cdap by caskdata.

the class ListDatasetInstancesCommand method perform.

@Override
public void perform(Arguments arguments, PrintStream output) throws Exception {
    List<DatasetSpecificationSummary> datasetMetas = datasetClient.list(cliConfig.getCurrentNamespace());
    Table table = Table.builder().setHeader("name", "type", "description").setRows(datasetMetas, new RowMaker<DatasetSpecificationSummary>() {

        @Override
        public List<?> makeRow(DatasetSpecificationSummary object) {
            return Lists.newArrayList(object.getName(), object.getType(), object.getDescription());
        }
    }).build();
    cliConfig.getTableRenderer().render(cliConfig, output, table);
}