Examples with DatasetSpecificationSummary io.cdap.cdap.proto.DatasetSpecificationSummary used on opensource projects

Example 6 with DatasetSpecificationSummary

use of io.cdap.cdap.proto.DatasetSpecificationSummary in project cdap by caskdata.

the class HBaseDatasetMetricsReporter method report.

private void report(Map<TableId, HBaseTableUtil.TableStats> tableStats) throws IOException, UnauthorizedException {
    Map<String, String> reverseNamespaceMap = hBaseTableUtil.getHBaseToCDAPNamespaceMap();
    for (Map.Entry<TableId, HBaseTableUtil.TableStats> statEntry : tableStats.entrySet()) {
        String hbaseNamespace = statEntry.getKey().getNamespace();
        String cdapNamespace = reverseNamespaceMap.get(hbaseNamespace);
        // tableNames that doesn't start with user are ignored
        if (NamespaceId.SYSTEM.getNamespace().equals(cdapNamespace)) {
            continue;
        }
        String tableName = statEntry.getKey().getTableName();
        try {
            Collection<DatasetSpecificationSummary> instances = dsFramework.getInstances(new NamespaceId(cdapNamespace));
            for (DatasetSpecificationSummary spec : instances) {
                DatasetSpecification specification = dsFramework.getDatasetSpec(new DatasetId(cdapNamespace, spec.getName()));
                if (specification.isParent(tableName)) {
                    MetricsContext collector = metricsService.getContext(ImmutableMap.of(Constants.Metrics.Tag.NAMESPACE, cdapNamespace, Constants.Metrics.Tag.DATASET, spec.getName()));
                    collector.gauge("dataset.size.mb", statEntry.getValue().getTotalSizeMB());
                    break;
                }
            }
        } catch (DatasetManagementException | ServiceUnavailableException e) {
        // No op
        }
    }
}

Example 7 with DatasetSpecificationSummary

use of io.cdap.cdap.proto.DatasetSpecificationSummary in project cdap by caskdata.

the class LevelDBDatasetMetricsReporter method report.

private void report(Map<TableId, LevelDBTableService.TableStats> datasetStat) throws DatasetManagementException, UnauthorizedException {
    for (Map.Entry<TableId, LevelDBTableService.TableStats> statEntry : datasetStat.entrySet()) {
        String namespace = statEntry.getKey().getNamespace();
        // emit metrics for only user datasets, tables in system namespace are ignored
        if (NamespaceId.SYSTEM.getNamespace().equals(namespace)) {
            continue;
        }
        String tableName = statEntry.getKey().getTableName();
        Collection<DatasetSpecificationSummary> instances = dsFramework.getInstances(new NamespaceId(namespace));
        for (DatasetSpecificationSummary spec : instances) {
            DatasetSpecification specification = dsFramework.getDatasetSpec(new DatasetId(namespace, spec.getName()));
            if (specification.isParent(tableName)) {
                MetricsContext collector = metricsService.getContext(ImmutableMap.of(Constants.Metrics.Tag.NAMESPACE, namespace, Constants.Metrics.Tag.DATASET, spec.getName()));
                int sizeInMb = (int) (statEntry.getValue().getDiskSizeBytes() / BYTES_IN_MB);
                collector.gauge("dataset.size.mb", sizeInMb);
                break;
            }
        }
    }
}

Example 8 with DatasetSpecificationSummary

use of io.cdap.cdap.proto.DatasetSpecificationSummary in project cdap by cdapio.

the class InMemoryDatasetFramework method getInstances.

@Override
public Collection<DatasetSpecificationSummary> getInstances(NamespaceId namespaceId, Map<String, String> properties) {
    readLock.lock();
    try {
        // don't expect this to be called a lot.
        // might be better to maintain this collection separately and just return it, but seems like its not worth it.
        Collection<DatasetSpecification> specs = instances.row(namespaceId).values();
        ImmutableList.Builder<DatasetSpecificationSummary> specSummaries = ImmutableList.builder();
        for (DatasetSpecification spec : specs) {
            if (properties.isEmpty() || Maps.difference(properties, spec.getProperties()).entriesOnlyOnLeft().isEmpty()) {
                specSummaries.add(new DatasetSpecificationSummary(spec.getName(), spec.getType(), spec.getProperties()));
            }
        }
        return specSummaries.build();
    } finally {
        readLock.unlock();
    }
}

Example 9 with DatasetSpecificationSummary

use of io.cdap.cdap.proto.DatasetSpecificationSummary in project cdap by cdapio.

the class DatasetUpgrader method upgradeSystemDatasets.

private void upgradeSystemDatasets(ExecutorService executor) throws Exception {
    Map<String, Future<?>> futures = new HashMap<>();
    for (final DatasetSpecificationSummary spec : dsFramework.getInstances(NamespaceId.SYSTEM)) {
        final DatasetId datasetId = NamespaceId.SYSTEM.dataset(spec.getName());
        Runnable runnable = new Runnable() {

            @Override
            public void run() {
                try {
                    LOG.info("Upgrading dataset in system namespace: {}, spec: {}", spec.getName(), spec.toString());
                    DatasetAdmin admin = dsFramework.getAdmin(datasetId, null);
                    // we know admin is not null, since we are looping over existing datasets
                    // noinspection ConstantConditions
                    admin.upgrade();
                    LOG.info("Upgraded dataset: {}", spec.getName());
                } catch (Exception e) {
                    throw new RuntimeException(e);
                }
            }
        };
        Future<?> future = executor.submit(runnable);
        futures.put(datasetId.toString(), future);
    }
    // Wait for the system dataset upgrades to complete
    Map<String, Throwable> failed = waitForUpgrade(futures);
    if (!failed.isEmpty()) {
        for (Map.Entry<String, Throwable> entry : failed.entrySet()) {
            LOG.error("Failed to upgrade system dataset {}", entry.getKey(), entry.getValue());
        }
        throw new Exception(String.format("Error upgrading system datasets. %s of %s failed", failed.size(), futures.size()));
    }
}

Example 10 with DatasetSpecificationSummary

use of io.cdap.cdap.proto.DatasetSpecificationSummary in project cdap by cdapio.

the class RunRecordCorrectorServiceTest method testLocalDatasetDeleteion.

@Test
public void testLocalDatasetDeleteion() throws Exception {
    // Deploy an app
    deploy(WorkflowAppWithLocalDataset.class, 200, Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE1);
    final ProgramId workflow = new NamespaceId(TEST_NAMESPACE1).app(WorkflowAppWithLocalDataset.APP_NAME).workflow(WorkflowAppWithLocalDataset.WORKFLOW_NAME);
    startProgram(Id.Program.fromEntityId(workflow), ImmutableMap.of("dataset.*.keep.local", "true"));
    // Wait until we have a COMPLETED run record
    Tasks.waitFor(1, () -> getProgramRuns(Id.Program.fromEntityId(workflow), ProgramRunStatus.COMPLETED).size(), 5, TimeUnit.SECONDS);
    // Get the RunRecord
    List<RunRecord> runRecords = getProgramRuns(Id.Program.fromEntityId(workflow), ProgramRunStatus.COMPLETED);
    Assert.assertEquals(1, runRecords.size());
    String pid = runRecords.get(0).getPid();
    // Get the local dataset specifications
    final Map<String, String> properties = ImmutableMap.of(Constants.AppFabric.WORKFLOW_LOCAL_DATASET_PROPERTY, "true");
    Collection<DatasetSpecificationSummary> instances = datasetFramework.getInstances(new NamespaceId(TEST_NAMESPACE1), properties);
    Assert.assertEquals(1, instances.size());
    DatasetSpecificationSummary summary = instances.iterator().next();
    Assert.assertTrue(summary.getName().endsWith(pid));
    // Update the dataset properties to remove keep.local so that local dataset deleter can delete it
    Map<String, String> updatedProperties = new HashMap<>(summary.getProperties());
    updatedProperties.remove(Constants.AppFabric.WORKFLOW_KEEP_LOCAL);
    datasetFramework.updateInstance(new DatasetId(TEST_NAMESPACE1, summary.getName()), DatasetProperties.of(updatedProperties));
    // Start the local dataset deletion service now
    CConfiguration testConf = CConfiguration.create();
    // set threshold to 0 so that it will actually correct the record
    testConf.set(Constants.AppFabric.LOCAL_DATASET_DELETER_INTERVAL_SECONDS, "1");
    testConf.set(Constants.AppFabric.LOCAL_DATASET_DELETER_INITIAL_DELAY_SECONDS, "1");
    new LocalRunRecordCorrectorService(testConf, store, programStateWriter, runtimeService, namespaceAdmin, datasetFramework).startUp();
    // 
    // Wait for the deletion of the local dataset
    Tasks.waitFor(0, () -> datasetFramework.getInstances(new NamespaceId(TEST_NAMESPACE1), properties).size(), 30, TimeUnit.SECONDS, 1, TimeUnit.SECONDS);
}