Examples with MetricsTable io.cdap.cdap.data2.dataset2.lib.table.MetricsTable used on opensource projects

Example 11 with MetricsTable

use of io.cdap.cdap.data2.dataset2.lib.table.MetricsTable in project cdap by caskdata.

the class DefaultMetricDatasetFactory method getOrCreateResolutionMetricsTable.

protected MetricsTable getOrCreateResolutionMetricsTable(String v3TableName, TableProperties.Builder props, int resolution) {
    try {
        String v2TableName = cConf.get(Constants.Metrics.METRICS_TABLE_PREFIX, Constants.Metrics.DEFAULT_METRIC_TABLE_PREFIX + ".ts." + resolution);
        // metrics tables are in the system namespace
        DatasetId v2TableId = NamespaceId.SYSTEM.dataset(v2TableName);
        MetricsTable v2Table = dsFramework.getDataset(v2TableId, ImmutableMap.<String, String>of(), null);
        props.add(HBaseTableAdmin.PROPERTY_SPLITS, GSON.toJson(getV3MetricsTableSplits(cConf.getInt(Constants.Metrics.METRICS_HBASE_TABLE_SPLITS))));
        props.add(Constants.Metrics.METRICS_HBASE_TABLE_SPLITS, cConf.getInt(Constants.Metrics.METRICS_HBASE_TABLE_SPLITS));
        DatasetId v3TableId = NamespaceId.SYSTEM.dataset(v3TableName);
        MetricsTable v3Table = DatasetsUtil.getOrCreateDataset(dsFramework, v3TableId, MetricsTable.class.getName(), props.build(), null);
        if (v2Table != null) {
            // the cluster is upgraded, so use Combined Metrics Table
            return new CombinedHBaseMetricsTable(v2Table, v3Table, resolution, cConf, dsFramework);
        }
        return v3Table;
    } catch (Exception e) {
        throw Throwables.propagate(e);
    }
}

Example 12 with MetricsTable

use of io.cdap.cdap.data2.dataset2.lib.table.MetricsTable in project cdap by caskdata.

the class DataMigrator method run.

@Override
public void run() {
    // if no v2 tables exist then exit this thread - data migration is complete.
    for (int resolution : resolutions) {
        try {
            DatasetId v2MetricsTableId = getMetricsDatasetId(v2TableNamePrefix, resolution);
            DatasetId v3MetricsTableId = getMetricsDatasetId(v3TableNamePrefix, resolution);
            if (MigrationTableHelper.hasInstanceWithRetry(datasetFramework, v2MetricsTableId)) {
                // this could happen when migrator thread might have started running before processing threads
                if (!MigrationTableHelper.hasInstanceWithRetry(datasetFramework, v3MetricsTableId)) {
                    metricDatasetFactory.getOrCreateFactTable(resolution);
                }
                try (MetricsTable v2MetricsTable = MigrationTableHelper.getDatasetWithRetry(datasetFramework, v2MetricsTableId);
                    MetricsTable v3MetricsTable = MigrationTableHelper.getDatasetWithRetry(datasetFramework, v3MetricsTableId)) {
                    MetricsTableMigration metricsTableMigration = new MetricsTableMigration(v2MetricsTableId.getDataset(), v2MetricsTable, v3MetricsTableId.getDataset(), v3MetricsTable);
                    metricsTableMigration.transferData(sleepMillisBetweenTransfer);
                    LOG.info("Metrics table data migration is complete for {}", v2MetricsTableId.getDataset());
                    // don't delete if we are stopping
                    if (stopping) {
                        break;
                    }
                    // now transfer is complete; its safe to delete the v2 metrics table
                    MigrationTableHelper.deleteInstanceWithRetry(datasetFramework, v2MetricsTableId);
                    LOG.info("Deleted Metrics table {}", v2MetricsTableId.getDataset());
                } catch (InterruptedException e) {
                    Thread.currentThread().interrupt();
                    break;
                }
            }
        } catch (Exception e) {
            LOG.error("Exception while performing dataset operation during metrics table migration", e);
            return;
        }
    }
}

Example 13 with MetricsTable

use of io.cdap.cdap.data2.dataset2.lib.table.MetricsTable in project cdap by caskdata.

the class NoSqlStructuredTableRegistry method registerSpecification.

@Override
public void registerSpecification(StructuredTableSpecification specification) throws TableAlreadyExistsException {
    LOG.debug("Registering table specification {}", specification);
    StructuredTableId tableId = specification.getTableId();
    MetricsTable table = getRegistryTable();
    try {
        byte[] rowKeyBytes = getRowKeyBytes(tableId);
        byte[] serialized = table.get(rowKeyBytes, SCHEMA_COL_BYTES);
        if (serialized != null) {
            throw new TableAlreadyExistsException(tableId);
        }
        serialized = Bytes.toBytes(GSON.toJson(specification));
        if (!table.swap(rowKeyBytes, SCHEMA_COL_BYTES, null, serialized)) {
            throw new TableAlreadyExistsException(tableId);
        }
    } finally {
        closeRegistryTable(table);
    }
}

Example 14 with MetricsTable

use of io.cdap.cdap.data2.dataset2.lib.table.MetricsTable in project cdap by caskdata.

the class SystemDatasetRuntimeModule method getStandaloneModules.

@Override
public Module getStandaloneModules() {
    return new AbstractModule() {

        @Override
        protected void configure() {
            MapBinder<String, DatasetModule> mapBinder = MapBinder.newMapBinder(binder(), String.class, DatasetModule.class, Constants.Dataset.Manager.DefaultDatasetModules.class);
            // NOTE: order is important due to dependencies between modules
            mapBinder.addBinding("orderedTable-leveldb").toInstance(new LevelDBTableModule());
            mapBinder.addBinding("metricsTable-leveldb").toInstance(new LevelDBMetricsTableModule());
            bindDefaultModules(mapBinder);
            bind(String.class).annotatedWith(Names.named(Constants.Dataset.TABLE_TYPE)).toInstance("table");
            bind(DatasetDefinition.class).annotatedWith(Names.named(Constants.Dataset.TABLE_TYPE)).to(LevelDBTableDefinition.class);
            bind(String.class).annotatedWith(Names.named(Constants.Dataset.TABLE_TYPE_NO_TX)).toInstance("table-no-tx");
            bind(DatasetDefinition.class).annotatedWith(Names.named(Constants.Dataset.TABLE_TYPE_NO_TX)).to(LevelDBMetricsTableDefinition.class);
            // Direct binding for the Metrics table definition such that metrics system doesn't need to go through
            // dataset service to get metrics table.
            bind(new TypeLiteral<DatasetDefinition<MetricsTable, DatasetAdmin>>() {
            }).toInstance(new LevelDBMetricsTableDefinition(MetricsTable.class.getName()));
        }
    };
}

Example 15 with MetricsTable

use of io.cdap.cdap.data2.dataset2.lib.table.MetricsTable in project cdap by caskdata.

the class CubeDatasetDefinition method getDataset.

@Override
public CubeDataset getDataset(DatasetContext datasetContext, DatasetSpecification spec, Map<String, String> arguments, ClassLoader classLoader) throws IOException {
    MetricsTable entityTable = metricsTableDef.getDataset(datasetContext, spec.getSpecification("entity"), arguments, classLoader);
    int[] resolutions = getResolutions(spec.getProperties());
    Map<Integer, Table> resolutionTables = Maps.newHashMap();
    for (int resolution : resolutions) {
        resolutionTables.put(resolution, tableDef.getDataset(datasetContext, spec.getSpecification(String.valueOf(resolution)), arguments, classLoader));
    }
    Map<String, Aggregation> aggregations = getAggregations(spec.getProperties());
    return new CubeDataset(spec.getName(), entityTable, resolutionTables, aggregations);
}