Examples with DatasetId co.cask.cdap.proto.id.DatasetId used on opensource projects

Example 56 with DatasetId

use of co.cask.cdap.proto.id.DatasetId in project cdap by caskdata.

the class HiveExploreServiceTestRun method testJoin.

@Test
public void testJoin() throws Exception {
    DatasetId myTable1 = NAMESPACE_ID.dataset("my_table_1");
    String myTable1Name = getDatasetHiveName(myTable1);
    // Performing admin operations to create dataset instance
    datasetFramework.addInstance("keyStructValueTable", myTable1, DatasetProperties.EMPTY);
    try {
        Transaction tx1 = transactionManager.startShort(100);
        // Accessing dataset instance to perform data operations
        KeyStructValueTableDefinition.KeyStructValueTable table = datasetFramework.getDataset(myTable1, DatasetDefinition.NO_ARGUMENTS, null);
        Assert.assertNotNull(table);
        table.startTx(tx1);
        KeyValue.Value value1 = new KeyValue.Value("two", Lists.newArrayList(20, 21, 22, 23, 24));
        KeyValue.Value value2 = new KeyValue.Value("third", Lists.newArrayList(30, 31, 32, 33, 34));
        table.put("2", value1);
        table.put("3", value2);
        Assert.assertEquals(value1, table.get("2"));
        Assert.assertTrue(table.commitTx());
        transactionManager.canCommit(tx1, table.getTxChanges());
        transactionManager.commit(tx1);
        table.postTxCommit();
        String query = String.format("select %s.key, %s.value from %s join %s on (%s.key=%s.key)", MY_TABLE_NAME, MY_TABLE_NAME, MY_TABLE_NAME, myTable1Name, MY_TABLE_NAME, myTable1Name);
        runCommand(NAMESPACE_ID, query, true, Lists.newArrayList(new ColumnDesc(MY_TABLE_NAME + ".key", "STRING", 1, null), new ColumnDesc(MY_TABLE_NAME + ".value", "struct<name:string,ints:array<int>>", 2, null)), Lists.newArrayList(new QueryResult(Lists.<Object>newArrayList("2", "{\"name\":\"two\",\"ints\":[10,11,12,13,14]}"))));
        query = String.format("select %s.key, %s.value, %s.key, %s.value " + "from %s right outer join %s on (%s.key=%s.key)", MY_TABLE_NAME, MY_TABLE_NAME, myTable1Name, myTable1Name, MY_TABLE_NAME, myTable1Name, MY_TABLE_NAME, myTable1Name);
        runCommand(NAMESPACE_ID, query, true, Lists.newArrayList(new ColumnDesc(MY_TABLE_NAME + ".key", "STRING", 1, null), new ColumnDesc(MY_TABLE_NAME + ".value", "struct<name:string,ints:array<int>>", 2, null), new ColumnDesc(myTable1Name + ".key", "STRING", 3, null), new ColumnDesc(myTable1Name + ".value", "struct<name:string,ints:array<int>>", 4, null)), Lists.newArrayList(new QueryResult(Lists.<Object>newArrayList("2", "{\"name\":\"two\",\"ints\":[10,11,12,13,14]}", "2", "{\"name\":\"two\",\"ints\":[20,21,22,23,24]}")), new QueryResult(Lists.<Object>newArrayList(null, null, "3", "{\"name\":\"third\",\"ints\":[30,31,32,33,34]}"))));
        query = String.format("select %s.key, %s.value, %s.key, %s.value from %s " + "left outer join %s on (%s.key=%s.key)", MY_TABLE_NAME, MY_TABLE_NAME, myTable1Name, myTable1Name, MY_TABLE_NAME, myTable1Name, MY_TABLE_NAME, myTable1Name);
        runCommand(NAMESPACE_ID, query, true, Lists.newArrayList(new ColumnDesc(MY_TABLE_NAME + ".key", "STRING", 1, null), new ColumnDesc(MY_TABLE_NAME + ".value", "struct<name:string,ints:array<int>>", 2, null), new ColumnDesc(myTable1Name + ".key", "STRING", 3, null), new ColumnDesc(myTable1Name + ".value", "struct<name:string,ints:array<int>>", 4, null)), Lists.newArrayList(new QueryResult(Lists.<Object>newArrayList("1", "{\"name\":\"first\",\"ints\":[1,2,3,4,5]}", null, null)), new QueryResult(Lists.<Object>newArrayList("2", "{\"name\":\"two\",\"ints\":[10,11,12,13,14]}", "2", "{\"name\":\"two\",\"ints\":[20,21,22,23,24]}"))));
        query = String.format("select %s.key, %s.value, %s.key, %s.value from %s " + "full outer join %s on (%s.key=%s.key)", MY_TABLE_NAME, MY_TABLE_NAME, myTable1Name, myTable1Name, MY_TABLE_NAME, myTable1Name, MY_TABLE_NAME, myTable1Name);
        runCommand(NAMESPACE_ID, query, true, Lists.newArrayList(new ColumnDesc(MY_TABLE_NAME + ".key", "STRING", 1, null), new ColumnDesc(MY_TABLE_NAME + ".value", "struct<name:string,ints:array<int>>", 2, null), new ColumnDesc(myTable1Name + ".key", "STRING", 3, null), new ColumnDesc(myTable1Name + ".value", "struct<name:string,ints:array<int>>", 4, null)), Lists.newArrayList(new QueryResult(Lists.<Object>newArrayList("1", "{\"name\":\"first\",\"ints\":[1,2,3,4,5]}", null, null)), new QueryResult(Lists.<Object>newArrayList("2", "{\"name\":\"two\",\"ints\":[10,11,12,13,14]}", "2", "{\"name\":\"two\",\"ints\":[20,21,22,23,24]}")), new QueryResult(Lists.<Object>newArrayList(null, null, "3", "{\"name\":\"third\",\"ints\":[30,31,32,33,34]}"))));
    } finally {
        datasetFramework.deleteInstance(myTable1);
    }
}

Example 57 with DatasetId

use of co.cask.cdap.proto.id.DatasetId in project cdap by caskdata.

the class HiveExploreStructuredRecordTestRun method testCreateDropCustomDBAndTable.

private void testCreateDropCustomDBAndTable(@Nullable String database, @Nullable String tableName) throws Exception {
    String datasetName = "cdccat";
    DatasetId datasetId = NAMESPACE_ID.dataset(datasetName);
    ExploreProperties.Builder props = ExploreProperties.builder();
    if (tableName != null) {
        props.setExploreTableName(tableName);
    } else {
        tableName = getDatasetHiveName(datasetId);
    }
    if (database != null) {
        runCommand(NAMESPACE_ID, "create database if not exists " + database, false, null, null);
        props.setExploreDatabaseName(database);
    }
    try {
        datasetFramework.addInstance("email", datasetId, props.build());
        if (database == null) {
            runCommand(NAMESPACE_ID, "show tables", true, null, Lists.newArrayList(new QueryResult(Lists.<Object>newArrayList(MY_TABLE_NAME)), new QueryResult(Lists.<Object>newArrayList(tableName))));
        } else {
            runCommand(NAMESPACE_ID, "show tables in " + database, true, null, Lists.newArrayList(new QueryResult(Lists.<Object>newArrayList(tableName))));
        }
        datasetFramework.deleteInstance(datasetId);
        if (database == null) {
            runCommand(NAMESPACE_ID, "show tables", true, null, Lists.newArrayList(new QueryResult(Lists.<Object>newArrayList(MY_TABLE_NAME))));
        } else {
            runCommand(NAMESPACE_ID, "show tables in " + database, false, null, Collections.<QueryResult>emptyList());
        }
    } finally {
        if (database != null) {
            runCommand(NAMESPACE_ID, "drop database if exists " + database + "cascade", false, null, null);
        }
    }
}

Example 58 with DatasetId

use of co.cask.cdap.proto.id.DatasetId in project cdap by caskdata.

the class DatasetServiceStore method startUp.

@Override
protected void startUp() throws Exception {
    final DatasetId serviceStoreDatasetInstanceId = NamespaceId.SYSTEM.dataset(Constants.Service.SERVICE_INSTANCE_TABLE_NAME);
    table = Retries.supplyWithRetries(new Supplier<NoTxKeyValueTable>() {

        @Override
        public NoTxKeyValueTable get() {
            try {
                return DatasetsUtil.getOrCreateDataset(dsFramework, serviceStoreDatasetInstanceId, NoTxKeyValueTable.class.getName(), DatasetProperties.EMPTY, null);
            } catch (Exception e) {
                // Throwing RetryableException here is just to make it retry getting the dataset
                // an exception here usually means there is an hbase problem
                LOG.warn("Error getting service store dataset {}. Will retry after some time: {}", serviceStoreDatasetInstanceId, e.getMessage());
                throw new RetryableException(e);
            }
        }
    }, RetryStrategies.exponentialDelay(1, 30, TimeUnit.SECONDS));
}

Example 59 with DatasetId

use of co.cask.cdap.proto.id.DatasetId in project cdap by caskdata.

the class UsageHandler method getProgramDatasetUsage.

@GET
@Path("/namespaces/{namespace-id}/apps/{app-id}/{program-type}/{program-id}/datasets")
public void getProgramDatasetUsage(HttpRequest request, HttpResponder responder, @PathParam("namespace-id") String namespaceId, @PathParam("app-id") String appId, @PathParam("program-type") String programType, @PathParam("program-id") String programId) {
    ProgramType type = ProgramType.valueOfCategoryName(programType);
    final ProgramId id = new ProgramId(namespaceId, appId, type, programId);
    Set<DatasetId> ids = registry.getDatasets(id);
    responder.sendJson(HttpResponseStatus.OK, BackwardCompatibility.IdDatasetInstance.transform(ids));
}

Example 60 with DatasetId

use of co.cask.cdap.proto.id.DatasetId in project cdap by caskdata.

the class DatasetInstanceCreator method createInstances.

/**
   * Receives an input containing application specification and location
   * and verifies both.
   *
   * @param namespaceId the namespace to create the dataset instance in
   * @param datasets the datasets to create
   * @param ownerPrincipal the principal of the owner for the datasets to be created.
   */
void createInstances(NamespaceId namespaceId, Map<String, DatasetCreationSpec> datasets, @Nullable KerberosPrincipalId ownerPrincipal) throws Exception {
    // create dataset instances
    for (Map.Entry<String, DatasetCreationSpec> instanceEntry : datasets.entrySet()) {
        String instanceName = instanceEntry.getKey();
        DatasetId instanceId = namespaceId.dataset(instanceName);
        DatasetCreationSpec instanceSpec = instanceEntry.getValue();
        DatasetSpecification existingSpec = datasetFramework.getDatasetSpec(instanceId);
        if (existingSpec == null) {
            LOG.info("Adding dataset instance: {}", instanceName);
            datasetFramework.addInstance(instanceSpec.getTypeName(), instanceId, instanceSpec.getProperties(), ownerPrincipal);
        } else {
            if (!existingSpec.getType().equals(instanceSpec.getTypeName())) {
                throw new IncompatibleUpdateException(String.format("Existing dataset '%s' of type '%s' may not be updated to type '%s'", instanceName, existingSpec.getType(), instanceSpec.getTypeName()));
            }
            if (allowDatasetUncheckedUpgrade) {
                LOG.info("Updating dataset instance: {}", instanceName);
                datasetFramework.updateInstance(instanceId, instanceSpec.getProperties());
            }
        }
    }
}