Examples with SchemaVersion org.apache.hadoop.hive.metastore.api.SchemaVersion used on opensource projects

Example 1 with SchemaVersion

use of org.apache.hadoop.hive.metastore.api.SchemaVersion in project hive by apache.

the class TestObjectStoreSchemaMethods method alterSchemaVersion.

@Test
public void alterSchemaVersion() throws TException {
    String dbName = createUniqueDatabaseForTest();
    String schemaName = "schema371234";
    int version = 1;
    SchemaVersion schemaVersion = objectStore.getSchemaVersion(new SchemaVersionDescriptor(new ISchemaName(dbName, schemaName), version));
    Assert.assertNull(schemaVersion);
    ISchema schema = new ISchemaBuilder().setSchemaType(SchemaType.AVRO).setName(schemaName).setDbName(dbName).build();
    objectStore.createISchema(schema);
    schemaVersion = new SchemaVersionBuilder().versionOf(schema).setVersion(version).addCol("a", ColumnType.INT_TYPE_NAME).addCol("b", ColumnType.FLOAT_TYPE_NAME).setState(SchemaVersionState.INITIATED).build();
    objectStore.addSchemaVersion(schemaVersion);
    schemaVersion = objectStore.getSchemaVersion(new SchemaVersionDescriptor(new ISchemaName(dbName, schemaName), version));
    Assert.assertNotNull(schemaVersion);
    Assert.assertEquals(schemaName, schemaVersion.getSchema().getSchemaName());
    Assert.assertEquals(dbName, schemaVersion.getSchema().getDbName());
    Assert.assertEquals(version, schemaVersion.getVersion());
    Assert.assertEquals(SchemaVersionState.INITIATED, schemaVersion.getState());
    schemaVersion.setState(SchemaVersionState.REVIEWED);
    String serdeName = "serde for " + schemaName;
    SerDeInfo serde = new SerDeInfo(serdeName, "", Collections.emptyMap());
    String serializer = "org.apache.hadoop.hive.metastore.test.Serializer";
    String deserializer = "org.apache.hadoop.hive.metastore.test.Deserializer";
    serde.setSerializerClass(serializer);
    serde.setDeserializerClass(deserializer);
    schemaVersion.setSerDe(serde);
    objectStore.alterSchemaVersion(new SchemaVersionDescriptor(new ISchemaName(dbName, schemaName), version), schemaVersion);
    schemaVersion = objectStore.getSchemaVersion(new SchemaVersionDescriptor(new ISchemaName(dbName, schemaName), version));
    Assert.assertNotNull(schemaVersion);
    Assert.assertEquals(schemaName, schemaVersion.getSchema().getSchemaName());
    Assert.assertEquals(dbName, schemaVersion.getSchema().getDbName());
    Assert.assertEquals(version, schemaVersion.getVersion());
    Assert.assertEquals(SchemaVersionState.REVIEWED, schemaVersion.getState());
    Assert.assertEquals(serdeName, schemaVersion.getSerDe().getName());
    Assert.assertEquals(serializer, schemaVersion.getSerDe().getSerializerClass());
    Assert.assertEquals(deserializer, schemaVersion.getSerDe().getDeserializerClass());
}

Example 2 with SchemaVersion

use of org.apache.hadoop.hive.metastore.api.SchemaVersion in project hive by apache.

the class TestObjectStoreSchemaMethods method addDuplicateSchemaVersion.

@Test(expected = AlreadyExistsException.class)
public void addDuplicateSchemaVersion() throws TException {
    String dbName = createUniqueDatabaseForTest();
    String schemaName = "schema1234";
    int version = 1;
    SchemaVersion schemaVersion = objectStore.getSchemaVersion(new SchemaVersionDescriptor(new ISchemaName(dbName, schemaName), version));
    Assert.assertNull(schemaVersion);
    ISchema schema = new ISchemaBuilder().setSchemaType(SchemaType.AVRO).setName(schemaName).setDbName(dbName).build();
    objectStore.createISchema(schema);
    schemaVersion = new SchemaVersionBuilder().versionOf(schema).setVersion(version).addCol("a", ColumnType.INT_TYPE_NAME).addCol("b", ColumnType.FLOAT_TYPE_NAME).build();
    objectStore.addSchemaVersion(schemaVersion);
    objectStore.addSchemaVersion(schemaVersion);
}

Example 3 with SchemaVersion

use of org.apache.hadoop.hive.metastore.api.SchemaVersion in project hive by apache.

the class TestObjectStoreSchemaMethods method addSchemaVersion.

@Test
public void addSchemaVersion() throws TException {
    String dbName = createUniqueDatabaseForTest();
    String schemaName = "schema37";
    int version = 1;
    SchemaVersion schemaVersion = objectStore.getSchemaVersion(new SchemaVersionDescriptor(new ISchemaName(dbName, schemaName), version));
    Assert.assertNull(schemaVersion);
    ISchema schema = new ISchemaBuilder().setSchemaType(SchemaType.AVRO).setName(schemaName).setDbName(dbName).build();
    objectStore.createISchema(schema);
    String description = "very descriptive";
    String schemaText = "this should look like json, but oh well";
    String fingerprint = "this should be an md5 string";
    String versionName = "why would I name a version?";
    long creationTime = 10;
    String serdeName = "serde_for_schema37";
    String serializer = "org.apache.hadoop.hive.metastore.test.Serializer";
    String deserializer = "org.apache.hadoop.hive.metastore.test.Deserializer";
    String serdeDescription = "how do you describe a serde?";
    schemaVersion = new SchemaVersionBuilder().versionOf(schema).setVersion(version).addCol("a", ColumnType.INT_TYPE_NAME).addCol("b", ColumnType.FLOAT_TYPE_NAME).setCreatedAt(creationTime).setState(SchemaVersionState.INITIATED).setDescription(description).setSchemaText(schemaText).setFingerprint(fingerprint).setName(versionName).setSerdeName(serdeName).setSerdeSerializerClass(serializer).setSerdeDeserializerClass(deserializer).setSerdeDescription(serdeDescription).build();
    objectStore.addSchemaVersion(schemaVersion);
    schemaVersion = objectStore.getSchemaVersion(new SchemaVersionDescriptor(new ISchemaName(dbName, schemaName), version));
    Assert.assertNotNull(schemaVersion);
    Assert.assertEquals(schemaName, schemaVersion.getSchema().getSchemaName());
    Assert.assertEquals(dbName, schemaVersion.getSchema().getDbName());
    Assert.assertEquals(version, schemaVersion.getVersion());
    Assert.assertEquals(creationTime, schemaVersion.getCreatedAt());
    Assert.assertEquals(SchemaVersionState.INITIATED, schemaVersion.getState());
    Assert.assertEquals(description, schemaVersion.getDescription());
    Assert.assertEquals(schemaText, schemaVersion.getSchemaText());
    Assert.assertEquals(fingerprint, schemaVersion.getFingerprint());
    Assert.assertEquals(versionName, schemaVersion.getName());
    Assert.assertEquals(serdeName, schemaVersion.getSerDe().getName());
    Assert.assertEquals(serializer, schemaVersion.getSerDe().getSerializerClass());
    Assert.assertEquals(deserializer, schemaVersion.getSerDe().getDeserializerClass());
    Assert.assertEquals(serdeDescription, schemaVersion.getSerDe().getDescription());
    Assert.assertEquals(2, schemaVersion.getColsSize());
    List<FieldSchema> cols = schemaVersion.getCols();
    Collections.sort(cols);
    Assert.assertEquals("a", cols.get(0).getName());
    Assert.assertEquals(ColumnType.INT_TYPE_NAME, cols.get(0).getType());
    Assert.assertEquals("b", cols.get(1).getName());
    Assert.assertEquals(ColumnType.FLOAT_TYPE_NAME, cols.get(1).getType());
    objectStore.dropSchemaVersion(new SchemaVersionDescriptor(new ISchemaName(dbName, schemaName), version));
    schemaVersion = objectStore.getSchemaVersion(new SchemaVersionDescriptor(new ISchemaName(dbName, schemaName), version));
    Assert.assertNull(schemaVersion);
}

Example 4 with SchemaVersion

use of org.apache.hadoop.hive.metastore.api.SchemaVersion in project hive by apache.

the class TestObjectStoreSchemaMethods method multipleSchemaVersions.

// Test that adding multiple versions of the same schema
@Test
public void multipleSchemaVersions() throws TException {
    String dbName = createUniqueDatabaseForTest();
    String schemaName = "schema195";
    ISchema schema = new ISchemaBuilder().setSchemaType(SchemaType.AVRO).setName(schemaName).setDbName(dbName).build();
    objectStore.createISchema(schema);
    SchemaVersion schemaVersion = new SchemaVersionBuilder().versionOf(schema).setVersion(1).addCol("a", ColumnType.BIGINT_TYPE_NAME).build();
    objectStore.addSchemaVersion(schemaVersion);
    schemaVersion = new SchemaVersionBuilder().versionOf(schema).setVersion(2).addCol("a", ColumnType.BIGINT_TYPE_NAME).addCol("b", ColumnType.DATE_TYPE_NAME).build();
    objectStore.addSchemaVersion(schemaVersion);
    schemaVersion = new SchemaVersionBuilder().versionOf(schema).setVersion(3).addCol("a", ColumnType.BIGINT_TYPE_NAME).addCol("b", ColumnType.DATE_TYPE_NAME).addCol("c", ColumnType.TIMESTAMP_TYPE_NAME).build();
    objectStore.addSchemaVersion(schemaVersion);
    schemaVersion = objectStore.getLatestSchemaVersion(new ISchemaName(dbName, schemaName));
    Assert.assertEquals(3, schemaVersion.getVersion());
    Assert.assertEquals(3, schemaVersion.getColsSize());
    List<FieldSchema> cols = schemaVersion.getCols();
    Collections.sort(cols);
    Assert.assertEquals("a", cols.get(0).getName());
    Assert.assertEquals("b", cols.get(1).getName());
    Assert.assertEquals("c", cols.get(2).getName());
    Assert.assertEquals(ColumnType.BIGINT_TYPE_NAME, cols.get(0).getType());
    Assert.assertEquals(ColumnType.DATE_TYPE_NAME, cols.get(1).getType());
    Assert.assertEquals(ColumnType.TIMESTAMP_TYPE_NAME, cols.get(2).getType());
    schemaVersion = objectStore.getLatestSchemaVersion(new ISchemaName(dbName, "no.such.schema.with.this.name"));
    Assert.assertNull(schemaVersion);
    List<SchemaVersion> versions = objectStore.getAllSchemaVersion(new ISchemaName(dbName, "there.really.isnt.a.schema.named.this"));
    Assert.assertNull(versions);
    versions = objectStore.getAllSchemaVersion(new ISchemaName(dbName, schemaName));
    Assert.assertEquals(3, versions.size());
    versions.sort(Comparator.comparingInt(SchemaVersion::getVersion));
    Assert.assertEquals(1, versions.get(0).getVersion());
    Assert.assertEquals(1, versions.get(0).getColsSize());
    Assert.assertEquals(ColumnType.BIGINT_TYPE_NAME, versions.get(0).getCols().get(0).getType());
    Assert.assertEquals(2, versions.get(1).getVersion());
    Assert.assertEquals(2, versions.get(1).getColsSize());
    cols = versions.get(1).getCols();
    Collections.sort(cols);
    Assert.assertEquals("a", cols.get(0).getName());
    Assert.assertEquals("b", cols.get(1).getName());
    Assert.assertEquals(ColumnType.BIGINT_TYPE_NAME, cols.get(0).getType());
    Assert.assertEquals(ColumnType.DATE_TYPE_NAME, cols.get(1).getType());
    Assert.assertEquals(3, versions.get(2).getVersion());
    Assert.assertEquals(3, versions.get(2).getColsSize());
    cols = versions.get(2).getCols();
    Collections.sort(cols);
    Assert.assertEquals("a", cols.get(0).getName());
    Assert.assertEquals("b", cols.get(1).getName());
    Assert.assertEquals("c", cols.get(2).getName());
    Assert.assertEquals(ColumnType.BIGINT_TYPE_NAME, cols.get(0).getType());
    Assert.assertEquals(ColumnType.DATE_TYPE_NAME, cols.get(1).getType());
    Assert.assertEquals(ColumnType.TIMESTAMP_TYPE_NAME, cols.get(2).getType());
}

Example 5 with SchemaVersion

use of org.apache.hadoop.hive.metastore.api.SchemaVersion in project hive by apache.

the class ObjectStore method getSchemaVersionsByColumns.

@Override
public List<SchemaVersion> getSchemaVersionsByColumns(String colName, String colNamespace, String type) throws MetaException {
    if (colName == null && colNamespace == null) {
        // Don't allow a query that returns everything, it will blow stuff up.
        throw new MetaException("You must specify column name or column namespace, else your query " + "may be too large");
    }
    boolean committed = false;
    Query query = null;
    try {
        openTransaction();
        if (colName != null)
            colName = normalizeIdentifier(colName);
        if (type != null)
            type = normalizeIdentifier(type);
        Map<String, String> parameters = new HashMap<>(3);
        StringBuilder sql = new StringBuilder("select SCHEMA_VERSION_ID from " + "SCHEMA_VERSION, COLUMNS_V2 where SCHEMA_VERSION.CD_ID = COLUMNS_V2.CD_ID ");
        if (colName != null) {
            sql.append("and COLUMNS_V2.COLUMN_NAME = :colName ");
            parameters.put("colName", colName);
        }
        if (colNamespace != null) {
            sql.append("and COLUMNS_V2.COMMENT = :colComment ");
            parameters.put("colComment", colNamespace);
        }
        if (type != null) {
            sql.append("and COLUMNS_V2.TYPE_NAME = :colType ");
            parameters.put("colType", type);
        }
        if (LOG.isDebugEnabled()) {
            LOG.debug("getSchemaVersionsByColumns going to execute query " + sql.toString());
            LOG.debug("With parameters");
            for (Map.Entry<String, String> p : parameters.entrySet()) {
                LOG.debug(p.getKey() + " : " + p.getValue());
            }
        }
        query = pm.newQuery("javax.jdo.query.SQL", sql.toString());
        query.setClass(MSchemaVersion.class);
        List<MSchemaVersion> mSchemaVersions = query.setNamedParameters(parameters).executeList();
        if (mSchemaVersions == null || mSchemaVersions.isEmpty())
            return Collections.emptyList();
        pm.retrieveAll(mSchemaVersions);
        List<SchemaVersion> schemaVersions = new ArrayList<>(mSchemaVersions.size());
        for (MSchemaVersion mSchemaVersion : mSchemaVersions) {
            pm.retrieveAll(mSchemaVersion.getCols());
            if (mSchemaVersion.getSerDe() != null)
                pm.retrieve(mSchemaVersion.getSerDe());
            schemaVersions.add(convertToSchemaVersion(mSchemaVersion));
        }
        committed = commitTransaction();
        return schemaVersions;
    } finally {
        rollbackAndCleanup(committed, query);
    }
}