Examples with Tuple org.apache.hadoop.hive.ql.parse.WarehouseInstance.Tuple used on opensource projects

Example 1 with Tuple

use of org.apache.hadoop.hive.ql.parse.WarehouseInstance.Tuple in project hive by apache.

the class TestReplicationScenariosAcrossInstances method testMoveOptimizationIncrementalFailureAfterCopy.

@Test
public void testMoveOptimizationIncrementalFailureAfterCopy() throws Throwable {
    String replicatedDbName_CM = replicatedDbName + "_CM";
    WarehouseInstance.Tuple bootstrapDump = primary.run("use " + primaryDbName).run("create table t2 (place string) partitioned by (country string)").run("ALTER TABLE t2 ADD PARTITION (country='india')").dump(primaryDbName);
    replica.load(replicatedDbName, primaryDbName);
    // delete load ack to reuse the dump
    new Path(bootstrapDump.dumpLocation).getFileSystem(conf).delete(new Path(bootstrapDump.dumpLocation + Path.SEPARATOR + ReplUtils.REPL_HIVE_BASE_DIR + Path.SEPARATOR + LOAD_ACKNOWLEDGEMENT.toString()), true);
    replica.load(replicatedDbName_CM, primaryDbName);
    replica.run("alter database " + replicatedDbName + " set DBPROPERTIES ('" + SOURCE_OF_REPLICATION + "' = '1,2,3')").run("alter database " + replicatedDbName_CM + " set DBPROPERTIES ('" + SOURCE_OF_REPLICATION + "' = '1,2,3')");
    WarehouseInstance.Tuple tuple = primary.run("use " + primaryDbName).run("insert into table t2 partition(country='india') values ('bangalore')").dump(primaryDbName, Collections.emptyList());
    testMoveOptimization(primaryDbName, replicatedDbName, replicatedDbName_CM, "t2", "INSERT", tuple);
}

Example 2 with Tuple

use of org.apache.hadoop.hive.ql.parse.WarehouseInstance.Tuple in project hive by apache.

the class TestReplicationScenariosAcrossInstances method testCreateFunctionWithFunctionBinaryJarsOnHDFS.

@Test
public void testCreateFunctionWithFunctionBinaryJarsOnHDFS() throws Throwable {
    Dependencies dependencies = dependencies("ivy://io.github.myui:hivemall:0.4.0-2", primary);
    String jarSubString = dependencies.toJarSubSql();
    List<String> withClause = Arrays.asList("'" + HiveConf.ConfVars.REPL_RUN_DATA_COPY_TASKS_ON_TARGET.varname + "'='false'");
    primary.run("CREATE FUNCTION " + primaryDbName + ".anotherFunction as 'hivemall.tools.string.StopwordUDF' " + "using " + jarSubString);
    WarehouseInstance.Tuple tuple = primary.dump(primaryDbName, withClause);
    replica.load(replicatedDbName, primaryDbName, withClause).run("SHOW FUNCTIONS LIKE '" + replicatedDbName + "%'").verifyResult(replicatedDbName + ".anotherFunction");
    assertFunctionJarsOnTarget("anotherFunction", dependencies.jarNames());
}

Example 3 with Tuple

use of org.apache.hadoop.hive.ql.parse.WarehouseInstance.Tuple in project hive by apache.

the class TestReplicationScenariosAcrossInstances method testIncrementalCreateFunctionWithFunctionBinaryJarsOnHDFS.

@Test
public void testIncrementalCreateFunctionWithFunctionBinaryJarsOnHDFS() throws Throwable {
    WarehouseInstance.Tuple bootStrapDump = primary.dump(primaryDbName);
    replica.load(replicatedDbName, primaryDbName).run("REPL STATUS " + replicatedDbName).verifyResult(bootStrapDump.lastReplicationId);
    Dependencies dependencies = dependencies("ivy://io.github.myui:hivemall:0.4.0-2", primary);
    String jarSubString = dependencies.toJarSubSql();
    primary.run("CREATE FUNCTION " + primaryDbName + ".anotherFunction as 'hivemall.tools.string.StopwordUDF' " + "using " + jarSubString);
    WarehouseInstance.Tuple tuple = primary.dump(primaryDbName);
    replica.load(replicatedDbName, primaryDbName).run("SHOW FUNCTIONS LIKE '" + replicatedDbName + "%'").verifyResult(replicatedDbName + ".anotherFunction");
    assertFunctionJarsOnTarget("anotherFunction", dependencies.jarNames());
}

Example 4 with Tuple

use of org.apache.hadoop.hive.ql.parse.WarehouseInstance.Tuple in project hive by apache.

the class TestReplicationScenariosAcrossInstances method testMetadataBootstrapDump.

@Test
public void testMetadataBootstrapDump() throws Throwable {
    WarehouseInstance.Tuple tuple = primary.run("use " + primaryDbName).run("create table  acid_table (key int, value int) partitioned by (load_date date) " + "clustered by(key) into 2 buckets stored as orc tblproperties ('transactional'='true')").run("create table table1 (i int, j int)").run("insert into table1 values (1,2)").dump(primaryDbName, Collections.singletonList("'hive.repl.dump.metadata.only'='true'"));
    replica.load(replicatedDbName, primaryDbName).run("use " + replicatedDbName).run("show tables").verifyResults(new String[] { "acid_table", "table1" }).run("select * from table1").verifyResults(Collections.emptyList());
}

Example 5 with Tuple

use of org.apache.hadoop.hive.ql.parse.WarehouseInstance.Tuple in project hive by apache.

the class TestReplicationScenariosAcrossInstances method testNonReplDBMetadataReplication.

@Test
public void testNonReplDBMetadataReplication() throws Throwable {
    String dbName = primaryDbName + "_metadata";
    WarehouseInstance.Tuple tuple = primary.run("create database " + dbName).run("use " + dbName).run("create table table1 (i int, j int)").run("create table table2 (a int, city string) partitioned by (country string)").run("create table table3 (i int, j int)").run("insert into table1 values (1,2)").dump(dbName, Collections.singletonList("'hive.repl.dump.metadata.only'='true'"));
    replica.load(replicatedDbName, dbName).run("use " + replicatedDbName).run("show tables").verifyResults(new String[] { "table1", "table2", "table3" }).run("select * from table1").verifyResults(Collections.emptyList());
    tuple = primary.run("use " + dbName).run("alter table table1 rename to renamed_table1").run("insert into table2 partition(country='india') values (1,'mumbai') ").run("create table table4 (i int, j int)").dump(dbName, Collections.singletonList("'hive.repl.dump.metadata.only'='true'"));
    replica.load(replicatedDbName, dbName).run("use " + replicatedDbName).run("show tables").verifyResults(new String[] { "renamed_table1", "table2", "table3", "table4" }).run("select * from renamed_table1").verifyResults(Collections.emptyList()).run("select * from table2").verifyResults(Collections.emptyList());
}