Examples with WriterContext org.apache.hive.hcatalog.data.transfer.WriterContext used on opensource projects

Example 1 with WriterContext

use of org.apache.hive.hcatalog.data.transfer.WriterContext in project hive by apache.

the class DataWriterMaster method runsInMaster.

private static WriterContext runsInMaster(Map<String, String> config) throws HCatException {
    WriteEntity.Builder builder = new WriteEntity.Builder();
    WriteEntity entity = builder.withTable(config.get("table")).build();
    HCatWriter writer = DataTransferFactory.getHCatWriter(entity, config);
    WriterContext info = writer.prepareWrite();
    return info;
}

Example 2 with WriterContext

use of org.apache.hive.hcatalog.data.transfer.WriterContext in project hive by apache.

the class DataWriterMaster method main.

public static void main(String[] args) throws FileNotFoundException, IOException, ClassNotFoundException {
    // This config contains all the configuration that master node wants to provide
    // to the HCatalog.
    Properties externalConfigs = new Properties();
    externalConfigs.load(new FileReader(args[0]));
    Map<String, String> config = new HashMap<String, String>();
    for (Entry<Object, Object> kv : externalConfigs.entrySet()) {
        System.err.println("k: " + kv.getKey() + "\t v: " + kv.getValue());
        config.put((String) kv.getKey(), (String) kv.getValue());
    }
    if (args.length == 3 && "commit".equalsIgnoreCase(args[2])) {
        // Then, master commits if everything goes well.
        ObjectInputStream ois = new ObjectInputStream(new FileInputStream(new File(args[1])));
        WriterContext cntxt = (WriterContext) ois.readObject();
        commit(config, true, cntxt);
        System.exit(0);
    }
    // This piece of code runs in master node and gets necessary context.
    WriterContext cntxt = runsInMaster(config);
    // Master node will serialize writercontext and will make it available at slaves.
    File f = new File(args[1]);
    f.delete();
    ObjectOutputStream oos = new ObjectOutputStream(new FileOutputStream(f));
    oos.writeObject(cntxt);
    oos.flush();
    oos.close();
}

Example 3 with WriterContext

use of org.apache.hive.hcatalog.data.transfer.WriterContext in project hive by apache.

the class TestReaderWriter method test.

@Test
public void test() throws Exception, IOException, ClassNotFoundException {
    driver.run("drop table mytbl");
    driver.run("create table mytbl (a string, b int)");
    Iterator<Entry<String, String>> itr = hiveConf.iterator();
    Map<String, String> map = new HashMap<String, String>();
    while (itr.hasNext()) {
        Entry<String, String> kv = itr.next();
        map.put(kv.getKey(), kv.getValue());
    }
    WriterContext cntxt = runsInMaster(map);
    File writeCntxtFile = File.createTempFile("hcat-write", "temp");
    writeCntxtFile.deleteOnExit();
    // Serialize context.
    ObjectOutputStream oos = new ObjectOutputStream(new FileOutputStream(writeCntxtFile));
    oos.writeObject(cntxt);
    oos.flush();
    oos.close();
    // Now, deserialize it.
    ObjectInputStream ois = new ObjectInputStream(new FileInputStream(writeCntxtFile));
    cntxt = (WriterContext) ois.readObject();
    ois.close();
    runsInSlave(cntxt);
    commit(map, true, cntxt);
    ReaderContext readCntxt = runsInMaster(map, false);
    File readCntxtFile = File.createTempFile("hcat-read", "temp");
    readCntxtFile.deleteOnExit();
    oos = new ObjectOutputStream(new FileOutputStream(readCntxtFile));
    oos.writeObject(readCntxt);
    oos.flush();
    oos.close();
    ois = new ObjectInputStream(new FileInputStream(readCntxtFile));
    readCntxt = (ReaderContext) ois.readObject();
    ois.close();
    for (int i = 0; i < readCntxt.numSplits(); i++) {
        runsInSlave(readCntxt, i);
    }
}

Example 4 with WriterContext

use of org.apache.hive.hcatalog.data.transfer.WriterContext in project hive by apache.

the class TestReaderWriter method runsInMaster.

private WriterContext runsInMaster(Map<String, String> config) throws Exception {
    WriteEntity.Builder builder = new WriteEntity.Builder();
    WriteEntity entity = builder.withTable("mytbl").build();
    HCatWriter writer = DataTransferFactory.getHCatWriter(entity, config);
    WriterContext info = writer.prepareWrite();
    return info;
}

Example 5 with WriterContext

use of org.apache.hive.hcatalog.data.transfer.WriterContext in project hive by apache.

the class DataWriterSlave method main.

public static void main(String[] args) throws FileNotFoundException, IOException, ClassNotFoundException {
    ObjectInputStream ois = new ObjectInputStream(new FileInputStream(args[0]));
    WriterContext cntxt = (WriterContext) ois.readObject();
    ois.close();
    HCatWriter writer = DataTransferFactory.getHCatWriter(cntxt);
    writer.write(new HCatRecordItr(args[1]));
}