Examples with InsertEventRequestData org.apache.hadoop.hive.metastore.api.InsertEventRequestData used on opensource projects

Example 1 with InsertEventRequestData

use of org.apache.hadoop.hive.metastore.api.InsertEventRequestData in project hive by apache.

the class Hive method fireInsertEvent.

private void fireInsertEvent(Table tbl, Map<String, String> partitionSpec, List<Path> newFiles) throws HiveException {
    if (conf.getBoolVar(ConfVars.FIRE_EVENTS_FOR_DML)) {
        LOG.debug("Firing dml insert event");
        if (tbl.isTemporary()) {
            LOG.debug("Not firing dml insert event as " + tbl.getTableName() + " is temporary");
            return;
        }
        try {
            FileSystem fileSystem = tbl.getDataLocation().getFileSystem(conf);
            FireEventRequestData data = new FireEventRequestData();
            InsertEventRequestData insertData = new InsertEventRequestData();
            data.setInsertData(insertData);
            if (newFiles != null && newFiles.size() > 0) {
                for (Path p : newFiles) {
                    insertData.addToFilesAdded(p.toString());
                    FileChecksum cksum = fileSystem.getFileChecksum(p);
                    // File checksum is not implemented for local filesystem (RawLocalFileSystem)
                    if (cksum != null) {
                        String checksumString = StringUtils.byteToHexString(cksum.getBytes(), 0, cksum.getLength());
                        insertData.addToFilesAddedChecksum(checksumString);
                    } else {
                        // Add an empty checksum string for filesystems that don't generate one
                        insertData.addToFilesAddedChecksum("");
                    }
                }
            } else {
                insertData.setFilesAdded(new ArrayList<String>());
            }
            FireEventRequest rqst = new FireEventRequest(true, data);
            rqst.setDbName(tbl.getDbName());
            rqst.setTableName(tbl.getTableName());
            if (partitionSpec != null && partitionSpec.size() > 0) {
                List<String> partVals = new ArrayList<String>(partitionSpec.size());
                for (FieldSchema fs : tbl.getPartitionKeys()) {
                    partVals.add(partitionSpec.get(fs.getName()));
                }
                rqst.setPartitionVals(partVals);
            }
            getMSC().fireListenerEvent(rqst);
        } catch (IOException | TException e) {
            throw new HiveException(e);
        }
    }
}

Example 2 with InsertEventRequestData

use of org.apache.hadoop.hive.metastore.api.InsertEventRequestData in project hive by apache.

the class TestDbNotificationListener method insertPartition.

@Test
public void insertPartition() throws Exception {
    String defaultDbName = "default";
    String tblName = "insertptn";
    String tblOwner = "me";
    String serdeLocation = "file:/tmp";
    String fileAdded = "/warehouse/mytable/b1";
    String checksumAdded = "1234";
    FieldSchema col1 = new FieldSchema("col1", "int", "no comment");
    List<FieldSchema> cols = new ArrayList<FieldSchema>();
    cols.add(col1);
    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
    StorageDescriptor sd = new StorageDescriptor(cols, serdeLocation, "input", "output", false, 0, serde, null, null, emptyParameters);
    FieldSchema partCol1 = new FieldSchema("ds", "string", "no comment");
    List<FieldSchema> partCols = new ArrayList<FieldSchema>();
    List<String> partCol1Vals = Arrays.asList("today");
    LinkedHashMap<String, String> partKeyVals = new LinkedHashMap<String, String>();
    partKeyVals.put("ds", "today");
    partCols.add(partCol1);
    Table table = new Table(tblName, defaultDbName, tblOwner, startTime, startTime, 0, sd, partCols, emptyParameters, null, null, null);
    // Event 1
    msClient.createTable(table);
    Partition partition = new Partition(partCol1Vals, defaultDbName, tblName, startTime, startTime, sd, emptyParameters);
    // Event 2
    msClient.add_partition(partition);
    FireEventRequestData data = new FireEventRequestData();
    InsertEventRequestData insertData = new InsertEventRequestData();
    data.setInsertData(insertData);
    insertData.addToFilesAdded(fileAdded);
    insertData.addToFilesAddedChecksum(checksumAdded);
    FireEventRequest rqst = new FireEventRequest(true, data);
    rqst.setDbName(defaultDbName);
    rqst.setTableName(tblName);
    rqst.setPartitionVals(partCol1Vals);
    // Event 3
    msClient.fireListenerEvent(rqst);
    // Get notifications from metastore
    NotificationEventResponse rsp = msClient.getNextNotification(firstEventId, 0, null);
    assertEquals(3, rsp.getEventsSize());
    NotificationEvent event = rsp.getEvents().get(2);
    assertEquals(firstEventId + 3, event.getEventId());
    assertTrue(event.getEventTime() >= startTime);
    assertEquals(EventType.INSERT.toString(), event.getEventType());
    assertEquals(defaultDbName, event.getDbName());
    assertEquals(tblName, event.getTableName());
    // Parse the message field
    verifyInsert(event, defaultDbName, tblName);
    InsertMessage insertMessage = md.getInsertMessage(event.getMessage());
    Map<String, String> partKeyValsFromNotif = insertMessage.getPartitionKeyValues();
    assertMapEquals(partKeyVals, partKeyValsFromNotif);
}

Example 3 with InsertEventRequestData

use of org.apache.hadoop.hive.metastore.api.InsertEventRequestData in project hive by apache.

the class TestDbNotificationListener method insertTable.

@Test
public void insertTable() throws Exception {
    String defaultDbName = "default";
    String tblName = "inserttbl";
    String tblOwner = "me";
    String serdeLocation = "file:/tmp";
    String fileAdded = "/warehouse/mytable/b1";
    String checksumAdded = "1234";
    FieldSchema col1 = new FieldSchema("col1", "int", "no comment");
    List<FieldSchema> cols = new ArrayList<FieldSchema>();
    cols.add(col1);
    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
    StorageDescriptor sd = new StorageDescriptor(cols, serdeLocation, "input", "output", false, 0, serde, null, null, emptyParameters);
    Table table = new Table(tblName, defaultDbName, tblOwner, startTime, startTime, 0, sd, null, emptyParameters, null, null, null);
    // Event 1
    msClient.createTable(table);
    FireEventRequestData data = new FireEventRequestData();
    InsertEventRequestData insertData = new InsertEventRequestData();
    data.setInsertData(insertData);
    insertData.addToFilesAdded(fileAdded);
    insertData.addToFilesAddedChecksum(checksumAdded);
    FireEventRequest rqst = new FireEventRequest(true, data);
    rqst.setDbName(defaultDbName);
    rqst.setTableName(tblName);
    // Event 2
    msClient.fireListenerEvent(rqst);
    // Get notifications from metastore
    NotificationEventResponse rsp = msClient.getNextNotification(firstEventId, 0, null);
    assertEquals(2, rsp.getEventsSize());
    NotificationEvent event = rsp.getEvents().get(1);
    assertEquals(firstEventId + 2, event.getEventId());
    assertTrue(event.getEventTime() >= startTime);
    assertEquals(EventType.INSERT.toString(), event.getEventType());
    assertEquals(defaultDbName, event.getDbName());
    assertEquals(tblName, event.getTableName());
    // Parse the message field
    verifyInsert(event, defaultDbName, tblName);
}