Examples with FireEventResponse org.apache.hadoop.hive.metastore.api.FireEventResponse used on opensource projects

Example 1 with FireEventResponse

use of org.apache.hadoop.hive.metastore.api.FireEventResponse in project hive by apache.

the class TestDbNotificationListener method insertPartition.

@Test
public void insertPartition() throws Exception {
    String defaultDbName = "default";
    String tblName = "insertptn";
    String tblOwner = "me";
    String serdeLocation = testTempDir;
    String fileAdded = "/warehouse/mytable/b1";
    String checksumAdded = "1234";
    FieldSchema col1 = new FieldSchema("col1", "int", "no comment");
    List<FieldSchema> cols = new ArrayList<FieldSchema>();
    cols.add(col1);
    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
    StorageDescriptor sd = new StorageDescriptor(cols, serdeLocation, "input", "output", false, 0, serde, null, null, emptyParameters);
    FieldSchema partCol1 = new FieldSchema("ds", "string", "no comment");
    List<FieldSchema> partCols = new ArrayList<FieldSchema>();
    List<String> partCol1Vals = Arrays.asList("today");
    List<String> partKeyVals = new ArrayList<String>();
    partKeyVals.add("today");
    partCols.add(partCol1);
    Table table = new Table(tblName, defaultDbName, tblOwner, startTime, startTime, 0, sd, partCols, emptyParameters, null, null, null);
    // Event 1
    msClient.createTable(table);
    Partition partition = new Partition(partCol1Vals, defaultDbName, tblName, startTime, startTime, sd, emptyParameters);
    // Event 2
    msClient.add_partition(partition);
    FireEventRequestData data = new FireEventRequestData();
    InsertEventRequestData insertData = new InsertEventRequestData();
    data.setInsertData(insertData);
    insertData.addToFilesAdded(fileAdded);
    insertData.addToFilesAddedChecksum(checksumAdded);
    insertData.setReplace(false);
    FireEventRequest rqst = new FireEventRequest(true, data);
    rqst.setDbName(defaultDbName);
    rqst.setTableName(tblName);
    rqst.setPartitionVals(partCol1Vals);
    // Event 3
    verifyInsertEventReceived(defaultDbName, tblName, Arrays.asList(partKeyVals), rqst, firstEventId + 3, 1);
    // Verify the eventID was passed to the non-transactional listener
    MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.INSERT, firstEventId + 3);
    MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.ADD_PARTITION, firstEventId + 2);
    MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_TABLE, firstEventId + 1);
    testEventCounts(defaultDbName, firstEventId, null, null, 3);
    // fire multiple insert events on partition
    // add some more partitions
    partition = new Partition(Arrays.asList("yesterday"), defaultDbName, tblName, startTime, startTime, sd, emptyParameters);
    // Event 4
    msClient.add_partition(partition);
    partition = new Partition(Arrays.asList("tomorrow"), defaultDbName, tblName, startTime, startTime, sd, emptyParameters);
    // Event 5
    msClient.add_partition(partition);
    data = new FireEventRequestData();
    List<InsertEventRequestData> insertEventRequestDataList = new ArrayList<>();
    data.setInsertDatas(insertEventRequestDataList);
    // event 6
    InsertEventRequestData insertData1 = new InsertEventRequestData();
    insertData1.addToFilesAdded(fileAdded);
    insertData1.addToFilesAddedChecksum(checksumAdded);
    insertData1.setReplace(false);
    insertData1.setPartitionVal(Arrays.asList("yesterday"));
    // event 7
    InsertEventRequestData insertData2 = new InsertEventRequestData();
    insertData2.addToFilesAdded(fileAdded);
    insertData2.addToFilesAddedChecksum(checksumAdded);
    insertData2.setReplace(false);
    insertData2.setPartitionVal(Arrays.asList("today"));
    // event 8
    InsertEventRequestData insertData3 = new InsertEventRequestData();
    insertData3.addToFilesAdded(fileAdded);
    insertData3.addToFilesAddedChecksum(checksumAdded);
    insertData3.setReplace(false);
    insertData3.setPartitionVal(Arrays.asList("tomorrow"));
    // fire insert event on 3 partitions
    insertEventRequestDataList.add(insertData1);
    insertEventRequestDataList.add(insertData2);
    insertEventRequestDataList.add(insertData3);
    rqst = new FireEventRequest(true, data);
    rqst.setDbName(defaultDbName);
    rqst.setTableName(tblName);
    verifyInsertEventReceived(defaultDbName, tblName, Arrays.asList(Arrays.asList("yesterday"), Arrays.asList("today"), Arrays.asList("tomorrow")), rqst, firstEventId + 6, 3);
    // negative test. partition values must be set when firing bulk insert events
    data.getInsertDatas().get(1).unsetPartitionVal();
    boolean threwException = false;
    try {
        FireEventResponse response = msClient.fireListenerEvent(rqst);
    } catch (MetaException ex) {
        threwException = true;
        Assert.assertTrue(ex instanceof MetaException);
        Assert.assertTrue(ex.getMessage().contains("Partition values must be set when firing multiple insert events"));
    }
    Assert.assertTrue("bulk insert event API didn't " + "throw exception when partition values were not set", threwException);
}

Example 2 with FireEventResponse

use of org.apache.hadoop.hive.metastore.api.FireEventResponse in project hive by apache.

the class TestDbNotificationListener method insertTable.

@Test
public void insertTable() throws Exception {
    String defaultDbName = "default";
    String tblName = "inserttbl";
    String tblOwner = "me";
    String serdeLocation = testTempDir;
    String fileAdded = "/warehouse/mytable/b1";
    String checksumAdded = "1234";
    FieldSchema col1 = new FieldSchema("col1", "int", "no comment");
    List<FieldSchema> cols = new ArrayList<FieldSchema>();
    cols.add(col1);
    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
    StorageDescriptor sd = new StorageDescriptor(cols, serdeLocation, "input", "output", false, 0, serde, null, null, emptyParameters);
    Table table = new Table(tblName, defaultDbName, tblOwner, startTime, startTime, 0, sd, null, emptyParameters, null, null, null);
    // Event 1
    msClient.createTable(table);
    FireEventRequestData data = new FireEventRequestData();
    InsertEventRequestData insertData = new InsertEventRequestData();
    data.setInsertData(insertData);
    insertData.addToFilesAdded(fileAdded);
    insertData.addToFilesAddedChecksum(checksumAdded);
    insertData.setReplace(false);
    FireEventRequest rqst = new FireEventRequest(true, data);
    rqst.setDbName(defaultDbName);
    rqst.setTableName(tblName);
    // Event 2
    FireEventResponse response = msClient.fireListenerEvent(rqst);
    assertTrue("Event id must be set in the fireEvent response", response.isSetEventIds());
    Assert.assertNotNull(response.getEventIds());
    Assert.assertTrue(response.getEventIds().size() == 1);
    Assert.assertEquals(firstEventId + 2, response.getEventIds().get(0).longValue());
    // Get notifications from metastore
    NotificationEventResponse rsp = msClient.getNextNotification(firstEventId, 0, null);
    assertEquals(2, rsp.getEventsSize());
    NotificationEvent event = rsp.getEvents().get(1);
    assertEquals(firstEventId + 2, event.getEventId());
    assertTrue(event.getEventTime() >= startTime);
    assertEquals(EventType.INSERT.toString(), event.getEventType());
    assertEquals(defaultDbName, event.getDbName());
    assertEquals(tblName, event.getTableName());
    // Parse the message field
    verifyInsert(event, defaultDbName, tblName);
    // Parse the message field
    InsertMessage insertMessage = md.getInsertMessage(event.getMessage());
    assertEquals(defaultDbName, insertMessage.getDB());
    assertEquals(tblName, insertMessage.getTable());
    assertEquals(TableType.MANAGED_TABLE.toString(), insertMessage.getTableType());
    assertFalse(insertMessage.isReplace());
    // Verify the eventID was passed to the non-transactional listener
    MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.INSERT, firstEventId + 2);
    MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_TABLE, firstEventId + 1);
    testEventCounts(defaultDbName, firstEventId, null, null, 2);
}

Example 3 with FireEventResponse

use of org.apache.hadoop.hive.metastore.api.FireEventResponse in project hive by apache.

the class TestDbNotificationListener method verifyInsertEventReceived.

private void verifyInsertEventReceived(String defaultDbName, String tblName, List<List<String>> partKeyVals, FireEventRequest rqst, long expectedStartEventId, int expectedNumOfEvents) throws Exception {
    FireEventResponse response = msClient.fireListenerEvent(rqst);
    assertTrue("Event id must be set in the fireEvent response", response.isSetEventIds());
    Assert.assertNotNull(response.getEventIds());
    Assert.assertTrue(response.getEventIds().size() == expectedNumOfEvents);
    for (int i = 0; i < expectedNumOfEvents; i++) {
        Assert.assertEquals(expectedStartEventId + i, response.getEventIds().get(i).longValue());
    }
    // Get notifications from metastore
    NotificationEventResponse rsp = msClient.getNextNotification(expectedStartEventId - 1, 0, null);
    assertEquals(expectedNumOfEvents, rsp.getEventsSize());
    for (int i = 0; i < expectedNumOfEvents; i++) {
        NotificationEvent event = rsp.getEvents().get(i);
        assertEquals(expectedStartEventId + i, event.getEventId());
        assertTrue(event.getEventTime() >= startTime);
        assertEquals(EventType.INSERT.toString(), event.getEventType());
        assertEquals(defaultDbName, event.getDbName());
        assertEquals(tblName, event.getTableName());
        // Parse the message field
        verifyInsert(event, defaultDbName, tblName);
        InsertMessage insertMessage = md.getInsertMessage(event.getMessage());
        List<String> ptnValues = insertMessage.getPtnObj().getValues();
        assertFalse(insertMessage.isReplace());
        assertEquals(partKeyVals.get(i), ptnValues);
    }
}