Examples with HCatNotificationEvent org.apache.hive.hcatalog.api.HCatNotificationEvent used on opensource projects

Example 11 with HCatNotificationEvent

use of org.apache.hive.hcatalog.api.HCatNotificationEvent in project hive by apache.

the class TestEximReplicationTasks method testAddPartition.

@Test
public void testAddPartition() throws IOException {
    Table t = new Table();
    t.setDbName("testdb");
    t.setTableName("testtable");
    List<FieldSchema> pkeys = HCatSchemaUtils.getFieldSchemas(HCatSchemaUtils.getHCatSchema("a:int,b:string").getFields());
    t.setPartitionKeys(pkeys);
    List<Partition> addedPtns = new ArrayList<Partition>();
    addedPtns.add(createPtn(t, Arrays.asList("120", "abc")));
    addedPtns.add(createPtn(t, Arrays.asList("201", "xyz")));
    NotificationEvent event = new NotificationEvent(getEventId(), getTime(), HCatConstants.HCAT_ADD_PARTITION_EVENT, msgFactory.buildAddPartitionMessage(t, addedPtns.iterator()).toString());
    event.setDbName(t.getDbName());
    event.setTableName(t.getTableName());
    HCatNotificationEvent hev = new HCatNotificationEvent(event);
    ReplicationTask rtask = ReplicationTask.create(client, hev);
    assertEquals(hev.toString(), rtask.getEvent().toString());
    verifyAddPartitionReplicationTask(rtask, t, addedPtns);
}

Example 12 with HCatNotificationEvent

use of org.apache.hive.hcatalog.api.HCatNotificationEvent in project hive by apache.

the class ReplicationV1CompatRule method doBackwardCompatibilityCheck.

public void doBackwardCompatibilityCheck(long testEventIdBefore, long testEventIdAfter) {
    // try to instantiate the old replv1 task generation on every event produced.
    long timeBefore = System.currentTimeMillis();
    Map<NotificationEvent, RuntimeException> unhandledTasks = new LinkedHashMap<>();
    Map<NotificationEvent, RuntimeException> incompatibleTasks = new LinkedHashMap<>();
    int eventCount = 0;
    LOG.info("Checking replv1 backward compatibility for events between : " + testEventIdBefore + " -> " + testEventIdAfter);
    IMetaStoreClient.NotificationFilter evFilter = new IMetaStoreClient.NotificationFilter() {

        @Override
        public boolean accept(NotificationEvent notificationEvent) {
            return true;
        }
    };
    EventUtils.MSClientNotificationFetcher evFetcher = new EventUtils.MSClientNotificationFetcher(metaStoreClient);
    try {
        EventUtils.NotificationEventIterator evIter = new EventUtils.NotificationEventIterator(evFetcher, testEventIdBefore, Ints.checkedCast(testEventIdAfter - testEventIdBefore) + 1, evFilter);
        ReplicationTask.resetFactory(null);
        assertTrue("We should have found some events", evIter.hasNext());
        while (evIter.hasNext()) {
            eventCount++;
            NotificationEvent ev = evIter.next();
            // convert to HCatNotificationEvent, and then try to instantiate a ReplicationTask on it.
            try {
                ReplicationTask rtask = ReplicationTask.create(HCatClient.create(hconf), new HCatNotificationEvent(ev));
                if (rtask instanceof ErroredReplicationTask) {
                    unhandledTasks.put(ev, ((ErroredReplicationTask) rtask).getCause());
                }
            } catch (RuntimeException re) {
                incompatibleTasks.put(ev, re);
            }
        }
    } catch (IOException e) {
        assertNull("Got an exception when we shouldn't have - replv1 backward incompatibility issue:", e);
    }
    if (unhandledTasks.size() > 0) {
        LOG.warn("Events found that would not be coverable by replv1 replication: " + unhandledTasks.size());
        for (NotificationEvent ev : unhandledTasks.keySet()) {
            RuntimeException re = unhandledTasks.get(ev);
            LOG.warn("ErroredReplicationTask encountered - new event type does not correspond to a replv1 task:" + ev.toString(), re);
        }
    }
    if (incompatibleTasks.size() > 0) {
        LOG.warn("Events found that caused errors in replv1 replication: " + incompatibleTasks.size());
        for (NotificationEvent ev : incompatibleTasks.keySet()) {
            RuntimeException re = incompatibleTasks.get(ev);
            LOG.warn("RuntimeException encountered - new event type caused a replv1 break." + ev.toString(), re);
        }
    }
    assertEquals(0, incompatibleTasks.size());
    long timeAfter = System.currentTimeMillis();
    LOG.info("Backward compatibility check timing:" + timeBefore + " -> " + timeAfter + ", ev: " + testEventIdBefore + " => " + testEventIdAfter + ", #events processed=" + eventCount);
}