Examples with StorageDescriptor org.apache.hadoop.hive.metastore.api.StorageDescriptor used on opensource projects

Example 96 with StorageDescriptor

use of org.apache.hadoop.hive.metastore.api.StorageDescriptor in project hive by apache.

the class TestDbNotificationListener method dropPartition.

@Test
public void dropPartition() throws Exception {
    String defaultDbName = "default";
    String tblName = "dropptn";
    String tblOwner = "me";
    String serdeLocation = testTempDir;
    FieldSchema col1 = new FieldSchema("col1", "int", "no comment");
    List<FieldSchema> cols = new ArrayList<FieldSchema>();
    cols.add(col1);
    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
    StorageDescriptor sd = new StorageDescriptor(cols, serdeLocation, "input", "output", false, 0, serde, null, null, emptyParameters);
    FieldSchema partCol1 = new FieldSchema("ds", "string", "no comment");
    List<FieldSchema> partCols = new ArrayList<FieldSchema>();
    List<String> partCol1Vals = Arrays.asList("today");
    partCols.add(partCol1);
    Table table = new Table(tblName, defaultDbName, tblOwner, startTime, startTime, 0, sd, partCols, emptyParameters, null, null, null);
    // Event 1
    msClient.createTable(table);
    Partition partition = new Partition(partCol1Vals, defaultDbName, tblName, startTime, startTime, sd, emptyParameters);
    // Event 2
    msClient.add_partition(partition);
    // Event 3
    msClient.dropPartition(defaultDbName, tblName, partCol1Vals, false);
    // Get notifications from metastore
    NotificationEventResponse rsp = msClient.getNextNotification(firstEventId, 0, null);
    assertEquals(3, rsp.getEventsSize());
    NotificationEvent event = rsp.getEvents().get(2);
    assertEquals(firstEventId + 3, event.getEventId());
    assertTrue(event.getEventTime() >= startTime);
    assertEquals(EventType.DROP_PARTITION.toString(), event.getEventType());
    assertEquals(defaultDbName, event.getDbName());
    assertEquals(tblName, event.getTableName());
    // Parse the message field
    DropPartitionMessage dropPtnMsg = md.getDropPartitionMessage(event.getMessage());
    assertEquals(defaultDbName, dropPtnMsg.getDB());
    assertEquals(tblName, dropPtnMsg.getTable());
    Table tableObj = dropPtnMsg.getTableObj();
    assertEquals(table.getDbName(), tableObj.getDbName());
    assertEquals(table.getTableName(), tableObj.getTableName());
    assertEquals(table.getOwner(), tableObj.getOwner());
    assertEquals(TableType.MANAGED_TABLE.toString(), dropPtnMsg.getTableType());
    // Verify the eventID was passed to the non-transactional listener
    MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.DROP_PARTITION, firstEventId + 3);
    MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.ADD_PARTITION, firstEventId + 2);
    MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_TABLE, firstEventId + 1);
    // When hive.metastore.transactional.event.listeners is set,
    // a failed event should not create a new notification
    List<String> newpartCol1Vals = Arrays.asList("tomorrow");
    partition = new Partition(newpartCol1Vals, defaultDbName, tblName, startTime, startTime, sd, emptyParameters);
    msClient.add_partition(partition);
    DummyRawStoreFailEvent.setEventSucceed(false);
    try {
        msClient.dropPartition(defaultDbName, tblName, newpartCol1Vals, false);
        fail("Error: drop partition should've failed");
    } catch (Exception ex) {
    // expected
    }
    rsp = msClient.getNextNotification(firstEventId, 0, null);
    assertEquals(4, rsp.getEventsSize());
    testEventCounts(defaultDbName, firstEventId, null, null, 4);
}

Example 97 with StorageDescriptor

use of org.apache.hadoop.hive.metastore.api.StorageDescriptor in project hive by apache.

the class TestDbNotificationListener method exchangePartition.

@Test
public void exchangePartition() throws Exception {
    String dbName = "default";
    List<FieldSchema> cols = new ArrayList<FieldSchema>();
    cols.add(new FieldSchema("col1", "int", "nocomment"));
    List<FieldSchema> partCols = new ArrayList<FieldSchema>();
    partCols.add(new FieldSchema("part", "int", ""));
    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
    StorageDescriptor sd1 = new StorageDescriptor(cols, Paths.get(testTempDir, "1").toString(), "input", "output", false, 0, serde, null, null, emptyParameters);
    Table tab1 = new Table("tab1", dbName, "me", startTime, startTime, 0, sd1, partCols, emptyParameters, null, null, null);
    msClient.createTable(tab1);
    NotificationEventResponse rsp = msClient.getNextNotification(firstEventId, 0, null);
    // add_table
    assertEquals(1, rsp.getEventsSize());
    StorageDescriptor sd2 = new StorageDescriptor(cols, Paths.get(testTempDir, "2").toString(), "input", "output", false, 0, serde, null, null, emptyParameters);
    Table tab2 = new Table("tab2", dbName, "me", startTime, startTime, 0, sd2, partCols, emptyParameters, null, null, // add_table
    null);
    msClient.createTable(tab2);
    rsp = msClient.getNextNotification(firstEventId + 1, 0, null);
    assertEquals(1, rsp.getEventsSize());
    StorageDescriptor sd1part = new StorageDescriptor(cols, Paths.get(testTempDir, "1", "part=1").toString(), "input", "output", false, 0, serde, null, null, emptyParameters);
    StorageDescriptor sd2part = new StorageDescriptor(cols, Paths.get(testTempDir, "1", "part=2").toString(), "input", "output", false, 0, serde, null, null, emptyParameters);
    StorageDescriptor sd3part = new StorageDescriptor(cols, Paths.get(testTempDir, "1", "part=3").toString(), "input", "output", false, 0, serde, null, null, emptyParameters);
    Partition part1 = new Partition(Arrays.asList("1"), "default", tab1.getTableName(), startTime, startTime, sd1part, emptyParameters);
    Partition part2 = new Partition(Arrays.asList("2"), "default", tab1.getTableName(), startTime, startTime, sd2part, emptyParameters);
    Partition part3 = new Partition(Arrays.asList("3"), "default", tab1.getTableName(), startTime, startTime, sd3part, emptyParameters);
    msClient.add_partitions(Arrays.asList(part1, part2, part3));
    rsp = msClient.getNextNotification(firstEventId + 2, 0, null);
    // add_partition
    assertEquals(1, rsp.getEventsSize());
    msClient.exchange_partition(ImmutableMap.of("part", "1"), dbName, tab1.getTableName(), dbName, tab2.getTableName());
    rsp = msClient.getNextNotification(firstEventId + 3, 0, null);
    assertEquals(2, rsp.getEventsSize());
    NotificationEvent event = rsp.getEvents().get(0);
    assertEquals(firstEventId + 4, event.getEventId());
    assertTrue(event.getEventTime() >= startTime);
    assertEquals(EventType.ADD_PARTITION.toString(), event.getEventType());
    assertEquals(dbName, event.getDbName());
    assertEquals(tab2.getTableName(), event.getTableName());
    // Parse the message field
    AddPartitionMessage addPtnMsg = md.getAddPartitionMessage(event.getMessage());
    assertEquals(dbName, addPtnMsg.getDB());
    assertEquals(tab2.getTableName(), addPtnMsg.getTable());
    Iterator<Partition> ptnIter = addPtnMsg.getPartitionObjs().iterator();
    assertEquals(TableType.MANAGED_TABLE.toString(), addPtnMsg.getTableType());
    assertTrue(ptnIter.hasNext());
    Partition msgPart = ptnIter.next();
    assertEquals(part1.getValues(), msgPart.getValues());
    assertEquals(dbName, msgPart.getDbName());
    assertEquals(tab2.getTableName(), msgPart.getTableName());
    event = rsp.getEvents().get(1);
    assertEquals(firstEventId + 5, event.getEventId());
    assertTrue(event.getEventTime() >= startTime);
    assertEquals(EventType.DROP_PARTITION.toString(), event.getEventType());
    assertEquals(dbName, event.getDbName());
    assertEquals(tab1.getTableName(), event.getTableName());
    // Parse the message field
    DropPartitionMessage dropPtnMsg = md.getDropPartitionMessage(event.getMessage());
    assertEquals(dbName, dropPtnMsg.getDB());
    assertEquals(tab1.getTableName(), dropPtnMsg.getTable());
    assertEquals(TableType.MANAGED_TABLE.toString(), dropPtnMsg.getTableType());
    Iterator<Map<String, String>> parts = dropPtnMsg.getPartitions().iterator();
    assertTrue(parts.hasNext());
    assertEquals(part1.getValues(), Lists.newArrayList(parts.next().values()));
    // Verify the eventID was passed to the non-transactional listener
    MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.DROP_PARTITION, firstEventId + 5);
    MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.ADD_PARTITION, firstEventId + 4);
    MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.ADD_PARTITION, firstEventId + 3);
    MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_TABLE, firstEventId + 2);
    MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_TABLE, firstEventId + 1);
    testEventCounts(dbName, firstEventId, null, null, 5);
}

Example 98 with StorageDescriptor

use of org.apache.hadoop.hive.metastore.api.StorageDescriptor in project hive by apache.

the class TestDbNotificationListener method insertPartition.

@Test
public void insertPartition() throws Exception {
    String defaultDbName = "default";
    String tblName = "insertptn";
    String tblOwner = "me";
    String serdeLocation = testTempDir;
    String fileAdded = "/warehouse/mytable/b1";
    String checksumAdded = "1234";
    FieldSchema col1 = new FieldSchema("col1", "int", "no comment");
    List<FieldSchema> cols = new ArrayList<FieldSchema>();
    cols.add(col1);
    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
    StorageDescriptor sd = new StorageDescriptor(cols, serdeLocation, "input", "output", false, 0, serde, null, null, emptyParameters);
    FieldSchema partCol1 = new FieldSchema("ds", "string", "no comment");
    List<FieldSchema> partCols = new ArrayList<FieldSchema>();
    List<String> partCol1Vals = Arrays.asList("today");
    List<String> partKeyVals = new ArrayList<String>();
    partKeyVals.add("today");
    partCols.add(partCol1);
    Table table = new Table(tblName, defaultDbName, tblOwner, startTime, startTime, 0, sd, partCols, emptyParameters, null, null, null);
    // Event 1
    msClient.createTable(table);
    Partition partition = new Partition(partCol1Vals, defaultDbName, tblName, startTime, startTime, sd, emptyParameters);
    // Event 2
    msClient.add_partition(partition);
    FireEventRequestData data = new FireEventRequestData();
    InsertEventRequestData insertData = new InsertEventRequestData();
    data.setInsertData(insertData);
    insertData.addToFilesAdded(fileAdded);
    insertData.addToFilesAddedChecksum(checksumAdded);
    insertData.setReplace(false);
    FireEventRequest rqst = new FireEventRequest(true, data);
    rqst.setDbName(defaultDbName);
    rqst.setTableName(tblName);
    rqst.setPartitionVals(partCol1Vals);
    // Event 3
    verifyInsertEventReceived(defaultDbName, tblName, Arrays.asList(partKeyVals), rqst, firstEventId + 3, 1);
    // Verify the eventID was passed to the non-transactional listener
    MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.INSERT, firstEventId + 3);
    MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.ADD_PARTITION, firstEventId + 2);
    MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_TABLE, firstEventId + 1);
    testEventCounts(defaultDbName, firstEventId, null, null, 3);
    // fire multiple insert events on partition
    // add some more partitions
    partition = new Partition(Arrays.asList("yesterday"), defaultDbName, tblName, startTime, startTime, sd, emptyParameters);
    // Event 4
    msClient.add_partition(partition);
    partition = new Partition(Arrays.asList("tomorrow"), defaultDbName, tblName, startTime, startTime, sd, emptyParameters);
    // Event 5
    msClient.add_partition(partition);
    data = new FireEventRequestData();
    List<InsertEventRequestData> insertEventRequestDataList = new ArrayList<>();
    data.setInsertDatas(insertEventRequestDataList);
    // event 6
    InsertEventRequestData insertData1 = new InsertEventRequestData();
    insertData1.addToFilesAdded(fileAdded);
    insertData1.addToFilesAddedChecksum(checksumAdded);
    insertData1.setReplace(false);
    insertData1.setPartitionVal(Arrays.asList("yesterday"));
    // event 7
    InsertEventRequestData insertData2 = new InsertEventRequestData();
    insertData2.addToFilesAdded(fileAdded);
    insertData2.addToFilesAddedChecksum(checksumAdded);
    insertData2.setReplace(false);
    insertData2.setPartitionVal(Arrays.asList("today"));
    // event 8
    InsertEventRequestData insertData3 = new InsertEventRequestData();
    insertData3.addToFilesAdded(fileAdded);
    insertData3.addToFilesAddedChecksum(checksumAdded);
    insertData3.setReplace(false);
    insertData3.setPartitionVal(Arrays.asList("tomorrow"));
    // fire insert event on 3 partitions
    insertEventRequestDataList.add(insertData1);
    insertEventRequestDataList.add(insertData2);
    insertEventRequestDataList.add(insertData3);
    rqst = new FireEventRequest(true, data);
    rqst.setDbName(defaultDbName);
    rqst.setTableName(tblName);
    verifyInsertEventReceived(defaultDbName, tblName, Arrays.asList(Arrays.asList("yesterday"), Arrays.asList("today"), Arrays.asList("tomorrow")), rqst, firstEventId + 6, 3);
    // negative test. partition values must be set when firing bulk insert events
    data.getInsertDatas().get(1).unsetPartitionVal();
    boolean threwException = false;
    try {
        FireEventResponse response = msClient.fireListenerEvent(rqst);
    } catch (MetaException ex) {
        threwException = true;
        Assert.assertTrue(ex instanceof MetaException);
        Assert.assertTrue(ex.getMessage().contains("Partition values must be set when firing multiple insert events"));
    }
    Assert.assertTrue("bulk insert event API didn't " + "throw exception when partition values were not set", threwException);
}

Example 99 with StorageDescriptor

use of org.apache.hadoop.hive.metastore.api.StorageDescriptor in project hive by apache.

the class TestDbNotificationListener method addPartition.

@Test
public void addPartition() throws Exception {
    String defaultDbName = "default";
    String tblName = "addptn";
    String tblName2 = "addptn2";
    String tblOwner = "me";
    String serdeLocation = testTempDir;
    FieldSchema col1 = new FieldSchema("col1", "int", "no comment");
    List<FieldSchema> cols = new ArrayList<FieldSchema>();
    cols.add(col1);
    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
    StorageDescriptor sd = new StorageDescriptor(cols, serdeLocation, "input", "output", false, 0, serde, null, null, emptyParameters);
    FieldSchema partCol1 = new FieldSchema("ds", "string", "no comment");
    List<FieldSchema> partCols = new ArrayList<FieldSchema>();
    List<String> partCol1Vals = Arrays.asList("today");
    partCols.add(partCol1);
    Table table = new Table(tblName, defaultDbName, tblOwner, startTime, startTime, 0, sd, partCols, emptyParameters, null, null, null);
    // Event 1
    msClient.createTable(table);
    Partition partition = new Partition(partCol1Vals, defaultDbName, tblName, startTime, startTime, sd, emptyParameters);
    // Event 2
    msClient.add_partition(partition);
    // Get notifications from metastore
    NotificationEventResponse rsp = msClient.getNextNotification(firstEventId, 0, null);
    assertEquals(2, rsp.getEventsSize());
    NotificationEvent event = rsp.getEvents().get(1);
    assertEquals(firstEventId + 2, event.getEventId());
    assertTrue(event.getEventTime() >= startTime);
    assertEquals(EventType.ADD_PARTITION.toString(), event.getEventType());
    assertEquals(defaultDbName, event.getDbName());
    assertEquals(tblName, event.getTableName());
    // Parse the message field
    AddPartitionMessage addPtnMsg = md.getAddPartitionMessage(event.getMessage());
    assertEquals(defaultDbName, addPtnMsg.getDB());
    assertEquals(tblName, addPtnMsg.getTable());
    Iterator<Partition> ptnIter = addPtnMsg.getPartitionObjs().iterator();
    assertTrue(ptnIter.hasNext());
    assertEquals(partition, ptnIter.next());
    assertEquals(TableType.MANAGED_TABLE.toString(), addPtnMsg.getTableType());
    // Verify the eventID was passed to the non-transactional listener
    MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.ADD_PARTITION, firstEventId + 2);
    MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_TABLE, firstEventId + 1);
    // When hive.metastore.transactional.event.listeners is set,
    // a failed event should not create a new notification
    partition = new Partition(Arrays.asList("tomorrow"), defaultDbName, tblName2, startTime, startTime, sd, emptyParameters);
    DummyRawStoreFailEvent.setEventSucceed(false);
    try {
        msClient.add_partition(partition);
        fail("Error: add partition should've failed");
    } catch (Exception ex) {
    // expected
    }
    rsp = msClient.getNextNotification(firstEventId, 0, null);
    assertEquals(2, rsp.getEventsSize());
    testEventCounts(defaultDbName, firstEventId, null, null, 2);
}

Example 100 with StorageDescriptor

use of org.apache.hadoop.hive.metastore.api.StorageDescriptor in project hive by apache.

the class Cleaner method clean.

private void clean(CompactionInfo ci, long minOpenTxnGLB, boolean metricsEnabled) throws MetaException {
    LOG.info("Starting cleaning for " + ci);
    PerfLogger perfLogger = PerfLogger.getPerfLogger(false);
    String cleanerMetric = MetricsConstants.COMPACTION_CLEANER_CYCLE + "_" + (ci.type != null ? ci.type.toString().toLowerCase() : null);
    try {
        if (metricsEnabled) {
            perfLogger.perfLogBegin(CLASS_NAME, cleanerMetric);
        }
        Optional<String> location = Optional.ofNullable(ci.properties).map(StringableMap::new).map(config -> config.get("location"));
        Callable<Boolean> cleanUpTask;
        Table t = null;
        Partition p = resolvePartition(ci);
        if (!location.isPresent()) {
            t = resolveTable(ci);
            if (t == null) {
                // The table was dropped before we got around to cleaning it.
                LOG.info("Unable to find table " + ci.getFullTableName() + ", assuming it was dropped." + idWatermark(ci));
                txnHandler.markCleaned(ci);
                return;
            }
            if (MetaStoreUtils.isNoCleanUpSet(t.getParameters())) {
                // The table was marked no clean up true.
                LOG.info("Skipping table " + ci.getFullTableName() + " clean up, as NO_CLEANUP set to true");
                txnHandler.markCleaned(ci);
                return;
            }
            if (ci.partName != null) {
                if (p == null) {
                    // The partition was dropped before we got around to cleaning it.
                    LOG.info("Unable to find partition " + ci.getFullPartitionName() + ", assuming it was dropped." + idWatermark(ci));
                    txnHandler.markCleaned(ci);
                    return;
                }
                if (MetaStoreUtils.isNoCleanUpSet(p.getParameters())) {
                    // The partition was marked no clean up true.
                    LOG.info("Skipping partition " + ci.getFullPartitionName() + " clean up, as NO_CLEANUP set to true");
                    txnHandler.markCleaned(ci);
                    return;
                }
            }
        }
        txnHandler.markCleanerStart(ci);
        if (t != null) {
            StorageDescriptor sd = resolveStorageDescriptor(t, p);
            cleanUpTask = () -> removeFiles(location.orElse(sd.getLocation()), minOpenTxnGLB, ci, ci.partName != null && p == null);
        } else {
            cleanUpTask = () -> removeFiles(location.get(), ci);
        }
        Ref<Boolean> removedFiles = Ref.from(false);
        if (runJobAsSelf(ci.runAs)) {
            removedFiles.value = cleanUpTask.call();
        } else {
            LOG.info("Cleaning as user " + ci.runAs + " for " + ci.getFullPartitionName());
            UserGroupInformation ugi = UserGroupInformation.createProxyUser(ci.runAs, UserGroupInformation.getLoginUser());
            ugi.doAs((PrivilegedExceptionAction<Void>) () -> {
                removedFiles.value = cleanUpTask.call();
                return null;
            });
            try {
                FileSystem.closeAllForUGI(ugi);
            } catch (IOException exception) {
                LOG.error("Could not clean up file-system handles for UGI: " + ugi + " for " + ci.getFullPartitionName() + idWatermark(ci), exception);
            }
        }
        if (removedFiles.value || isDynPartAbort(t, ci)) {
            txnHandler.markCleaned(ci);
        } else {
            txnHandler.clearCleanerStart(ci);
            LOG.warn("No files were removed. Leaving queue entry " + ci + " in ready for cleaning state.");
        }
    } catch (Exception e) {
        LOG.error("Caught exception when cleaning, unable to complete cleaning of " + ci + " " + StringUtils.stringifyException(e));
        ci.errorMessage = e.getMessage();
        if (metricsEnabled) {
            Metrics.getOrCreateCounter(MetricsConstants.COMPACTION_CLEANER_FAILURE_COUNTER).inc();
        }
        txnHandler.markFailed(ci);
    } finally {
        if (metricsEnabled) {
            perfLogger.perfLogEnd(CLASS_NAME, cleanerMetric);
        }
    }
}