Examples with ShowCompactResponseElement org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement used on opensource projects

Example 96 with ShowCompactResponseElement

use of org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement in project hive by apache.

the class TestCompactionMetricData method aCompaction.

private static ShowCompactResponseElement aCompaction(long id, String initiatorId, String initiatorVersion, String workerId, String workerVersion) {
    ShowCompactResponseElement e = new ShowCompactResponseElement("db_name", UUID.randomUUID().toString(), CompactionType.MAJOR, INITIATED);
    e.setId(id);
    e.setInitiatorId(initiatorId);
    e.setInitiatorVersion(initiatorVersion);
    e.setWorkerid(workerId);
    e.setWorkerVersion(workerVersion);
    return e;
}

Example 97 with ShowCompactResponseElement

use of org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement in project hive by apache.

the class TestCompactionMetricData method aCompaction.

private static ShowCompactResponseElement aCompaction(String workerVersion, String state, Long enqueuedTime, Long startTime, Long endTime) {
    ShowCompactResponseElement e = new ShowCompactResponseElement("db_name", "table_name", CompactionType.MINOR, state);
    e.setWorkerVersion(workerVersion);
    if (enqueuedTime != null) {
        e.setEnqueueTime(enqueuedTime);
    }
    if (startTime != null) {
        e.setStart(startTime);
    }
    if (endTime != null) {
        e.setEndTime(endTime);
    }
    return e;
}

Example 98 with ShowCompactResponseElement

use of org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement in project hive by apache.

the class TxnHandler method showCompact.

@RetrySemantics.ReadOnly
public ShowCompactResponse showCompact(ShowCompactRequest rqst) throws MetaException {
    ShowCompactResponse response = new ShowCompactResponse(new ArrayList<>());
    Connection dbConn = null;
    Statement stmt = null;
    try {
        try {
            dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
            stmt = dbConn.createStatement();
            String s = "" + // -1 because 'null' literal doesn't work for all DBs...
            "SELECT " + "  \"CQ_DATABASE\", \"CQ_TABLE\", \"CQ_PARTITION\", \"CQ_STATE\", \"CQ_TYPE\", \"CQ_WORKER_ID\", " + "  \"CQ_START\", -1 \"CC_END\", \"CQ_RUN_AS\", \"CQ_HADOOP_JOB_ID\", \"CQ_ID\", \"CQ_ERROR_MESSAGE\", " + "  \"CQ_ENQUEUE_TIME\", \"CQ_WORKER_VERSION\", \"CQ_INITIATOR_ID\", \"CQ_INITIATOR_VERSION\", " + "  \"CQ_CLEANER_START\"" + "FROM " + "  \"COMPACTION_QUEUE\" " + "UNION ALL " + "SELECT " + "  \"CC_DATABASE\", \"CC_TABLE\", \"CC_PARTITION\", \"CC_STATE\", \"CC_TYPE\", \"CC_WORKER_ID\", " + "  \"CC_START\", \"CC_END\", \"CC_RUN_AS\", \"CC_HADOOP_JOB_ID\", \"CC_ID\", \"CC_ERROR_MESSAGE\", " + "  \"CC_ENQUEUE_TIME\", \"CC_WORKER_VERSION\", \"CC_INITIATOR_ID\", \"CC_INITIATOR_VERSION\", " + "  -1 " + "FROM " + // todo: sort by cq_id?
            "  \"COMPLETED_COMPACTIONS\"";
            // what I want is order by cc_end desc, cc_start asc (but derby has a bug https://issues.apache.org/jira/browse/DERBY-6013)
            // to sort so that currently running jobs are at the end of the list (bottom of screen)
            // and currently running ones are in sorted by start time
            // w/o order by likely currently running compactions will be first (LHS of Union)
            LOG.debug("Going to execute query <" + s + ">");
            ResultSet rs = stmt.executeQuery(s);
            while (rs.next()) {
                ShowCompactResponseElement e = new ShowCompactResponseElement();
                e.setDbname(rs.getString(1));
                e.setTablename(rs.getString(2));
                e.setPartitionname(rs.getString(3));
                e.setState(compactorStateToResponse(rs.getString(4).charAt(0)));
                try {
                    e.setType(dbCompactionType2ThriftType(rs.getString(5).charAt(0)));
                } catch (MetaException ex) {
                // do nothing to handle RU/D if we add another status
                }
                e.setWorkerid(rs.getString(6));
                long start = rs.getLong(7);
                if (!rs.wasNull()) {
                    e.setStart(start);
                }
                long endTime = rs.getLong(8);
                if (endTime != -1) {
                    e.setEndTime(endTime);
                }
                e.setRunAs(rs.getString(9));
                e.setHadoopJobId(rs.getString(10));
                e.setId(rs.getLong(11));
                e.setErrorMessage(rs.getString(12));
                long enqueueTime = rs.getLong(13);
                if (!rs.wasNull()) {
                    e.setEnqueueTime(enqueueTime);
                }
                e.setWorkerVersion(rs.getString(14));
                e.setInitiatorId(rs.getString(15));
                e.setInitiatorVersion(rs.getString(16));
                long cleanerStart = rs.getLong(17);
                if (!rs.wasNull() && (cleanerStart != -1)) {
                    e.setCleanerStart(cleanerStart);
                }
                response.addToCompacts(e);
            }
        } catch (SQLException e) {
            checkRetryable(e, "showCompact(" + rqst + ")");
            throw new MetaException("Unable to select from transaction database " + StringUtils.stringifyException(e));
        } finally {
            closeStmt(stmt);
            closeDbConn(dbConn);
        }
        return response;
    } catch (RetryException e) {
        return showCompact(rqst);
    }
}

Example 99 with ShowCompactResponseElement

use of org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement in project hive by apache.

the class CompactionMetricData method init.

private void init() {
    final Map<String, ShowCompactResponseElement> lastElements = new HashMap<>();
    oldestEnqueueTime = OLDEST_TIME_NO_VALUE;
    oldestWorkingTime = OLDEST_TIME_NO_VALUE;
    oldestCleaningTime = OLDEST_TIME_NO_VALUE;
    for (ShowCompactResponseElement element : compacts) {
        final String key = element.getDbname() + "/" + element.getTablename() + (element.getPartitionname() != null ? "/" + element.getPartitionname() : "");
        // If new key, add the element, if there is an existing one, change to the element if the element.id is greater than old.id
        lastElements.compute(key, (k, old) -> (old == null) ? element : (element.getId() > old.getId() ? element : old));
        // find the oldest elements with initiated and working states
        String state = element.getState();
        if (TxnStore.INITIATED_RESPONSE.equals(state) && (oldestEnqueueTime > element.getEnqueueTime())) {
            oldestEnqueueTime = element.getEnqueueTime();
        }
        if (element.isSetStart()) {
            if (TxnStore.WORKING_RESPONSE.equals(state) && (oldestWorkingTime > element.getStart())) {
                oldestWorkingTime = element.getStart();
            }
        }
        if (element.isSetCleanerStart()) {
            if (TxnStore.CLEANING_RESPONSE.equals(state) && (oldestCleaningTime > element.getCleanerStart())) {
                oldestCleaningTime = element.getCleanerStart();
            }
        }
    }
    stateCount = lastElements.values().stream().collect(Collectors.groupingBy(ShowCompactResponseElement::getState, Collectors.counting()));
    failedCompactionPercentage = calculateFailedPercentage(stateCount);
    initiatorsCount = lastElements.values().stream().filter(e -> !MANUALLY_INITIATED_COMPACTION.equals(getThreadIdFromId(e.getInitiatorId()))).map(e -> getHostFromId(e.getInitiatorId())).filter(e -> !NO_VAL.equals(e)).distinct().count();
    initiatorVersionsCount = lastElements.values().stream().map(ShowCompactResponseElement::getInitiatorVersion).filter(Objects::nonNull).distinct().count();
    workersCount = lastElements.values().stream().map(e -> getHostFromId(e.getWorkerid())).filter(e -> !NO_VAL.equals(e)).distinct().count();
    workerVersionsCount = lastElements.values().stream().map(ShowCompactResponseElement::getWorkerVersion).filter(Objects::nonNull).distinct().count();
}

Example 100 with ShowCompactResponseElement

use of org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement in project hive by apache.

the class TestPreUpgradeTool method testUpgrade.

/**
 * preUpgrade: test tables that need to be compacted, waits for compaction
 * postUpgrade: generates scripts w/o asserts
 */
@Test
public void testUpgrade() throws Exception {
    int[][] data = { { 1, 2 }, { 3, 4 }, { 5, 6 } };
    int[][] dataPart = { { 1, 2, 10 }, { 3, 4, 11 }, { 5, 6, 12 } };
    runStatementOnDriver("drop table if exists TAcid");
    runStatementOnDriver("drop table if exists TAcidPart");
    runStatementOnDriver("drop table if exists TFlat");
    runStatementOnDriver("drop table if exists TFlatText");
    try {
        runStatementOnDriver("create table TAcid (a int, b int) clustered by (b) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true')");
        runStatementOnDriver("create table TAcidPart (a int, b int) partitioned by (p tinyint)  clustered by (b) into 2 buckets  stored" + " as orc TBLPROPERTIES ('transactional'='true')");
        // on 2.x these are guaranteed to not be acid
        runStatementOnDriver("create table TFlat (a int, b int) stored as orc tblproperties('transactional'='false')");
        runStatementOnDriver("create table TFlatText (a int, b int) stored as textfile tblproperties('transactional'='false')");
        // this needs major compaction
        runStatementOnDriver("insert into TAcid" + makeValuesClause(data));
        runStatementOnDriver("update TAcid set a = 1 where b = 2");
        // this table needs to be converted to CRUD Acid
        runStatementOnDriver("insert into TFlat" + makeValuesClause(data));
        // this table needs to be converted to MM
        runStatementOnDriver("insert into TFlatText" + makeValuesClause(data));
        // p=10 needs major compaction
        runStatementOnDriver("insert into TAcidPart partition(p)" + makeValuesClause(dataPart));
        runStatementOnDriver("update TAcidPart set a = 1 where b = 2 and p = 10");
        // todo: add partitioned table that needs conversion to MM/Acid
        // todo: rename files case
        String[] args = { "-location", getTestDataDir(), "-execute" };
        PreUpgradeTool.callback = new PreUpgradeTool.Callback() {

            @Override
            void onWaitForCompaction() throws MetaException {
                runWorker(hiveConf);
            }
        };
        PreUpgradeTool.pollIntervalMs = 1;
        PreUpgradeTool.hiveConf = hiveConf;
        PreUpgradeTool.main(args);
        String[] scriptFiles = getScriptFiles();
        assertThat(scriptFiles.length, is(1));
        List<String> scriptContent = loadScriptContent(new File(getTestDataDir(), scriptFiles[0]));
        assertThat(scriptContent.size(), is(2));
        assertThat(scriptContent, hasItem(is("ALTER TABLE default.tacid COMPACT 'major';")));
        assertThat(scriptContent, hasItem(is("ALTER TABLE default.tacidpart PARTITION(p=10Y) COMPACT 'major';")));
        TxnStore txnHandler = TxnUtils.getTxnStore(hiveConf);
        ShowCompactResponse resp = txnHandler.showCompact(new ShowCompactRequest());
        Assert.assertEquals(2, resp.getCompactsSize());
        for (ShowCompactResponseElement e : resp.getCompacts()) {
            Assert.assertEquals(e.toString(), TxnStore.CLEANING_RESPONSE, e.getState());
        }
        // Check whether compaction was successful in the first run
        File secondRunDataDir = new File(getTestDataDir(), "secondRun");
        if (!secondRunDataDir.exists()) {
            if (!secondRunDataDir.mkdir()) {
                throw new IOException("Unable to create directory" + secondRunDataDir.getAbsolutePath());
            }
        }
        String[] args2 = { "-location", secondRunDataDir.getAbsolutePath() };
        PreUpgradeTool.main(args2);
        scriptFiles = secondRunDataDir.list();
        assertThat(scriptFiles, is(not(nullValue())));
        assertThat(scriptFiles.length, is(0));
    } finally {
        runStatementOnDriver("drop table if exists TAcid");
        runStatementOnDriver("drop table if exists TAcidPart");
        runStatementOnDriver("drop table if exists TFlat");
        runStatementOnDriver("drop table if exists TFlatText");
    }
}