Examples with Table org.apache.hadoop.hive.metastore.api.Table used on opensource projects

Example 96 with Table

use of org.apache.hadoop.hive.metastore.api.Table in project hive by apache.

the class TestReplicationTask method testCreate.

@Test
public static void testCreate() throws HCatException {
    Table t = new Table();
    t.setDbName("testdb");
    t.setTableName("testtable");
    NotificationEvent event = new NotificationEvent(0, (int) System.currentTimeMillis(), HCatConstants.HCAT_CREATE_TABLE_EVENT, msgFactory.buildCreateTableMessage(t).toString());
    event.setDbName(t.getDbName());
    event.setTableName(t.getTableName());
    ReplicationTask.resetFactory(null);
    ReplicationTask rtask = ReplicationTask.create(HCatClient.create(new HiveConf()), new HCatNotificationEvent(event));
    assertTrue("Provided factory instantiation should yield CreateTableReplicationTask", rtask instanceof CreateTableReplicationTask);
    ReplicationTask.resetFactory(NoopFactory.class);
    rtask = ReplicationTask.create(HCatClient.create(new HiveConf()), new HCatNotificationEvent(event));
    assertTrue("Provided factory instantiation should yield NoopReplicationTask", rtask instanceof NoopReplicationTask);
    ReplicationTask.resetFactory(null);
}

Example 97 with Table

use of org.apache.hadoop.hive.metastore.api.Table in project hive by apache.

the class TestEximReplicationTasks method testCreateTable.

@Test
public void testCreateTable() throws IOException {
    Table t = new Table();
    t.setDbName("testdb");
    t.setTableName("testtable");
    NotificationEvent event = new NotificationEvent(getEventId(), getTime(), HCatConstants.HCAT_CREATE_TABLE_EVENT, msgFactory.buildCreateTableMessage(t).toString());
    event.setDbName(t.getDbName());
    event.setTableName(t.getTableName());
    HCatNotificationEvent hev = new HCatNotificationEvent(event);
    ReplicationTask rtask = ReplicationTask.create(client, hev);
    assertEquals(hev.toString(), rtask.getEvent().toString());
    verifyCreateTableReplicationTask(rtask);
}

Example 98 with Table

use of org.apache.hadoop.hive.metastore.api.Table in project hive by apache.

the class TestEximReplicationTasks method testInsert.

@Test
public void testInsert() throws HCatException {
    Table t = new Table();
    t.setDbName("testdb");
    t.setTableName("testtable");
    List<FieldSchema> pkeys = HCatSchemaUtils.getFieldSchemas(HCatSchemaUtils.getHCatSchema("a:int,b:string").getFields());
    t.setPartitionKeys(pkeys);
    Partition p = createPtn(t, Arrays.asList("102", "lmn"));
    List<String> files = Arrays.asList("/tmp/test123");
    NotificationEvent event = new NotificationEvent(getEventId(), getTime(), HCatConstants.HCAT_INSERT_EVENT, msgFactory.buildInsertMessage(t.getDbName(), t.getTableName(), getPtnDesc(t, p), files).toString());
    event.setDbName(t.getDbName());
    event.setTableName(t.getTableName());
    HCatNotificationEvent hev = new HCatNotificationEvent(event);
    ReplicationTask rtask = ReplicationTask.create(client, hev);
    assertEquals(hev.toString(), rtask.getEvent().toString());
    verifyInsertReplicationTask(rtask, t, p);
}

Example 99 with Table

use of org.apache.hadoop.hive.metastore.api.Table in project hive by apache.

the class HiveAlterHandler method alterPartitions.

@Override
public List<Partition> alterPartitions(final RawStore msdb, Warehouse wh, final String dbname, final String name, final List<Partition> new_parts, EnvironmentContext environmentContext, HMSHandler handler) throws InvalidOperationException, InvalidObjectException, AlreadyExistsException, MetaException {
    List<Partition> oldParts = new ArrayList<Partition>();
    List<List<String>> partValsList = new ArrayList<List<String>>();
    List<MetaStoreEventListener> transactionalListeners = null;
    if (handler != null) {
        transactionalListeners = handler.getTransactionalListeners();
    }
    Table tbl = msdb.getTable(dbname, name);
    if (tbl == null) {
        throw new InvalidObjectException("Unable to alter partitions because table or database does not exist.");
    }
    boolean success = false;
    try {
        msdb.openTransaction();
        for (Partition tmpPart : new_parts) {
            // Set DDL time to now if not specified
            if (tmpPart.getParameters() == null || tmpPart.getParameters().get(hive_metastoreConstants.DDL_TIME) == null || Integer.parseInt(tmpPart.getParameters().get(hive_metastoreConstants.DDL_TIME)) == 0) {
                tmpPart.putToParameters(hive_metastoreConstants.DDL_TIME, Long.toString(System.currentTimeMillis() / 1000));
            }
            Partition oldTmpPart = msdb.getPartition(dbname, name, tmpPart.getValues());
            oldParts.add(oldTmpPart);
            partValsList.add(tmpPart.getValues());
            if (MetaStoreUtils.requireCalStats(hiveConf, oldTmpPart, tmpPart, tbl, environmentContext)) {
                // Check if stats are same, no need to update
                if (MetaStoreUtils.isFastStatsSame(oldTmpPart, tmpPart)) {
                    MetaStoreUtils.updateBasicState(environmentContext, tmpPart.getParameters());
                } else {
                    MetaStoreUtils.updatePartitionStatsFast(tmpPart, wh, false, true, environmentContext);
                }
            }
            updatePartColumnStats(msdb, dbname, name, oldTmpPart.getValues(), tmpPart);
        }
        msdb.alterPartitions(dbname, name, partValsList, new_parts);
        Iterator<Partition> oldPartsIt = oldParts.iterator();
        for (Partition newPart : new_parts) {
            Partition oldPart;
            if (oldPartsIt.hasNext()) {
                oldPart = oldPartsIt.next();
            } else {
                throw new InvalidOperationException("Missing old partition corresponding to new partition " + "when invoking MetaStoreEventListener for alterPartitions event.");
            }
            if (transactionalListeners != null && transactionalListeners.size() > 0) {
                AlterPartitionEvent alterPartitionEvent = new AlterPartitionEvent(oldPart, newPart, tbl, true, handler);
                for (MetaStoreEventListener transactionalListener : transactionalListeners) {
                    transactionalListener.onAlterPartition(alterPartitionEvent);
                }
            }
        }
        success = msdb.commitTransaction();
    } catch (InvalidObjectException | NoSuchObjectException e) {
        throw new InvalidOperationException("Alter partition operation failed: " + e);
    } finally {
        if (!success) {
            msdb.rollbackTransaction();
        }
    }
    return oldParts;
}

Example 100 with Table

use of org.apache.hadoop.hive.metastore.api.Table in project hive by apache.

the class HiveAlterHandler method alterPartition.

@Override
public Partition alterPartition(final RawStore msdb, Warehouse wh, final String dbname, final String name, final List<String> part_vals, final Partition new_part, EnvironmentContext environmentContext, HMSHandler handler) throws InvalidOperationException, InvalidObjectException, AlreadyExistsException, MetaException {
    boolean success = false;
    Path srcPath = null;
    Path destPath = null;
    FileSystem srcFs = null;
    FileSystem destFs;
    Partition oldPart = null;
    String oldPartLoc = null;
    String newPartLoc = null;
    List<MetaStoreEventListener> transactionalListeners = null;
    if (handler != null) {
        transactionalListeners = handler.getTransactionalListeners();
    }
    // Set DDL time to now if not specified
    if (new_part.getParameters() == null || new_part.getParameters().get(hive_metastoreConstants.DDL_TIME) == null || Integer.parseInt(new_part.getParameters().get(hive_metastoreConstants.DDL_TIME)) == 0) {
        new_part.putToParameters(hive_metastoreConstants.DDL_TIME, Long.toString(System.currentTimeMillis() / 1000));
    }
    Table tbl = msdb.getTable(dbname, name);
    if (tbl == null) {
        throw new InvalidObjectException("Unable to alter partition because table or database does not exist.");
    }
    //alter partition
    if (part_vals == null || part_vals.size() == 0) {
        try {
            msdb.openTransaction();
            oldPart = msdb.getPartition(dbname, name, new_part.getValues());
            if (MetaStoreUtils.requireCalStats(hiveConf, oldPart, new_part, tbl, environmentContext)) {
                // if stats are same, no need to update
                if (MetaStoreUtils.isFastStatsSame(oldPart, new_part)) {
                    MetaStoreUtils.updateBasicState(environmentContext, new_part.getParameters());
                } else {
                    MetaStoreUtils.updatePartitionStatsFast(new_part, wh, false, true, environmentContext);
                }
            }
            updatePartColumnStats(msdb, dbname, name, new_part.getValues(), new_part);
            msdb.alterPartition(dbname, name, new_part.getValues(), new_part);
            if (transactionalListeners != null && transactionalListeners.size() > 0) {
                AlterPartitionEvent alterPartitionEvent = new AlterPartitionEvent(oldPart, new_part, tbl, true, handler);
                alterPartitionEvent.setEnvironmentContext(environmentContext);
                for (MetaStoreEventListener transactionalListener : transactionalListeners) {
                    transactionalListener.onAlterPartition(alterPartitionEvent);
                }
            }
            success = msdb.commitTransaction();
        } catch (InvalidObjectException e) {
            throw new InvalidOperationException("alter is not possible");
        } catch (NoSuchObjectException e) {
            //old partition does not exist
            throw new InvalidOperationException("alter is not possible");
        } finally {
            if (!success) {
                msdb.rollbackTransaction();
            }
        }
        return oldPart;
    }
    //rename partition
    try {
        msdb.openTransaction();
        try {
            oldPart = msdb.getPartition(dbname, name, part_vals);
        } catch (NoSuchObjectException e) {
            // this means there is no existing partition
            throw new InvalidObjectException("Unable to rename partition because old partition does not exist");
        }
        Partition check_part;
        try {
            check_part = msdb.getPartition(dbname, name, new_part.getValues());
        } catch (NoSuchObjectException e) {
            // this means there is no existing partition
            check_part = null;
        }
        if (check_part != null) {
            throw new AlreadyExistsException("Partition already exists:" + dbname + "." + name + "." + new_part.getValues());
        }
        // if the external partition is renamed, the file should not change
        if (tbl.getTableType().equals(TableType.EXTERNAL_TABLE.toString())) {
            new_part.getSd().setLocation(oldPart.getSd().getLocation());
            String oldPartName = Warehouse.makePartName(tbl.getPartitionKeys(), oldPart.getValues());
            try {
                //existing partition column stats is no longer valid, remove
                msdb.deletePartitionColumnStatistics(dbname, name, oldPartName, oldPart.getValues(), null);
            } catch (NoSuchObjectException nsoe) {
            //ignore
            } catch (InvalidInputException iie) {
                throw new InvalidOperationException("Unable to update partition stats in table rename." + iie);
            }
            msdb.alterPartition(dbname, name, part_vals, new_part);
        } else {
            try {
                destPath = new Path(wh.getTablePath(msdb.getDatabase(dbname), name), Warehouse.makePartName(tbl.getPartitionKeys(), new_part.getValues()));
                destPath = constructRenamedPath(destPath, new Path(new_part.getSd().getLocation()));
            } catch (NoSuchObjectException e) {
                LOG.debug("Didn't find object in metastore ", e);
                throw new InvalidOperationException("Unable to change partition or table. Database " + dbname + " does not exist" + " Check metastore logs for detailed stack." + e.getMessage());
            }
            if (destPath != null) {
                newPartLoc = destPath.toString();
                oldPartLoc = oldPart.getSd().getLocation();
                LOG.info("srcPath:" + oldPartLoc);
                LOG.info("descPath:" + newPartLoc);
                srcPath = new Path(oldPartLoc);
                srcFs = wh.getFs(srcPath);
                destFs = wh.getFs(destPath);
                // check that src and dest are on the same file system
                if (!FileUtils.equalsFileSystem(srcFs, destFs)) {
                    throw new InvalidOperationException("New table location " + destPath + " is on a different file system than the old location " + srcPath + ". This operation is not supported.");
                }
                try {
                    srcFs.exists(srcPath);
                    if (newPartLoc.compareTo(oldPartLoc) != 0 && destFs.exists(destPath)) {
                        throw new InvalidOperationException("New location for this table " + tbl.getDbName() + "." + tbl.getTableName() + " already exists : " + destPath);
                    }
                } catch (IOException e) {
                    throw new InvalidOperationException("Unable to access new location " + destPath + " for partition " + tbl.getDbName() + "." + tbl.getTableName() + " " + new_part.getValues());
                }
                new_part.getSd().setLocation(newPartLoc);
                if (MetaStoreUtils.requireCalStats(hiveConf, oldPart, new_part, tbl, environmentContext)) {
                    MetaStoreUtils.updatePartitionStatsFast(new_part, wh, false, true, environmentContext);
                }
                String oldPartName = Warehouse.makePartName(tbl.getPartitionKeys(), oldPart.getValues());
                try {
                    //existing partition column stats is no longer valid, remove
                    msdb.deletePartitionColumnStatistics(dbname, name, oldPartName, oldPart.getValues(), null);
                } catch (NoSuchObjectException nsoe) {
                //ignore
                } catch (InvalidInputException iie) {
                    throw new InvalidOperationException("Unable to update partition stats in table rename." + iie);
                }
                msdb.alterPartition(dbname, name, part_vals, new_part);
            }
        }
        if (transactionalListeners != null && transactionalListeners.size() > 0) {
            AlterPartitionEvent alterPartitionEvent = new AlterPartitionEvent(oldPart, new_part, tbl, true, handler);
            alterPartitionEvent.setEnvironmentContext(environmentContext);
            for (MetaStoreEventListener transactionalListener : transactionalListeners) {
                transactionalListener.onAlterPartition(alterPartitionEvent);
            }
        }
        success = msdb.commitTransaction();
    } finally {
        if (!success) {
            msdb.rollbackTransaction();
        }
        if (success && newPartLoc != null && newPartLoc.compareTo(oldPartLoc) != 0) {
            //rename the data directory
            try {
                if (srcFs.exists(srcPath)) {
                    //if destPath's parent path doesn't exist, we should mkdir it
                    Path destParentPath = destPath.getParent();
                    if (!wh.mkdirs(destParentPath, true)) {
                        throw new IOException("Unable to create path " + destParentPath);
                    }
                    wh.renameDir(srcPath, destPath, true);
                    LOG.info("Partition directory rename from " + srcPath + " to " + destPath + " done.");
                }
            } catch (IOException ex) {
                LOG.error("Cannot rename partition directory from " + srcPath + " to " + destPath, ex);
                boolean revertMetaDataTransaction = false;
                try {
                    msdb.openTransaction();
                    msdb.alterPartition(dbname, name, new_part.getValues(), oldPart);
                    if (transactionalListeners != null && transactionalListeners.size() > 0) {
                        AlterPartitionEvent alterPartitionEvent = new AlterPartitionEvent(new_part, oldPart, tbl, true, handler);
                        alterPartitionEvent.setEnvironmentContext(environmentContext);
                        for (MetaStoreEventListener transactionalListener : transactionalListeners) {
                            transactionalListener.onAlterPartition(alterPartitionEvent);
                        }
                    }
                    revertMetaDataTransaction = msdb.commitTransaction();
                } catch (Exception ex2) {
                    LOG.error("Attempt to revert partition metadata change failed. The revert was attempted " + "because associated filesystem rename operation failed with exception " + ex.getMessage(), ex2);
                    if (!revertMetaDataTransaction) {
                        msdb.rollbackTransaction();
                    }
                }
                throw new InvalidOperationException("Unable to access old location " + srcPath + " for partition " + tbl.getDbName() + "." + tbl.getTableName() + " " + part_vals);
            }
        }
    }
    return oldPart;
}