Examples with HiveTable org.apache.gobblin.hive.HiveTable used on opensource projects

Example 1 with HiveTable

use of org.apache.gobblin.hive.HiveTable in project incubator-gobblin by apache.

the class HiveMetaStoreBasedRegister method ensureHiveTableExistenceBeforeAlternation.

/**
 * If table existed on Hive side will return false;
 * Or will create the table thru. RPC and return retVal from remote MetaStore.
 */
private boolean ensureHiveTableExistenceBeforeAlternation(String tableName, String dbName, IMetaStoreClient client, Table table, HiveSpec spec) throws TException {
    try (AutoCloseableLock lock = this.locks.getTableLock(dbName, tableName)) {
        try {
            try (Timer.Context context = this.metricContext.timer(CREATE_HIVE_TABLE).time()) {
                client.createTable(getTableWithCreateTimeNow(table));
                log.info(String.format("Created Hive table %s in db %s", tableName, dbName));
                return true;
            } catch (AlreadyExistsException e) {
            }
        } catch (TException e) {
            log.error(String.format("Unable to create Hive table %s in db %s: " + e.getMessage(), tableName, dbName), e);
            throw e;
        }
        log.info("Table {} already exists in db {}.", tableName, dbName);
        try {
            HiveTable existingTable;
            try (Timer.Context context = this.metricContext.timer(GET_HIVE_TABLE).time()) {
                existingTable = HiveMetaStoreUtils.getHiveTable(client.getTable(dbName, tableName));
            }
            if (needToUpdateTable(existingTable, spec.getTable())) {
                try (Timer.Context context = this.metricContext.timer(ALTER_TABLE).time()) {
                    client.alter_table(dbName, tableName, getTableWithCreateTime(table, existingTable));
                }
                log.info(String.format("updated Hive table %s in db %s", tableName, dbName));
            }
        } catch (TException e2) {
            log.error(String.format("Unable to create or alter Hive table %s in db %s: " + e2.getMessage(), tableName, dbName), e2);
            throw e2;
        }
        // When the logic up to here it means table already existed in db and alteration happen. Return false.
        return false;
    }
}

Example 2 with HiveTable

use of org.apache.gobblin.hive.HiveTable in project incubator-gobblin by apache.

the class HiveMetaStoreUtils method getTable.

/**
 * Convert a {@link HiveTable} into a {@link Table}.
 */
public static Table getTable(HiveTable hiveTable) {
    State props = hiveTable.getProps();
    Table table = new Table();
    table.setDbName(hiveTable.getDbName());
    table.setTableName(hiveTable.getTableName());
    table.setParameters(getParameters(props));
    if (hiveTable.getCreateTime().isPresent()) {
        table.setCreateTime(Ints.checkedCast(hiveTable.getCreateTime().get()));
    }
    if (hiveTable.getLastAccessTime().isPresent()) {
        table.setLastAccessTime(Ints.checkedCast(hiveTable.getLastAccessTime().get()));
    }
    if (hiveTable.getOwner().isPresent()) {
        table.setOwner(hiveTable.getOwner().get());
    }
    if (hiveTable.getRetention().isPresent()) {
        table.setRetention(Ints.checkedCast(hiveTable.getRetention().get()));
    }
    if (hiveTable.getTableType().isPresent()) {
        table.setTableType(hiveTable.getTableType().get());
    } else {
        table.setTableType(DEFAULT_TABLE_TYPE.toString());
    }
    if (table.getTableType().equals(TableType.EXTERNAL_TABLE.toString())) {
        table.getParameters().put(EXTERNAL, Boolean.TRUE.toString().toUpperCase());
    }
    table.setPartitionKeys(getFieldSchemas(hiveTable.getPartitionKeys()));
    table.setSd(getStorageDescriptor(hiveTable));
    return table;
}

Example 3 with HiveTable

use of org.apache.gobblin.hive.HiveTable in project incubator-gobblin by apache.

the class HiveMetaStoreUtils method getHiveTable.

/**
 * Convert a {@link Table} into a {@link HiveTable}.
 */
public static HiveTable getHiveTable(Table table) {
    State tableProps = getTableProps(table);
    State storageProps = getStorageProps(table.getSd());
    State serDeProps = getSerDeProps(table.getSd().getSerdeInfo());
    HiveTable hiveTable = new HiveTable.Builder().withDbName(table.getDbName()).withTableName(table.getTableName()).withPartitionKeys(getColumns(table.getPartitionKeys())).withProps(tableProps).withStorageProps(storageProps).withSerdeProps(serDeProps).build();
    if (table.getCreateTime() > 0) {
        hiveTable.setCreateTime(table.getCreateTime());
    }
    if (table.getSd().getCols() != null) {
        hiveTable.setColumns(getColumns(table.getSd().getCols()));
    }
    if (table.getSd().getBucketCols() != null) {
        hiveTable.setBucketColumns(table.getSd().getBucketCols());
    }
    return hiveTable;
}

Example 4 with HiveTable

use of org.apache.gobblin.hive.HiveTable in project incubator-gobblin by apache.

the class HiveRegistrationPolicyBase method getTable.

/**
 * A base implementation for creating a non bucketed, external {@link HiveTable} for a {@link Path}.
 *
 * @param path a {@link Path} used to create the {@link HiveTable}.
 * @param dbName the database name for the created {@link HiveTable}.
 * @param tableName the table name for the created {@link HiveTable}.
 * @return a {@link HiveTable}s for the given {@link Path}.
 * @throws IOException
 */
protected HiveTable getTable(Path path, String dbName, String tableName) throws IOException {
    HiveTable table = new HiveTable.Builder().withDbName(dbName).withTableName(tableName).withSerdeManaager(HiveSerDeManager.get(this.props)).build();
    table.setLocation(this.fs.makeQualified(getTableLocation(path)).toString());
    table.setSerDeProps(path);
    // Setting table-level props.
    State tableProps = new State(this.props.getTablePartitionProps());
    if (this.props.getRuntimeTableProps().isPresent()) {
        tableProps.setProp(HiveMetaStoreUtils.RUNTIME_PROPS, this.props.getRuntimeTableProps().get());
    }
    table.setProps(tableProps);
    table.setStorageProps(this.props.getStorageProps());
    table.setSerDeProps(this.props.getSerdeProps());
    table.setNumBuckets(-1);
    table.setBucketColumns(Lists.<String>newArrayList());
    table.setTableType(TableType.EXTERNAL_TABLE.toString());
    return table;
}

Example 5 with HiveTable

use of org.apache.gobblin.hive.HiveTable in project incubator-gobblin by apache.

the class HiveSnapshotRegistrationPolicy method getHiveSpecs.

/**
 * @param path The root directory of snapshots. This directory may contain zero or more snapshots.
 */
@Override
public Collection<HiveSpec> getHiveSpecs(Path path) throws IOException {
    List<HiveTable> tables = getTables(path);
    if (tables.isEmpty()) {
        return ImmutableList.<HiveSpec>of();
    }
    Collection<HiveSpec> specs = Lists.newArrayList();
    for (HiveTable table : tables) {
        specs.add(new SimpleHiveSpec.Builder<>(path).withTable(table).withPartition(getPartition(path, table)).build());
    }
    return specs;
}