use of alluxio.grpc.table.SyncStatus in project alluxio by Alluxio.
the class SyncDatabaseCommand method run.
@Override
public int run(CommandLine cli) throws AlluxioStatusException {
SyncStatus status = mClient.syncDatabase(cli.getArgs()[0]);
TableShellUtils.printSyncStatus(status, LOG, PRINT_MAX_ERRORS);
return 0;
}
use of alluxio.grpc.table.SyncStatus in project alluxio by Alluxio.
the class Database method sync.
/**
* Syncs the metadata from the under db. To avoid concurrent sync operations, this requires
* external synchronization.
*
* @param context journal context
* @return the resulting sync status
*/
public SyncStatus sync(JournalContext context) throws IOException {
// Keep track of the status of each syncing table.
// Synchronization is necessary if accessed concurrently from multiple threads
SyncStatus.Builder builder = SyncStatus.newBuilder();
if (!mConfigPath.equals(CatalogProperty.DB_CONFIG_FILE.getDefaultValue())) {
if (!Files.exists(Paths.get(mConfigPath))) {
throw new FileNotFoundException(mConfigPath);
}
ObjectMapper mapper = new ObjectMapper();
try {
mDbConfig = mapper.readValue(new File(mConfigPath), DbConfig.class);
} catch (JsonProcessingException e) {
LOG.error("Failed to deserialize UDB config file {}, stays unsynced", mConfigPath, e);
throw e;
}
}
DatabaseInfo newDbInfo = mUdb.getDatabaseInfo();
if (!newDbInfo.equals(mDatabaseInfo)) {
applyAndJournal(context, Journal.JournalEntry.newBuilder().setUpdateDatabaseInfo(toJournalProto(newDbInfo, mName)).build());
}
Set<String> udbTableNames = new HashSet<>(mUdb.getTableNames());
// keeps track of how many tables have been synced
final AtomicInteger tablesSynced = new AtomicInteger();
// # of synced tables, after which a log message is printed for progress
final int progressBatch = (udbTableNames.size() < 100) ? udbTableNames.size() : udbTableNames.size() / 10;
// sync each table in parallel, with the executor service
List<Callable<Void>> tasks = new ArrayList<>(udbTableNames.size());
final Database thisDb = this;
for (String tableName : udbTableNames) {
if (mIgnoreTables.contains(tableName)) {
// this table should be ignored.
builder.addTablesIgnored(tableName);
tablesSynced.incrementAndGet();
continue;
}
tasks.add(() -> {
// Save all exceptions
try {
Table previousTable = mTables.get(tableName);
UdbTable udbTable = mUdb.getTable(tableName, mDbConfig.getUdbBypassSpec());
Table newTable = Table.create(thisDb, udbTable, previousTable);
if (newTable != null) {
// table was created or was updated
alluxio.proto.journal.Table.AddTableEntry addTableEntry = newTable.getTableJournalProto();
Journal.JournalEntry entry = Journal.JournalEntry.newBuilder().setAddTable(addTableEntry).build();
applyAndJournal(context, entry);
// separate the possible big table entry into multiple smaller table partitions entry
newTable.getTablePartitionsJournalProto().forEach((partitionsEntry) -> {
applyAndJournal(context, Journal.JournalEntry.newBuilder().setAddTablePartitions(partitionsEntry).build());
});
synchronized (builder) {
builder.addTablesUpdated(tableName);
}
} else {
synchronized (builder) {
builder.addTablesUnchanged(tableName);
}
}
} catch (Exception e) {
LOG.error(String.format("Sync thread failed for %s.%s", thisDb.mName, tableName), e);
synchronized (builder) {
builder.putTablesErrors(tableName, e.toString());
}
} finally {
int syncedTables = tablesSynced.incrementAndGet();
int percentage = -1;
// Only log at regular intervals, or when complete
if (syncedTables % progressBatch == 0) {
// compute percentage, cap at 99%
percentage = Math.min(Math.round(100.0f * syncedTables / udbTableNames.size()), 99);
}
if (syncedTables == udbTableNames.size()) {
percentage = 100;
}
if (percentage != -1) {
LOG.info("Syncing db {} progress: completed {} of {} tables ({}%)", mName, syncedTables, udbTableNames.size(), percentage);
}
}
return null;
});
}
// create a thread pool to parallelize the sync
int threads;
try {
threads = Integer.parseInt(mConfig.get(CatalogProperty.DB_SYNC_THREADS));
} catch (NumberFormatException e) {
LOG.warn("Catalog property {} with value {} cannot be parsed as an int", CatalogProperty.DB_SYNC_THREADS.getName(), mConfig.get(CatalogProperty.DB_SYNC_THREADS));
threads = CatalogProperty.DEFAULT_DB_SYNC_THREADS;
}
if (threads < 1) {
// if invalid, set to the default
threads = CatalogProperty.DEFAULT_DB_SYNC_THREADS;
}
ExecutorService service = ExecutorServiceFactories.fixedThreadPool(String.format("Catalog-Sync-%s", mName), threads).create();
try {
CommonUtils.invokeAll(service, tasks, mUdbSyncTimeoutMs);
} catch (Exception e) {
throw new IOException("Failed to sync database " + mName + ". error: " + e.toString(), e);
} finally {
// shutdown the thread pool
service.shutdownNow();
String errorMessage = String.format("waiting for db-sync thread pool to shut down. db: %s", mName);
try {
if (!service.awaitTermination(5, TimeUnit.SECONDS)) {
LOG.warn("Timed out " + errorMessage);
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
LOG.warn("Interrupted while " + errorMessage);
}
}
for (Table existingTable : mTables.values()) {
if (!udbTableNames.contains(existingTable.getName())) {
// this table no longer exists in udb
alluxio.proto.journal.Table.RemoveTableEntry removeTableEntry = alluxio.proto.journal.Table.RemoveTableEntry.newBuilder().setDbName(mName).setTableName(existingTable.getName()).setVersion(existingTable.getVersion()).build();
Journal.JournalEntry entry = Journal.JournalEntry.newBuilder().setRemoveTable(removeTableEntry).build();
applyAndJournal(context, entry);
builder.addTablesRemoved(existingTable.getName());
}
}
return builder.build();
}
use of alluxio.grpc.table.SyncStatus in project alluxio by Alluxio.
the class AlluxioCatalog method attachDatabase.
/**
* Attaches an udb database to Alluxio catalog.
*
* @param journalContext journal context
* @param udbType the database type
* @param udbConnectionUri the udb connection uri
* @param udbDbName the database name in the udb
* @param dbName the database name in Alluxio
* @param map the configuration
* @param ignoreSyncErrors if true, will ignore syncing errors during the attach
* @return the sync status for the attach
*/
public SyncStatus attachDatabase(JournalContext journalContext, String udbType, String udbConnectionUri, String udbDbName, String dbName, Map<String, String> map, boolean ignoreSyncErrors) throws IOException {
try (LockResource l = getDbLock(dbName)) {
if (mDBs.containsKey(dbName)) {
throw new IOException(String.format("Unable to attach database. Database name %s (type: %s) already exists.", dbName, udbType));
}
applyAndJournal(journalContext, Journal.JournalEntry.newBuilder().setAttachDb(alluxio.proto.journal.Table.AttachDbEntry.newBuilder().setUdbType(udbType).setUdbConnectionUri(udbConnectionUri).setUdbDbName(udbDbName).setDbName(dbName).putAllConfig(map).build()).build());
boolean syncError = false;
try {
SyncStatus status = mDBs.get(dbName).sync(journalContext);
syncError = status.getTablesErrorsCount() > 0;
return status;
} catch (Exception e) {
// Failed to connect to and sync the udb.
syncError = true;
// log the error and stack
LOG.error(String.format("Sync (during attach) failed for db '%s'.", dbName), e);
throw new IOException(String.format("Failed to connect underDb for Alluxio db '%s': %s", dbName, e.getMessage()), e);
} finally {
if (syncError && !ignoreSyncErrors) {
applyAndJournal(journalContext, Journal.JournalEntry.newBuilder().setDetachDb(alluxio.proto.journal.Table.DetachDbEntry.newBuilder().setDbName(dbName).build()).build());
}
}
}
}
use of alluxio.grpc.table.SyncStatus in project alluxio by Alluxio.
the class AttachDatabaseCommand method run.
@Override
public int run(CommandLine cl) throws AlluxioStatusException {
String[] args = cl.getArgs();
String udbType = args[0];
String udbConnectionUri = args[1];
String udbDbName = args[2];
String dbName = udbDbName;
if (cl.hasOption(DB_OPTION.getLongOpt())) {
String optDbName = cl.getOptionValue(DB_OPTION.getLongOpt());
if (optDbName != null && !optDbName.isEmpty()) {
dbName = optDbName;
}
}
Properties p = cl.getOptionProperties(OPTION_OPTION.getOpt());
boolean ignoreSyncErrors = cl.hasOption(IGNORE_SYNC_ERRORS_OPTION.getLongOpt());
SyncStatus status = mClient.attachDatabase(udbType, udbConnectionUri, udbDbName, dbName, Maps.fromProperties(p), ignoreSyncErrors);
TableShellUtils.printSyncStatus(status, LOG, PRINT_MAX_ERRORS);
if (!ignoreSyncErrors && status.getTablesErrorsCount() > 0) {
System.out.println(String.format("%nDatabase is not attached. To keep it attached even with errors, please re-run '%s' " + "with the '--%s' option.", COMMAND_NAME, IGNORE_SYNC_ERRORS_OPTION.getLongOpt()));
}
return 0;
}
use of alluxio.grpc.table.SyncStatus in project alluxio by Alluxio.
the class TableIntegrationTest method attachMountFail.
@Test
public void attachMountFail() throws Exception {
SyncStatus status = sTableMaster.attachDatabase("hive", "thrift://" + sHmsAddress + ":" + sHmsPort, DB_NAME_ERRORS, DB_NAME_ERRORS, Collections.emptyMap(), false);
assertEquals(1, status.getTablesErrorsCount());
assertTrue(status.containsTablesErrors(TEST_TABLE_ERROR));
try {
sTableMaster.getDatabase(DB_NAME_ERRORS);
fail("DB with a bad table should be not be attached.");
} catch (Exception e) {
// this is expected
}
status = sTableMaster.attachDatabase("hive", "thrift://" + sHmsAddress + ":" + sHmsPort, DB_NAME_ERRORS, DB_NAME_ERRORS, Collections.emptyMap(), true);
assertEquals(1, status.getTablesErrorsCount());
assertTrue(status.containsTablesErrors(TEST_TABLE_ERROR));
assertNotNull(sTableMaster.getDatabase(DB_NAME_ERRORS));
}
Aggregations