Examples with SyncStatus alluxio.grpc.table.SyncStatus used on opensource projects

Example 1 with SyncStatus

use of alluxio.grpc.table.SyncStatus in project alluxio by Alluxio.

the class SyncDatabaseCommand method run.

@Override
public int run(CommandLine cli) throws AlluxioStatusException {
    SyncStatus status = mClient.syncDatabase(cli.getArgs()[0]);
    TableShellUtils.printSyncStatus(status, LOG, PRINT_MAX_ERRORS);
    return 0;
}

Example 2 with SyncStatus

use of alluxio.grpc.table.SyncStatus in project alluxio by Alluxio.

the class Database method sync.

/**
 * Syncs the metadata from the under db. To avoid concurrent sync operations, this requires
 * external synchronization.
 *
 * @param context journal context
 * @return the resulting sync status
 */
public SyncStatus sync(JournalContext context) throws IOException {
    // Keep track of the status of each syncing table.
    // Synchronization is necessary if accessed concurrently from multiple threads
    SyncStatus.Builder builder = SyncStatus.newBuilder();
    if (!mConfigPath.equals(CatalogProperty.DB_CONFIG_FILE.getDefaultValue())) {
        if (!Files.exists(Paths.get(mConfigPath))) {
            throw new FileNotFoundException(mConfigPath);
        }
        ObjectMapper mapper = new ObjectMapper();
        try {
            mDbConfig = mapper.readValue(new File(mConfigPath), DbConfig.class);
        } catch (JsonProcessingException e) {
            LOG.error("Failed to deserialize UDB config file {}, stays unsynced", mConfigPath, e);
            throw e;
        }
    }
    DatabaseInfo newDbInfo = mUdb.getDatabaseInfo();
    if (!newDbInfo.equals(mDatabaseInfo)) {
        applyAndJournal(context, Journal.JournalEntry.newBuilder().setUpdateDatabaseInfo(toJournalProto(newDbInfo, mName)).build());
    }
    Set<String> udbTableNames = new HashSet<>(mUdb.getTableNames());
    // keeps track of how many tables have been synced
    final AtomicInteger tablesSynced = new AtomicInteger();
    // # of synced tables, after which a log message is printed for progress
    final int progressBatch = (udbTableNames.size() < 100) ? udbTableNames.size() : udbTableNames.size() / 10;
    // sync each table in parallel, with the executor service
    List<Callable<Void>> tasks = new ArrayList<>(udbTableNames.size());
    final Database thisDb = this;
    for (String tableName : udbTableNames) {
        if (mIgnoreTables.contains(tableName)) {
            // this table should be ignored.
            builder.addTablesIgnored(tableName);
            tablesSynced.incrementAndGet();
            continue;
        }
        tasks.add(() -> {
            // Save all exceptions
            try {
                Table previousTable = mTables.get(tableName);
                UdbTable udbTable = mUdb.getTable(tableName, mDbConfig.getUdbBypassSpec());
                Table newTable = Table.create(thisDb, udbTable, previousTable);
                if (newTable != null) {
                    // table was created or was updated
                    alluxio.proto.journal.Table.AddTableEntry addTableEntry = newTable.getTableJournalProto();
                    Journal.JournalEntry entry = Journal.JournalEntry.newBuilder().setAddTable(addTableEntry).build();
                    applyAndJournal(context, entry);
                    // separate the possible big table entry into multiple smaller table partitions entry
                    newTable.getTablePartitionsJournalProto().forEach((partitionsEntry) -> {
                        applyAndJournal(context, Journal.JournalEntry.newBuilder().setAddTablePartitions(partitionsEntry).build());
                    });
                    synchronized (builder) {
                        builder.addTablesUpdated(tableName);
                    }
                } else {
                    synchronized (builder) {
                        builder.addTablesUnchanged(tableName);
                    }
                }
            } catch (Exception e) {
                LOG.error(String.format("Sync thread failed for %s.%s", thisDb.mName, tableName), e);
                synchronized (builder) {
                    builder.putTablesErrors(tableName, e.toString());
                }
            } finally {
                int syncedTables = tablesSynced.incrementAndGet();
                int percentage = -1;
                // Only log at regular intervals, or when complete
                if (syncedTables % progressBatch == 0) {
                    // compute percentage, cap at 99%
                    percentage = Math.min(Math.round(100.0f * syncedTables / udbTableNames.size()), 99);
                }
                if (syncedTables == udbTableNames.size()) {
                    percentage = 100;
                }
                if (percentage != -1) {
                    LOG.info("Syncing db {} progress: completed {} of {} tables ({}%)", mName, syncedTables, udbTableNames.size(), percentage);
                }
            }
            return null;
        });
    }
    // create a thread pool to parallelize the sync
    int threads;
    try {
        threads = Integer.parseInt(mConfig.get(CatalogProperty.DB_SYNC_THREADS));
    } catch (NumberFormatException e) {
        LOG.warn("Catalog property {} with value {} cannot be parsed as an int", CatalogProperty.DB_SYNC_THREADS.getName(), mConfig.get(CatalogProperty.DB_SYNC_THREADS));
        threads = CatalogProperty.DEFAULT_DB_SYNC_THREADS;
    }
    if (threads < 1) {
        // if invalid, set to the default
        threads = CatalogProperty.DEFAULT_DB_SYNC_THREADS;
    }
    ExecutorService service = ExecutorServiceFactories.fixedThreadPool(String.format("Catalog-Sync-%s", mName), threads).create();
    try {
        CommonUtils.invokeAll(service, tasks, mUdbSyncTimeoutMs);
    } catch (Exception e) {
        throw new IOException("Failed to sync database " + mName + ". error: " + e.toString(), e);
    } finally {
        // shutdown the thread pool
        service.shutdownNow();
        String errorMessage = String.format("waiting for db-sync thread pool to shut down. db: %s", mName);
        try {
            if (!service.awaitTermination(5, TimeUnit.SECONDS)) {
                LOG.warn("Timed out " + errorMessage);
            }
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
            LOG.warn("Interrupted while " + errorMessage);
        }
    }
    for (Table existingTable : mTables.values()) {
        if (!udbTableNames.contains(existingTable.getName())) {
            // this table no longer exists in udb
            alluxio.proto.journal.Table.RemoveTableEntry removeTableEntry = alluxio.proto.journal.Table.RemoveTableEntry.newBuilder().setDbName(mName).setTableName(existingTable.getName()).setVersion(existingTable.getVersion()).build();
            Journal.JournalEntry entry = Journal.JournalEntry.newBuilder().setRemoveTable(removeTableEntry).build();
            applyAndJournal(context, entry);
            builder.addTablesRemoved(existingTable.getName());
        }
    }
    return builder.build();
}

Example 3 with SyncStatus

use of alluxio.grpc.table.SyncStatus in project alluxio by Alluxio.

the class AlluxioCatalog method attachDatabase.

/**
 * Attaches an udb database to Alluxio catalog.
 *
 * @param journalContext journal context
 * @param udbType the database type
 * @param udbConnectionUri the udb connection uri
 * @param udbDbName the database name in the udb
 * @param dbName the database name in Alluxio
 * @param map the configuration
 * @param ignoreSyncErrors if true, will ignore syncing errors during the attach
 * @return the sync status for the attach
 */
public SyncStatus attachDatabase(JournalContext journalContext, String udbType, String udbConnectionUri, String udbDbName, String dbName, Map<String, String> map, boolean ignoreSyncErrors) throws IOException {
    try (LockResource l = getDbLock(dbName)) {
        if (mDBs.containsKey(dbName)) {
            throw new IOException(String.format("Unable to attach database. Database name %s (type: %s) already exists.", dbName, udbType));
        }
        applyAndJournal(journalContext, Journal.JournalEntry.newBuilder().setAttachDb(alluxio.proto.journal.Table.AttachDbEntry.newBuilder().setUdbType(udbType).setUdbConnectionUri(udbConnectionUri).setUdbDbName(udbDbName).setDbName(dbName).putAllConfig(map).build()).build());
        boolean syncError = false;
        try {
            SyncStatus status = mDBs.get(dbName).sync(journalContext);
            syncError = status.getTablesErrorsCount() > 0;
            return status;
        } catch (Exception e) {
            // Failed to connect to and sync the udb.
            syncError = true;
            // log the error and stack
            LOG.error(String.format("Sync (during attach) failed for db '%s'.", dbName), e);
            throw new IOException(String.format("Failed to connect underDb for Alluxio db '%s': %s", dbName, e.getMessage()), e);
        } finally {
            if (syncError && !ignoreSyncErrors) {
                applyAndJournal(journalContext, Journal.JournalEntry.newBuilder().setDetachDb(alluxio.proto.journal.Table.DetachDbEntry.newBuilder().setDbName(dbName).build()).build());
            }
        }
    }
}

Example 4 with SyncStatus

use of alluxio.grpc.table.SyncStatus in project alluxio by Alluxio.

the class AttachDatabaseCommand method run.

@Override
public int run(CommandLine cl) throws AlluxioStatusException {
    String[] args = cl.getArgs();
    String udbType = args[0];
    String udbConnectionUri = args[1];
    String udbDbName = args[2];
    String dbName = udbDbName;
    if (cl.hasOption(DB_OPTION.getLongOpt())) {
        String optDbName = cl.getOptionValue(DB_OPTION.getLongOpt());
        if (optDbName != null && !optDbName.isEmpty()) {
            dbName = optDbName;
        }
    }
    Properties p = cl.getOptionProperties(OPTION_OPTION.getOpt());
    boolean ignoreSyncErrors = cl.hasOption(IGNORE_SYNC_ERRORS_OPTION.getLongOpt());
    SyncStatus status = mClient.attachDatabase(udbType, udbConnectionUri, udbDbName, dbName, Maps.fromProperties(p), ignoreSyncErrors);
    TableShellUtils.printSyncStatus(status, LOG, PRINT_MAX_ERRORS);
    if (!ignoreSyncErrors && status.getTablesErrorsCount() > 0) {
        System.out.println(String.format("%nDatabase is not attached. To keep it attached even with errors, please re-run '%s' " + "with the '--%s' option.", COMMAND_NAME, IGNORE_SYNC_ERRORS_OPTION.getLongOpt()));
    }
    return 0;
}

Example 5 with SyncStatus

use of alluxio.grpc.table.SyncStatus in project alluxio by Alluxio.

the class TableIntegrationTest method attachMountFail.

@Test
public void attachMountFail() throws Exception {
    SyncStatus status = sTableMaster.attachDatabase("hive", "thrift://" + sHmsAddress + ":" + sHmsPort, DB_NAME_ERRORS, DB_NAME_ERRORS, Collections.emptyMap(), false);
    assertEquals(1, status.getTablesErrorsCount());
    assertTrue(status.containsTablesErrors(TEST_TABLE_ERROR));
    try {
        sTableMaster.getDatabase(DB_NAME_ERRORS);
        fail("DB with a bad table should be not be attached.");
    } catch (Exception e) {
    // this is expected
    }
    status = sTableMaster.attachDatabase("hive", "thrift://" + sHmsAddress + ":" + sHmsPort, DB_NAME_ERRORS, DB_NAME_ERRORS, Collections.emptyMap(), true);
    assertEquals(1, status.getTablesErrorsCount());
    assertTrue(status.containsTablesErrors(TEST_TABLE_ERROR));
    assertNotNull(sTableMaster.getDatabase(DB_NAME_ERRORS));
}