Examples with SimpleThreadPool org.apache.accumulo.core.util.SimpleThreadPool used on opensource projects

Example 6 with SimpleThreadPool

use of org.apache.accumulo.core.util.SimpleThreadPool in project accumulo by apache.

the class BatchWriterFlushIT method runMultiThreadedBinningTest.

@Test
public void runMultiThreadedBinningTest() throws Exception {
    Connector c = getConnector();
    String[] tableNames = getUniqueNames(1);
    String tableName = tableNames[0];
    c.tableOperations().create(tableName);
    for (int x = 0; x < NUM_THREADS; x++) {
        c.tableOperations().addSplits(tableName, new TreeSet<>(Collections.singleton(new Text(Integer.toString(x * NUM_TO_FLUSH)))));
    }
    c.instanceOperations().waitForBalance();
    // Logger.getLogger(TabletServerBatchWriter.class).setLevel(Level.TRACE);
    final List<Set<Mutation>> allMuts = new LinkedList<>();
    List<Mutation> data = new ArrayList<>();
    for (int i = 0; i < NUM_THREADS; i++) {
        final int thread = i;
        for (int j = 0; j < NUM_TO_FLUSH; j++) {
            int row = thread * NUM_TO_FLUSH + j;
            Mutation m = new Mutation(new Text(String.format("%10d", row)));
            m.put(new Text("cf" + thread), new Text("cq"), new Value(("" + row).getBytes()));
            data.add(m);
        }
    }
    Assert.assertEquals(NUM_THREADS * NUM_TO_FLUSH, data.size());
    Collections.shuffle(data);
    for (int n = 0; n < (NUM_THREADS * NUM_TO_FLUSH); n += NUM_TO_FLUSH) {
        Set<Mutation> muts = new HashSet<>(data.subList(n, n + NUM_TO_FLUSH));
        allMuts.add(muts);
    }
    SimpleThreadPool threads = new SimpleThreadPool(NUM_THREADS, "ClientThreads");
    threads.allowCoreThreadTimeOut(false);
    threads.prestartAllCoreThreads();
    BatchWriterConfig cfg = new BatchWriterConfig();
    cfg.setMaxLatency(10, TimeUnit.SECONDS);
    cfg.setMaxMemory(1 * 1024 * 1024);
    cfg.setMaxWriteThreads(NUM_THREADS);
    final BatchWriter bw = getConnector().createBatchWriter(tableName, cfg);
    for (int k = 0; k < NUM_THREADS; k++) {
        final int idx = k;
        threads.execute(new Runnable() {

            @Override
            public void run() {
                try {
                    bw.addMutations(allMuts.get(idx));
                    bw.flush();
                } catch (MutationsRejectedException e) {
                    Assert.fail("Error adding mutations to batch writer");
                }
            }
        });
    }
    threads.shutdown();
    threads.awaitTermination(3, TimeUnit.MINUTES);
    bw.close();
    try (Scanner scanner = getConnector().createScanner(tableName, Authorizations.EMPTY)) {
        for (Entry<Key, Value> e : scanner) {
            Mutation m = new Mutation(e.getKey().getRow());
            m.put(e.getKey().getColumnFamily(), e.getKey().getColumnQualifier(), e.getValue());
            boolean found = false;
            for (int l = 0; l < NUM_THREADS; l++) {
                if (allMuts.get(l).contains(m)) {
                    found = true;
                    allMuts.get(l).remove(m);
                    break;
                }
            }
            Assert.assertTrue("Mutation not found: " + m.toString(), found);
        }
        for (int m = 0; m < NUM_THREADS; m++) {
            Assert.assertEquals(0, allMuts.get(m).size());
        }
    }
}

Example 7 with SimpleThreadPool

use of org.apache.accumulo.core.util.SimpleThreadPool in project accumulo by apache.

the class RandomizeVolumes method randomize.

public static int randomize(Connector c, String tableName) throws IOException, AccumuloSecurityException, AccumuloException, TableNotFoundException {
    final VolumeManager vm = VolumeManagerImpl.get();
    if (vm.getVolumes().size() < 2) {
        log.error("There are not enough volumes configured");
        return 1;
    }
    String tblStr = c.tableOperations().tableIdMap().get(tableName);
    if (null == tblStr) {
        log.error("Could not determine the table ID for table {}", tableName);
        return 2;
    }
    Table.ID tableId = Table.ID.of(tblStr);
    TableState tableState = TableManager.getInstance().getTableState(tableId);
    if (TableState.OFFLINE != tableState) {
        log.info("Taking {} offline", tableName);
        c.tableOperations().offline(tableName, true);
        log.info("{} offline", tableName);
    }
    SimpleThreadPool pool = new SimpleThreadPool(50, "directory maker");
    log.info("Rewriting entries for {}", tableName);
    Scanner scanner = c.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
    DIRECTORY_COLUMN.fetch(scanner);
    scanner.setRange(TabletsSection.getRange(tableId));
    BatchWriter writer = c.createBatchWriter(MetadataTable.NAME, null);
    int count = 0;
    for (Entry<Key, Value> entry : scanner) {
        String oldLocation = entry.getValue().toString();
        String directory;
        if (oldLocation.contains(":")) {
            String[] parts = oldLocation.split(Path.SEPARATOR);
            Table.ID tableIdEntry = Table.ID.of(parts[parts.length - 2]);
            if (!tableIdEntry.equals(tableId)) {
                log.error("Unexpected table id found: {}, expected {}; skipping", tableIdEntry, tableId);
                continue;
            }
            directory = parts[parts.length - 1];
        } else {
            directory = oldLocation.substring(Path.SEPARATOR.length());
        }
        Key key = entry.getKey();
        Mutation m = new Mutation(key.getRow());
        VolumeChooserEnvironment chooserEnv = new VolumeChooserEnvironment(tableId);
        final String newLocation = vm.choose(chooserEnv, ServerConstants.getBaseUris()) + Path.SEPARATOR + ServerConstants.TABLE_DIR + Path.SEPARATOR + tableId + Path.SEPARATOR + directory;
        m.put(key.getColumnFamily(), key.getColumnQualifier(), new Value(newLocation.getBytes(UTF_8)));
        if (log.isTraceEnabled()) {
            log.trace("Replacing {} with {}", oldLocation, newLocation);
        }
        writer.addMutation(m);
        pool.submit(new Runnable() {

            @Override
            public void run() {
                try {
                    vm.mkdirs(new Path(newLocation));
                } catch (IOException ex) {
                // nevermind
                }
            }
        });
        count++;
    }
    writer.close();
    pool.shutdown();
    while (!pool.isTerminated()) {
        log.trace("Waiting for mkdir() calls to finish");
        try {
            pool.awaitTermination(5, TimeUnit.SECONDS);
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
            break;
        }
    }
    log.info("Updated {} entries for table {}", count, tableName);
    if (TableState.OFFLINE != tableState) {
        c.tableOperations().online(tableName, true);
        log.info("table {} back online", tableName);
    }
    return 0;
}

Example 8 with SimpleThreadPool

use of org.apache.accumulo.core.util.SimpleThreadPool in project accumulo by apache.

the class TabletServer method run.

// main loop listens for client requests
@Override
public void run() {
    SecurityUtil.serverLogin(SiteConfiguration.getInstance());
    // We can just make the zookeeper paths before we try to use.
    try {
        ZooKeeperInitialization.ensureZooKeeperInitialized(ZooReaderWriter.getInstance(), ZooUtil.getRoot(getInstance()));
    } catch (KeeperException | InterruptedException e) {
        log.error("Could not ensure that ZooKeeper is properly initialized", e);
        throw new RuntimeException(e);
    }
    Metrics tserverMetrics = metricsFactory.createTabletServerMetrics(this);
    // Register MBeans
    try {
        tserverMetrics.register();
        mincMetrics.register();
        scanMetrics.register();
        updateMetrics.register();
    } catch (Exception e) {
        log.error("Error registering with JMX", e);
    }
    if (null != authKeyWatcher) {
        log.info("Seeding ZooKeeper watcher for authentication keys");
        try {
            authKeyWatcher.updateAuthKeys();
        } catch (KeeperException | InterruptedException e) {
            // TODO Does there need to be a better check? What are the error conditions that we'd fall out here? AUTH_FAILURE?
            // If we get the error, do we just put it on a timer and retry the exists(String, Watcher) call?
            log.error("Failed to perform initial check for authentication tokens in ZooKeeper. Delegation token authentication will be unavailable.", e);
        }
    }
    try {
        clientAddress = startTabletClientService();
    } catch (UnknownHostException e1) {
        throw new RuntimeException("Failed to start the tablet client service", e1);
    }
    announceExistence();
    try {
        walMarker.initWalMarker(getTabletSession());
    } catch (Exception e) {
        log.error("Unable to create WAL marker node in zookeeper", e);
        throw new RuntimeException(e);
    }
    ThreadPoolExecutor distWorkQThreadPool = new SimpleThreadPool(getConfiguration().getCount(Property.TSERV_WORKQ_THREADS), "distributed work queue");
    bulkFailedCopyQ = new DistributedWorkQueue(ZooUtil.getRoot(getInstance()) + Constants.ZBULK_FAILED_COPYQ, getConfiguration());
    try {
        bulkFailedCopyQ.startProcessing(new BulkFailedCopyProcessor(), distWorkQThreadPool);
    } catch (Exception e1) {
        throw new RuntimeException("Failed to start distributed work queue for copying ", e1);
    }
    try {
        logSorter.startWatchingForRecoveryLogs(distWorkQThreadPool);
    } catch (Exception ex) {
        log.error("Error setting watches for recoveries");
        throw new RuntimeException(ex);
    }
    // Start the thrift service listening for incoming replication requests
    try {
        replicationAddress = startReplicationService();
    } catch (UnknownHostException e) {
        throw new RuntimeException("Failed to start replication service", e);
    }
    // Start the pool to handle outgoing replications
    final ThreadPoolExecutor replicationThreadPool = new SimpleThreadPool(getConfiguration().getCount(Property.REPLICATION_WORKER_THREADS), "replication task");
    replWorker.setExecutor(replicationThreadPool);
    replWorker.run();
    // Check the configuration value for the size of the pool and, if changed, resize the pool, every 5 seconds);
    final AccumuloConfiguration aconf = getConfiguration();
    Runnable replicationWorkThreadPoolResizer = new Runnable() {

        @Override
        public void run() {
            int maxPoolSize = aconf.getCount(Property.REPLICATION_WORKER_THREADS);
            if (replicationThreadPool.getMaximumPoolSize() != maxPoolSize) {
                log.info("Resizing thread pool for sending replication work from {} to {}", replicationThreadPool.getMaximumPoolSize(), maxPoolSize);
                replicationThreadPool.setMaximumPoolSize(maxPoolSize);
            }
        }
    };
    SimpleTimer.getInstance(aconf).schedule(replicationWorkThreadPoolResizer, 10000, 30000);
    final long CLEANUP_BULK_LOADED_CACHE_MILLIS = 15 * 60 * 1000;
    SimpleTimer.getInstance(aconf).schedule(new BulkImportCacheCleaner(this), CLEANUP_BULK_LOADED_CACHE_MILLIS, CLEANUP_BULK_LOADED_CACHE_MILLIS);
    HostAndPort masterHost;
    while (!serverStopRequested) {
        // send all of the pending messages
        try {
            MasterMessage mm = null;
            MasterClientService.Client iface = null;
            try {
                // was requested
                while (mm == null && !serverStopRequested) {
                    mm = masterMessages.poll(1000, TimeUnit.MILLISECONDS);
                }
                // have a message to send to the master, so grab a
                // connection
                masterHost = getMasterAddress();
                iface = masterConnection(masterHost);
                TServiceClient client = iface;
                // then finally block should place mm back on queue
                while (!serverStopRequested && mm != null && client != null && client.getOutputProtocol() != null && client.getOutputProtocol().getTransport() != null && client.getOutputProtocol().getTransport().isOpen()) {
                    try {
                        mm.send(rpcCreds(), getClientAddressString(), iface);
                        mm = null;
                    } catch (TException ex) {
                        log.warn("Error sending message: queuing message again");
                        masterMessages.putFirst(mm);
                        mm = null;
                        throw ex;
                    }
                    // if any messages are immediately available grab em and
                    // send them
                    mm = masterMessages.poll();
                }
            } finally {
                if (mm != null) {
                    masterMessages.putFirst(mm);
                }
                returnMasterConnection(iface);
                sleepUninterruptibly(1, TimeUnit.SECONDS);
            }
        } catch (InterruptedException e) {
            log.info("Interrupt Exception received, shutting down");
            serverStopRequested = true;
        } catch (Exception e) {
            // may have lost connection with master
            // loop back to the beginning and wait for a new one
            // this way we survive master failures
            log.error(getClientAddressString() + ": TServerInfo: Exception. Master down?", e);
        }
    }
    // get prematurely finalized
    synchronized (this) {
        while (!shutdownComplete) {
            try {
                this.wait(1000);
            } catch (InterruptedException e) {
                log.error(e.toString());
            }
        }
    }
    log.debug("Stopping Replication Server");
    TServerUtils.stopTServer(this.replServer);
    log.debug("Stopping Thrift Servers");
    TServerUtils.stopTServer(server);
    try {
        log.debug("Closing filesystem");
        fs.close();
    } catch (IOException e) {
        log.warn("Failed to close filesystem : {}", e.getMessage(), e);
    }
    gcLogger.logGCInfo(getConfiguration());
    log.info("TServerInfo: stop requested. exiting ... ");
    try {
        tabletServerLock.unlock();
    } catch (Exception e) {
        log.warn("Failed to release tablet server lock", e);
    }
}

Example 9 with SimpleThreadPool

use of org.apache.accumulo.core.util.SimpleThreadPool in project accumulo by apache.

the class BalanceWithOfflineTableIT method test.

@Test
public void test() throws Exception {
    final String[] tableNames = getUniqueNames(2);
    final String tableName = tableNames[0];
    // create a table with a bunch of splits
    final Connector c = getConnector();
    log.info("Creating table {}", tableName);
    c.tableOperations().create(tableName);
    final SortedSet<Text> splits = new TreeSet<>();
    for (String split : "a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,w,x,y,z".split(",")) {
        splits.add(new Text(split));
    }
    log.info("Splitting table {}", tableName);
    c.tableOperations().addSplits(tableName, splits);
    log.info("Balancing");
    c.instanceOperations().waitForBalance();
    log.info("Balanced");
    // create a new table which will unbalance the cluster
    final String table2 = tableNames[1];
    log.info("Creating table {}", table2);
    c.tableOperations().create(table2);
    log.info("Creating splits {}", table2);
    c.tableOperations().addSplits(table2, splits);
    // offline the table, hopefully while there are some migrations going on
    log.info("Offlining {}", table2);
    c.tableOperations().offline(table2, true);
    log.info("Offlined {}", table2);
    log.info("Waiting for balance");
    SimpleThreadPool pool = new SimpleThreadPool(1, "waitForBalance");
    Future<Boolean> wait = pool.submit(new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            c.instanceOperations().waitForBalance();
            return true;
        }
    });
    wait.get(20, TimeUnit.SECONDS);
    log.info("Balance succeeded with an offline table");
}