Examples with VolumeManager org.apache.accumulo.server.fs.VolumeManager used on opensource projects

Example 1 with VolumeManager

use of org.apache.accumulo.server.fs.VolumeManager in project accumulo by apache.

the class BulkImporter method importFiles.

public AssignmentStats importFiles(List<String> files, Path failureDir) throws IOException, AccumuloException, AccumuloSecurityException, ThriftTableOperationException {
    int numThreads = context.getConfiguration().getCount(Property.TSERV_BULK_PROCESS_THREADS);
    int numAssignThreads = context.getConfiguration().getCount(Property.TSERV_BULK_ASSIGNMENT_THREADS);
    timer = new StopWatch<>(Timers.class);
    timer.start(Timers.TOTAL);
    Configuration conf = CachedConfiguration.getInstance();
    VolumeManagerImpl.get(context.getConfiguration());
    final VolumeManager fs = VolumeManagerImpl.get(context.getConfiguration());
    Set<Path> paths = new HashSet<>();
    for (String file : files) {
        paths.add(new Path(file));
    }
    AssignmentStats assignmentStats = new AssignmentStats(paths.size());
    final Map<Path, List<KeyExtent>> completeFailures = Collections.synchronizedSortedMap(new TreeMap<Path, List<KeyExtent>>());
    ClientService.Client client = null;
    final TabletLocator locator = TabletLocator.getLocator(context, Table.ID.of(tableId));
    try {
        final Map<Path, List<TabletLocation>> assignments = Collections.synchronizedSortedMap(new TreeMap<Path, List<TabletLocation>>());
        timer.start(Timers.EXAMINE_MAP_FILES);
        ExecutorService threadPool = Executors.newFixedThreadPool(numThreads, new NamingThreadFactory("findOverlapping"));
        for (Path path : paths) {
            final Path mapFile = path;
            Runnable getAssignments = new Runnable() {

                @Override
                public void run() {
                    List<TabletLocation> tabletsToAssignMapFileTo = Collections.emptyList();
                    try {
                        tabletsToAssignMapFileTo = findOverlappingTablets(context, fs, locator, mapFile);
                    } catch (Exception ex) {
                        log.warn("Unable to find tablets that overlap file " + mapFile.toString(), ex);
                    }
                    log.debug("Map file {} found to overlap {} tablets", mapFile, tabletsToAssignMapFileTo.size());
                    if (tabletsToAssignMapFileTo.size() == 0) {
                        List<KeyExtent> empty = Collections.emptyList();
                        completeFailures.put(mapFile, empty);
                    } else
                        assignments.put(mapFile, tabletsToAssignMapFileTo);
                }
            };
            threadPool.submit(new TraceRunnable(new LoggingRunnable(log, getAssignments)));
        }
        threadPool.shutdown();
        while (!threadPool.isTerminated()) {
            try {
                threadPool.awaitTermination(60, TimeUnit.SECONDS);
            } catch (InterruptedException e) {
                throw new RuntimeException(e);
            }
        }
        timer.stop(Timers.EXAMINE_MAP_FILES);
        assignmentStats.attemptingAssignments(assignments);
        Map<Path, List<KeyExtent>> assignmentFailures = assignMapFiles(context, conf, fs, tableId, assignments, paths, numAssignThreads, numThreads);
        assignmentStats.assignmentsFailed(assignmentFailures);
        Map<Path, Integer> failureCount = new TreeMap<>();
        for (Entry<Path, List<KeyExtent>> entry : assignmentFailures.entrySet()) failureCount.put(entry.getKey(), 1);
        long sleepTime = 2 * 1000;
        while (assignmentFailures.size() > 0) {
            sleepTime = Math.min(sleepTime * 2, 60 * 1000);
            locator.invalidateCache();
            // assumption about assignment failures is that it caused by a split
            // happening or a missing location
            // 
            // for splits we need to find children key extents that cover the
            // same key range and are contiguous (no holes, no overlap)
            timer.start(Timers.SLEEP);
            sleepUninterruptibly(sleepTime, TimeUnit.MILLISECONDS);
            timer.stop(Timers.SLEEP);
            log.debug("Trying to assign {} map files that previously failed on some key extents", assignmentFailures.size());
            assignments.clear();
            // assign to
            for (Entry<Path, List<KeyExtent>> entry : assignmentFailures.entrySet()) {
                Iterator<KeyExtent> keListIter = entry.getValue().iterator();
                List<TabletLocation> tabletsToAssignMapFileTo = new ArrayList<>();
                while (keListIter.hasNext()) {
                    KeyExtent ke = keListIter.next();
                    timer.start(Timers.QUERY_METADATA);
                    try {
                        tabletsToAssignMapFileTo.addAll(findOverlappingTablets(context, fs, locator, entry.getKey(), ke));
                        keListIter.remove();
                    } catch (Exception ex) {
                        log.warn("Exception finding overlapping tablets, will retry tablet " + ke, ex);
                    }
                    timer.stop(Timers.QUERY_METADATA);
                }
                if (tabletsToAssignMapFileTo.size() > 0)
                    assignments.put(entry.getKey(), tabletsToAssignMapFileTo);
            }
            assignmentStats.attemptingAssignments(assignments);
            Map<Path, List<KeyExtent>> assignmentFailures2 = assignMapFiles(context, conf, fs, tableId, assignments, paths, numAssignThreads, numThreads);
            assignmentStats.assignmentsFailed(assignmentFailures2);
            // merge assignmentFailures2 into assignmentFailures
            for (Entry<Path, List<KeyExtent>> entry : assignmentFailures2.entrySet()) {
                assignmentFailures.get(entry.getKey()).addAll(entry.getValue());
                Integer fc = failureCount.get(entry.getKey());
                if (fc == null)
                    fc = 0;
                failureCount.put(entry.getKey(), fc + 1);
            }
            // remove map files that have no more key extents to assign
            Iterator<Entry<Path, List<KeyExtent>>> afIter = assignmentFailures.entrySet().iterator();
            while (afIter.hasNext()) {
                Entry<Path, List<KeyExtent>> entry = afIter.next();
                if (entry.getValue().size() == 0)
                    afIter.remove();
            }
            Set<Entry<Path, Integer>> failureIter = failureCount.entrySet();
            for (Entry<Path, Integer> entry : failureIter) {
                int retries = context.getConfiguration().getCount(Property.TSERV_BULK_RETRY);
                if (entry.getValue() > retries && assignmentFailures.get(entry.getKey()) != null) {
                    log.error("Map file {} failed more than {} times, giving up.", entry.getKey(), retries);
                    completeFailures.put(entry.getKey(), assignmentFailures.get(entry.getKey()));
                    assignmentFailures.remove(entry.getKey());
                }
            }
        }
        assignmentStats.assignmentsAbandoned(completeFailures);
        Set<Path> failedFailures = processFailures(completeFailures);
        assignmentStats.unrecoveredMapFiles(failedFailures);
        timer.stop(Timers.TOTAL);
        printReport(paths);
        return assignmentStats;
    } finally {
        if (client != null) {
            ServerClient.close(client);
        }
    }
}

Example 2 with VolumeManager

use of org.apache.accumulo.server.fs.VolumeManager in project accumulo by apache.

the class MiniAccumuloClusterImpl method start.

/**
 * Starts Accumulo and Zookeeper processes. Can only be called once.
 */
@Override
public synchronized void start() throws IOException, InterruptedException {
    if (config.useMiniDFS() && miniDFS == null) {
        throw new IllegalStateException("Cannot restart mini when using miniDFS");
    }
    MiniAccumuloClusterControl control = getClusterControl();
    if (config.useExistingInstance()) {
        Configuration acuConf = config.getAccumuloConfiguration();
        Configuration hadoopConf = config.getHadoopConfiguration();
        ConfigurationCopy cc = new ConfigurationCopy(acuConf);
        VolumeManager fs;
        try {
            fs = VolumeManagerImpl.get(cc, hadoopConf);
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
        Path instanceIdPath = Accumulo.getAccumuloInstanceIdPath(fs);
        String instanceIdFromFile = ZooUtil.getInstanceIDFromHdfs(instanceIdPath, cc, hadoopConf);
        IZooReaderWriter zrw = new ZooReaderWriterFactory().getZooReaderWriter(cc.get(Property.INSTANCE_ZK_HOST), (int) cc.getTimeInMillis(Property.INSTANCE_ZK_TIMEOUT), cc.get(Property.INSTANCE_SECRET));
        String rootPath = ZooUtil.getRoot(instanceIdFromFile);
        String instanceName = null;
        try {
            for (String name : zrw.getChildren(Constants.ZROOT + Constants.ZINSTANCES)) {
                String instanceNamePath = Constants.ZROOT + Constants.ZINSTANCES + "/" + name;
                byte[] bytes = zrw.getData(instanceNamePath, new Stat());
                String iid = new String(bytes, UTF_8);
                if (iid.equals(instanceIdFromFile)) {
                    instanceName = name;
                }
            }
        } catch (KeeperException e) {
            throw new RuntimeException("Unable to read instance name from zookeeper.", e);
        }
        if (instanceName == null)
            throw new RuntimeException("Unable to read instance name from zookeeper.");
        config.setInstanceName(instanceName);
        if (!AccumuloStatus.isAccumuloOffline(zrw, rootPath))
            throw new RuntimeException("The Accumulo instance being used is already running. Aborting.");
    } else {
        if (!initialized) {
            Runtime.getRuntime().addShutdownHook(new Thread() {

                @Override
                public void run() {
                    try {
                        MiniAccumuloClusterImpl.this.stop();
                    } catch (IOException e) {
                        log.error("IOException while attempting to stop the MiniAccumuloCluster.", e);
                    } catch (InterruptedException e) {
                        log.error("The stopping of MiniAccumuloCluster was interrupted.", e);
                    }
                }
            });
        }
        if (!config.useExistingZooKeepers())
            control.start(ServerType.ZOOKEEPER);
        if (!initialized) {
            if (!config.useExistingZooKeepers()) {
                // sleep a little bit to let zookeeper come up before calling init, seems to work better
                long startTime = System.currentTimeMillis();
                while (true) {
                    Socket s = null;
                    try {
                        s = new Socket("localhost", config.getZooKeeperPort());
                        s.setReuseAddress(true);
                        s.getOutputStream().write("ruok\n".getBytes());
                        s.getOutputStream().flush();
                        byte[] buffer = new byte[100];
                        int n = s.getInputStream().read(buffer);
                        if (n >= 4 && new String(buffer, 0, 4).equals("imok"))
                            break;
                    } catch (Exception e) {
                        if (System.currentTimeMillis() - startTime >= config.getZooKeeperStartupTime()) {
                            throw new ZooKeeperBindException("Zookeeper did not start within " + (config.getZooKeeperStartupTime() / 1000) + " seconds. Check the logs in " + config.getLogDir() + " for errors.  Last exception: " + e);
                        }
                        // Don't spin absurdly fast
                        Thread.sleep(250);
                    } finally {
                        if (s != null)
                            s.close();
                    }
                }
            }
            LinkedList<String> args = new LinkedList<>();
            args.add("--instance-name");
            args.add(config.getInstanceName());
            args.add("--user");
            args.add(config.getRootUserName());
            args.add("--clear-instance-name");
            // If we aren't using SASL, add in the root password
            final String saslEnabled = config.getSiteConfig().get(Property.INSTANCE_RPC_SASL_ENABLED.getKey());
            if (null == saslEnabled || !Boolean.parseBoolean(saslEnabled)) {
                args.add("--password");
                args.add(config.getRootPassword());
            }
            Process initProcess = exec(Initialize.class, args.toArray(new String[0]));
            int ret = initProcess.waitFor();
            if (ret != 0) {
                throw new RuntimeException("Initialize process returned " + ret + ". Check the logs in " + config.getLogDir() + " for errors.");
            }
            initialized = true;
        }
    }
    log.info("Starting MAC against instance {} and zookeeper(s) {}.", config.getInstanceName(), config.getZooKeepers());
    control.start(ServerType.TABLET_SERVER);
    int ret = 0;
    for (int i = 0; i < 5; i++) {
        ret = exec(Main.class, SetGoalState.class.getName(), MasterGoalState.NORMAL.toString()).waitFor();
        if (ret == 0)
            break;
        sleepUninterruptibly(1, TimeUnit.SECONDS);
    }
    if (ret != 0) {
        throw new RuntimeException("Could not set master goal state, process returned " + ret + ". Check the logs in " + config.getLogDir() + " for errors.");
    }
    control.start(ServerType.MASTER);
    control.start(ServerType.GARBAGE_COLLECTOR);
    if (null == executor) {
        executor = Executors.newSingleThreadExecutor();
    }
}

Example 3 with VolumeManager

use of org.apache.accumulo.server.fs.VolumeManager in project accumulo by apache.

the class Master method main.

public static void main(String[] args) throws Exception {
    try {
        final String app = "master";
        ServerOpts opts = new ServerOpts();
        opts.parseArgs(app, args);
        SecurityUtil.serverLogin(SiteConfiguration.getInstance());
        String hostname = opts.getAddress();
        Instance instance = HdfsZooInstance.getInstance();
        ServerConfigurationFactory conf = new ServerConfigurationFactory(instance);
        VolumeManager fs = VolumeManagerImpl.get();
        MetricsSystemHelper.configure(Master.class.getSimpleName());
        Accumulo.init(fs, instance, conf, app);
        Master master = new Master(instance, conf, fs, hostname);
        DistributedTrace.enable(hostname, app, conf.getSystemConfiguration());
        master.run();
    } catch (Exception ex) {
        log.error("Unexpected exception, exiting", ex);
        System.exit(1);
    } finally {
        DistributedTrace.disable();
    }
}

Example 4 with VolumeManager

use of org.apache.accumulo.server.fs.VolumeManager in project accumulo by apache.

the class SimpleGarbageCollector method main.

public static void main(String[] args) throws IOException {
    final String app = "gc";
    Opts opts = new Opts();
    opts.parseArgs(app, args);
    SecurityUtil.serverLogin(SiteConfiguration.getInstance());
    Instance instance = HdfsZooInstance.getInstance();
    ServerConfigurationFactory conf = new ServerConfigurationFactory(instance);
    log.info("Version " + Constants.VERSION);
    log.info("Instance " + instance.getInstanceID());
    final VolumeManager fs = VolumeManagerImpl.get();
    MetricsSystemHelper.configure(SimpleGarbageCollector.class.getSimpleName());
    Accumulo.init(fs, instance, conf, app);
    SimpleGarbageCollector gc = new SimpleGarbageCollector(opts, instance, fs, conf);
    DistributedTrace.enable(opts.getAddress(), app, conf.getSystemConfiguration());
    try {
        gc.run();
    } finally {
        DistributedTrace.disable();
    }
}

Example 5 with VolumeManager

use of org.apache.accumulo.server.fs.VolumeManager in project accumulo by apache.

the class GarbageCollectWriteAheadLogsTest method deleteUnreferenceLogOnDeadServer.

@Test
public void deleteUnreferenceLogOnDeadServer() throws Exception {
    AccumuloServerContext context = EasyMock.createMock(AccumuloServerContext.class);
    VolumeManager fs = EasyMock.createMock(VolumeManager.class);
    WalStateManager marker = EasyMock.createMock(WalStateManager.class);
    LiveTServerSet tserverSet = EasyMock.createMock(LiveTServerSet.class);
    Connector conn = EasyMock.createMock(Connector.class);
    Scanner mscanner = EasyMock.createMock(Scanner.class);
    Scanner rscanner = EasyMock.createMock(Scanner.class);
    GCStatus status = new GCStatus(null, null, null, new GcCycleStats());
    EasyMock.expect(tserverSet.getCurrentServers()).andReturn(Collections.singleton(server1));
    EasyMock.expect(marker.getAllMarkers()).andReturn(markers2).once();
    EasyMock.expect(marker.state(server2, id)).andReturn(new Pair<>(WalState.OPEN, path));
    EasyMock.expect(context.getConnector()).andReturn(conn);
    EasyMock.expect(conn.createScanner(ReplicationTable.NAME, Authorizations.EMPTY)).andReturn(rscanner);
    rscanner.fetchColumnFamily(ReplicationSchema.StatusSection.NAME);
    EasyMock.expectLastCall().once();
    EasyMock.expect(rscanner.iterator()).andReturn(emptyKV);
    EasyMock.expect(conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY)).andReturn(mscanner);
    mscanner.fetchColumnFamily(MetadataSchema.ReplicationSection.COLF);
    EasyMock.expectLastCall().once();
    mscanner.setRange(MetadataSchema.ReplicationSection.getRange());
    EasyMock.expectLastCall().once();
    EasyMock.expect(mscanner.iterator()).andReturn(emptyKV);
    EasyMock.expect(fs.deleteRecursively(path)).andReturn(true).once();
    marker.removeWalMarker(server2, id);
    EasyMock.expectLastCall().once();
    marker.forget(server2);
    EasyMock.expectLastCall().once();
    EasyMock.replay(context, fs, marker, tserverSet, conn, rscanner, mscanner);
    GarbageCollectWriteAheadLogs gc = new GarbageCollectWriteAheadLogs(context, fs, false, tserverSet, marker, tabletOnServer1List);
    gc.collect(status);
    EasyMock.verify(context, fs, marker, tserverSet, conn, rscanner, mscanner);
}