Examples with VolumeManager org.apache.accumulo.server.fs.VolumeManager used on opensource projects

Example 46 with VolumeManager

use of org.apache.accumulo.server.fs.VolumeManager in project accumulo by apache.

the class CheckForMetadataProblems method main.

public static void main(String[] args) throws Exception {
    ClientOpts opts = new ClientOpts();
    opts.parseArgs(CheckForMetadataProblems.class.getName(), args);
    VolumeManager fs = VolumeManagerImpl.get();
    checkMetadataAndRootTableEntries(RootTable.NAME, opts, fs);
    checkMetadataAndRootTableEntries(MetadataTable.NAME, opts, fs);
    opts.stopTracing();
    if (sawProblems)
        throw new RuntimeException();
}

Example 47 with VolumeManager

use of org.apache.accumulo.server.fs.VolumeManager in project accumulo by apache.

the class RemoveEntriesForMissingFiles method checkTable.

private static int checkTable(ClientContext context, String tableName, Range range, boolean fix) throws Exception {
    @SuppressWarnings({ "rawtypes" }) Map cache = new LRUMap(100000);
    Set<Path> processing = new HashSet<>();
    ExecutorService threadPool = Executors.newFixedThreadPool(16);
    System.out.printf("Scanning : %s %s\n", tableName, range);
    VolumeManager fs = VolumeManagerImpl.get();
    Connector connector = context.getConnector();
    Scanner metadata = connector.createScanner(tableName, Authorizations.EMPTY);
    metadata.setRange(range);
    metadata.fetchColumnFamily(DataFileColumnFamily.NAME);
    int count = 0;
    AtomicInteger missing = new AtomicInteger(0);
    AtomicReference<Exception> exceptionRef = new AtomicReference<>(null);
    BatchWriter writer = null;
    if (fix)
        writer = connector.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig());
    for (Entry<Key, Value> entry : metadata) {
        if (exceptionRef.get() != null)
            break;
        count++;
        Key key = entry.getKey();
        Path map = fs.getFullPath(key);
        synchronized (processing) {
            while (processing.size() >= 64 || processing.contains(map)) processing.wait();
            if (cache.get(map) != null) {
                continue;
            }
            processing.add(map);
        }
        threadPool.submit(new CheckFileTask(cache, fs, missing, writer, key, map, processing, exceptionRef));
    }
    threadPool.shutdown();
    synchronized (processing) {
        while (processing.size() > 0) processing.wait();
    }
    if (exceptionRef.get() != null)
        throw new AccumuloException(exceptionRef.get());
    if (writer != null && missing.get() > 0)
        writer.close();
    System.out.printf("Scan finished, %d files of %d missing\n\n", missing.get(), count);
    return missing.get();
}

Example 48 with VolumeManager

use of org.apache.accumulo.server.fs.VolumeManager in project accumulo by apache.

the class AccumuloTest method testUpdateAccumuloVersion.

@Test
public void testUpdateAccumuloVersion() throws Exception {
    Volume v1 = createMock(Volume.class);
    FileSystem fs1 = createMock(FileSystem.class);
    Path baseVersion1 = new Path("hdfs://volume1/accumulo/version");
    Path oldVersion1 = new Path("hdfs://volume1/accumulo/version/7");
    Path newVersion1 = new Path("hdfs://volume1/accumulo/version/" + Integer.toString(ServerConstants.DATA_VERSION));
    FileStatus[] files1 = mockPersistentVersion("7");
    expect(fs1.listStatus(baseVersion1)).andReturn(files1);
    replay(fs1);
    FSDataOutputStream fsdos1 = createMock(FSDataOutputStream.class);
    expect(v1.getFileSystem()).andReturn(fs1);
    expect(v1.prefixChild(ServerConstants.VERSION_DIR)).andReturn(baseVersion1).times(2);
    replay(v1);
    fsdos1.close();
    replay(fsdos1);
    Volume v2 = createMock(Volume.class);
    FileSystem fs2 = createMock(FileSystem.class);
    Path baseVersion2 = new Path("hdfs://volume2/accumulo/version");
    Path oldVersion2 = new Path("hdfs://volume2/accumulo/version/7");
    Path newVersion2 = new Path("hdfs://volume2/accumulo/version/" + Integer.toString(ServerConstants.DATA_VERSION));
    FileStatus[] files2 = mockPersistentVersion("7");
    expect(fs2.listStatus(baseVersion2)).andReturn(files2);
    replay(fs2);
    FSDataOutputStream fsdos2 = createMock(FSDataOutputStream.class);
    expect(v2.getFileSystem()).andReturn(fs2);
    expect(v2.prefixChild(ServerConstants.VERSION_DIR)).andReturn(baseVersion2).times(2);
    replay(v2);
    fsdos2.close();
    replay(fsdos2);
    VolumeManager vm = createMock(VolumeManager.class);
    expect(vm.getVolumes()).andReturn(Sets.newHashSet(v1, v2));
    expect(vm.delete(oldVersion1)).andReturn(true);
    expect(vm.create(newVersion1)).andReturn(fsdos1);
    expect(vm.delete(oldVersion2)).andReturn(true);
    expect(vm.create(newVersion2)).andReturn(fsdos2);
    replay(vm);
    Accumulo.updateAccumuloVersion(vm, 7);
}

Example 49 with VolumeManager

use of org.apache.accumulo.server.fs.VolumeManager in project accumulo by apache.

the class ImportTableTest method testTabletDir.

@Test
public void testTabletDir() {
    Master master = EasyMock.createMock(Master.class);
    VolumeManager volumeManager = EasyMock.createMock(VolumeManager.class);
    ImportedTableInfo iti = new ImportedTableInfo();
    iti.tableId = Table.ID.of("5");
    // Different volumes with different paths
    String[] tableDirs = new String[] { "hdfs://nn1:8020/apps/accumulo1/tables", "hdfs://nn2:8020/applications/accumulo/tables" };
    // This needs to be unique WRT the importtable command
    String tabletDir = "/c-00000001";
    EasyMock.expect(master.getFileSystem()).andReturn(volumeManager);
    // Choose the 2nd element
    VolumeChooserEnvironment chooserEnv = new VolumeChooserEnvironment(iti.tableId);
    EasyMock.expect(volumeManager.choose(EasyMock.eq(chooserEnv), EasyMock.eq(tableDirs))).andReturn(tableDirs[1]);
    EasyMock.replay(master, volumeManager);
    PopulateMetadataTable pmt = new PopulateMetadataTable(iti);
    Assert.assertEquals(tableDirs[1] + "/" + iti.tableId + "/" + tabletDir, pmt.getClonedTabletDir(master, tableDirs, tabletDir));
    EasyMock.verify(master, volumeManager);
}

Example 50 with VolumeManager

use of org.apache.accumulo.server.fs.VolumeManager in project accumulo by apache.

the class LoadFiles method call.

@Override
public Repo<Master> call(final long tid, final Master master) throws Exception {
    master.updateBulkImportStatus(source, BulkImportState.LOADING);
    ExecutorService executor = getThreadPool(master);
    final AccumuloConfiguration conf = master.getConfiguration();
    VolumeManager fs = master.getFileSystem();
    List<FileStatus> files = new ArrayList<>();
    for (FileStatus entry : fs.listStatus(new Path(bulk))) {
        files.add(entry);
    }
    log.debug("tid " + tid + " importing " + files.size() + " files");
    Path writable = new Path(this.errorDir, ".iswritable");
    if (!fs.createNewFile(writable)) {
        // Maybe this is a re-try... clear the flag and try again
        fs.delete(writable);
        if (!fs.createNewFile(writable))
            throw new AcceptableThriftTableOperationException(tableId.canonicalID(), null, TableOperation.BULK_IMPORT, TableOperationExceptionType.BULK_BAD_ERROR_DIRECTORY, "Unable to write to " + this.errorDir);
    }
    fs.delete(writable);
    final Set<String> filesToLoad = Collections.synchronizedSet(new HashSet<String>());
    for (FileStatus f : files) filesToLoad.add(f.getPath().toString());
    final int RETRIES = Math.max(1, conf.getCount(Property.MASTER_BULK_RETRIES));
    for (int attempt = 0; attempt < RETRIES && filesToLoad.size() > 0; attempt++) {
        List<Future<List<String>>> results = new ArrayList<>();
        if (master.onlineTabletServers().size() == 0)
            log.warn("There are no tablet server to process bulk import, waiting (tid = " + tid + ")");
        while (master.onlineTabletServers().size() == 0) {
            sleepUninterruptibly(500, TimeUnit.MILLISECONDS);
        }
        // Use the threadpool to assign files one-at-a-time to the server
        final List<String> loaded = Collections.synchronizedList(new ArrayList<String>());
        final Random random = new Random();
        final TServerInstance[] servers;
        String prop = conf.get(Property.MASTER_BULK_TSERVER_REGEX);
        if (null == prop || "".equals(prop)) {
            servers = master.onlineTabletServers().toArray(new TServerInstance[0]);
        } else {
            Pattern regex = Pattern.compile(prop);
            List<TServerInstance> subset = new ArrayList<>();
            master.onlineTabletServers().forEach(t -> {
                if (regex.matcher(t.host()).matches()) {
                    subset.add(t);
                }
            });
            if (0 == subset.size()) {
                log.warn("There are no tablet servers online that match supplied regex: {}", conf.get(Property.MASTER_BULK_TSERVER_REGEX));
            }
            servers = subset.toArray(new TServerInstance[0]);
        }
        if (servers.length > 0) {
            for (final String file : filesToLoad) {
                results.add(executor.submit(new Callable<List<String>>() {

                    @Override
                    public List<String> call() {
                        List<String> failures = new ArrayList<>();
                        ClientService.Client client = null;
                        HostAndPort server = null;
                        try {
                            // get a connection to a random tablet server, do not prefer cached connections because
                            // this is running on the master and there are lots of connections to tablet servers
                            // serving the metadata tablets
                            long timeInMillis = master.getConfiguration().getTimeInMillis(Property.MASTER_BULK_TIMEOUT);
                            // Pair<String,Client> pair = ServerClient.getConnection(master, false, timeInMillis);
                            server = servers[random.nextInt(servers.length)].getLocation();
                            client = ThriftUtil.getTServerClient(server, master, timeInMillis);
                            List<String> attempt = Collections.singletonList(file);
                            log.debug("Asking " + server + " to bulk import " + file);
                            List<String> fail = client.bulkImportFiles(Tracer.traceInfo(), master.rpcCreds(), tid, tableId.canonicalID(), attempt, errorDir, setTime);
                            if (fail.isEmpty()) {
                                loaded.add(file);
                            } else {
                                failures.addAll(fail);
                            }
                        } catch (Exception ex) {
                            log.error("rpc failed server:" + server + ", tid:" + tid + " " + ex);
                        } finally {
                            ThriftUtil.returnClient(client);
                        }
                        return failures;
                    }
                }));
            }
        }
        Set<String> failures = new HashSet<>();
        for (Future<List<String>> f : results) failures.addAll(f.get());
        filesToLoad.removeAll(loaded);
        if (filesToLoad.size() > 0) {
            log.debug("tid " + tid + " attempt " + (attempt + 1) + " " + sampleList(filesToLoad, 10) + " failed");
            sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
        }
    }
    FSDataOutputStream failFile = fs.create(new Path(errorDir, BulkImport.FAILURES_TXT), true);
    try (BufferedWriter out = new BufferedWriter(new OutputStreamWriter(failFile, UTF_8))) {
        for (String f : filesToLoad) {
            out.write(f);
            out.write("\n");
        }
    }
    // return the next step, which will perform cleanup
    return new CompleteBulkImport(tableId, source, bulk, errorDir);
}