Examples with VolumeManager org.apache.accumulo.server.fs.VolumeManager used on opensource projects

Example 36 with VolumeManager

use of org.apache.accumulo.server.fs.VolumeManager in project accumulo by apache.

the class Tablet method getFirstAndLastKeys.

private Map<FileRef, Pair<Key, Key>> getFirstAndLastKeys(SortedMap<FileRef, DataFileValue> allFiles) throws IOException {
    Map<FileRef, Pair<Key, Key>> result = new HashMap<>();
    FileOperations fileFactory = FileOperations.getInstance();
    VolumeManager fs = getTabletServer().getFileSystem();
    for (Entry<FileRef, DataFileValue> entry : allFiles.entrySet()) {
        FileRef file = entry.getKey();
        FileSystem ns = fs.getVolumeByPath(file.path()).getFileSystem();
        try (FileSKVIterator openReader = fileFactory.newReaderBuilder().forFile(file.path().toString(), ns, ns.getConf()).withTableConfiguration(this.getTableConfiguration()).seekToBeginning().build()) {
            Key first = openReader.getFirstKey();
            Key last = openReader.getLastKey();
            result.put(file, new Pair<>(first, last));
        }
    }
    return result;
}

Example 37 with VolumeManager

use of org.apache.accumulo.server.fs.VolumeManager in project accumulo by apache.

the class SortedLogRecoveryTest method recover.

private static List<Mutation> recover(Map<String, KeyValue[]> logs, Set<String> files, KeyExtent extent) throws IOException {
    TemporaryFolder root = new TemporaryFolder(new File(System.getProperty("user.dir") + "/target"));
    root.create();
    final String workdir = root.getRoot().getAbsolutePath() + "/workdir";
    VolumeManager fs = VolumeManagerImpl.getLocal(workdir);
    final Path workdirPath = new Path("file://" + workdir);
    fs.deleteRecursively(workdirPath);
    ArrayList<Path> dirs = new ArrayList<>();
    try {
        for (Entry<String, KeyValue[]> entry : logs.entrySet()) {
            String path = workdir + "/" + entry.getKey();
            FileSystem ns = fs.getVolumeByPath(new Path(path)).getFileSystem();
            @SuppressWarnings("deprecation") Writer map = new MapFile.Writer(ns.getConf(), ns, path + "/log1", LogFileKey.class, LogFileValue.class);
            for (KeyValue lfe : entry.getValue()) {
                map.append(lfe.key, lfe.value);
            }
            map.close();
            ns.create(SortedLogState.getFinishedMarkerPath(path)).close();
            dirs.add(new Path(path));
        }
        // Recover
        SortedLogRecovery recovery = new SortedLogRecovery(fs);
        CaptureMutations capture = new CaptureMutations();
        recovery.recover(extent, dirs, files, capture);
        return capture.result;
    } finally {
        root.delete();
    }
}

Example 38 with VolumeManager

use of org.apache.accumulo.server.fs.VolumeManager in project accumulo by apache.

the class ReplicationProcessorTest method noPeerConfigurationThrowsAnException.

@Test(expected = IllegalArgumentException.class)
public void noPeerConfigurationThrowsAnException() {
    Instance inst = EasyMock.createMock(Instance.class);
    VolumeManager fs = EasyMock.createMock(VolumeManager.class);
    Credentials creds = new Credentials("foo", new PasswordToken("bar"));
    ClientContext context = new ClientContext(inst, creds, ClientConfiguration.create());
    Map<String, String> data = new HashMap<>();
    ConfigurationCopy conf = new ConfigurationCopy(data);
    ReplicationProcessor proc = new ReplicationProcessor(context, conf, fs);
    proc.getPeerType("foo");
}

Example 39 with VolumeManager

use of org.apache.accumulo.server.fs.VolumeManager in project accumulo by apache.

the class RootFilesTest method testFileReplacement.

@SuppressWarnings("deprecation")
@Test
public void testFileReplacement() throws IOException {
    ConfigurationCopy conf = new ConfigurationCopy();
    conf.set(Property.INSTANCE_DFS_URI, "file:///");
    conf.set(Property.INSTANCE_DFS_DIR, "/");
    conf.set(Property.GENERAL_VOLUME_CHOOSER, RandomVolumeChooser.class.getName());
    VolumeManager vm = VolumeManagerImpl.get(conf);
    TestWrapper wrapper = new TestWrapper(vm, conf, "A00004.rf", "A00002.rf", "F00003.rf");
    wrapper.prepareReplacement();
    wrapper.renameReplacement();
    wrapper.finishReplacement();
    wrapper.assertFiles("A00004.rf");
    wrapper = new TestWrapper(vm, conf, "A00004.rf", "A00002.rf", "F00003.rf");
    wrapper.prepareReplacement();
    wrapper.cleanupReplacement("A00002.rf", "F00003.rf");
    wrapper.assertFiles("A00002.rf", "F00003.rf");
    wrapper = new TestWrapper(vm, conf, "A00004.rf", "A00002.rf", "F00003.rf");
    wrapper.prepareReplacement();
    wrapper.renameReplacement();
    wrapper.cleanupReplacement("A00004.rf");
    wrapper.assertFiles("A00004.rf");
    wrapper = new TestWrapper(vm, conf, "A00004.rf", "A00002.rf", "F00003.rf");
    wrapper.prepareReplacement();
    wrapper.renameReplacement();
    wrapper.finishReplacement();
    wrapper.cleanupReplacement("A00004.rf");
    wrapper.assertFiles("A00004.rf");
}

Example 40 with VolumeManager

use of org.apache.accumulo.server.fs.VolumeManager in project accumulo by apache.

the class CollectTabletStats method reportHdfsBlockLocations.

private static void reportHdfsBlockLocations(List<FileRef> files) throws Exception {
    VolumeManager fs = VolumeManagerImpl.get();
    System.out.println("\t\tFile block report : ");
    for (FileRef file : files) {
        FileStatus status = fs.getFileStatus(file.path());
        if (status.isDirectory()) {
            // assume it is a map file
            status = fs.getFileStatus(new Path(file + "/data"));
        }
        FileSystem ns = fs.getVolumeByPath(file.path()).getFileSystem();
        BlockLocation[] locs = ns.getFileBlockLocations(status, 0, status.getLen());
        System.out.println("\t\t\tBlocks for : " + file);
        for (BlockLocation blockLocation : locs) {
            System.out.printf("\t\t\t\t offset : %,13d  hosts :", blockLocation.getOffset());
            for (String host : blockLocation.getHosts()) {
                System.out.print(" " + host);
            }
            System.out.println();
        }
    }
    System.out.println();
}