Examples with KeyExtent org.apache.accumulo.core.data.impl.KeyExtent used on opensource projects

Example 1 with KeyExtent

use of org.apache.accumulo.core.data.impl.KeyExtent in project accumulo by apache.

the class SplitRecoveryIT method test.

@Test
public void test() throws Exception {
    String tableName = getUniqueNames(1)[0];
    for (int tn = 0; tn < 2; tn++) {
        Connector connector = getConnector();
        // create a table and put some data in it
        connector.tableOperations().create(tableName);
        BatchWriter bw = connector.createBatchWriter(tableName, new BatchWriterConfig());
        bw.addMutation(m("a"));
        bw.addMutation(m("b"));
        bw.addMutation(m("c"));
        bw.close();
        // take the table offline
        connector.tableOperations().offline(tableName);
        while (!isOffline(tableName, connector)) sleepUninterruptibly(200, TimeUnit.MILLISECONDS);
        // poke a partial split into the metadata table
        connector.securityOperations().grantTablePermission(getAdminPrincipal(), MetadataTable.NAME, TablePermission.WRITE);
        Table.ID tableId = Table.ID.of(connector.tableOperations().tableIdMap().get(tableName));
        KeyExtent extent = new KeyExtent(tableId, null, new Text("b"));
        Mutation m = extent.getPrevRowUpdateMutation();
        TabletsSection.TabletColumnFamily.SPLIT_RATIO_COLUMN.put(m, new Value(Double.toString(0.5).getBytes()));
        TabletsSection.TabletColumnFamily.OLD_PREV_ROW_COLUMN.put(m, KeyExtent.encodePrevEndRow(null));
        bw = connector.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig());
        bw.addMutation(m);
        if (tn == 1) {
            bw.flush();
            try (Scanner scanner = connector.createScanner(MetadataTable.NAME, Authorizations.EMPTY)) {
                scanner.setRange(extent.toMetadataRange());
                scanner.fetchColumnFamily(DataFileColumnFamily.NAME);
                KeyExtent extent2 = new KeyExtent(tableId, new Text("b"), null);
                m = extent2.getPrevRowUpdateMutation();
                TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value("/t2".getBytes()));
                TabletsSection.ServerColumnFamily.TIME_COLUMN.put(m, new Value("M0".getBytes()));
                for (Entry<Key, Value> entry : scanner) {
                    m.put(DataFileColumnFamily.NAME, entry.getKey().getColumnQualifier(), entry.getValue());
                }
                bw.addMutation(m);
            }
        }
        bw.close();
        // bring the table online
        connector.tableOperations().online(tableName);
        // verify the tablets went online
        try (Scanner scanner = connector.createScanner(tableName, Authorizations.EMPTY)) {
            int i = 0;
            String[] expected = { "a", "b", "c" };
            for (Entry<Key, Value> entry : scanner) {
                assertEquals(expected[i], entry.getKey().getRow().toString());
                i++;
            }
            assertEquals(3, i);
            connector.tableOperations().delete(tableName);
        }
    }
}

Example 2 with KeyExtent

use of org.apache.accumulo.core.data.impl.KeyExtent in project accumulo by apache.

the class VolumeIT method testRelativePaths.

@Test
public void testRelativePaths() throws Exception {
    List<String> expected = new ArrayList<>();
    Connector connector = getConnector();
    String tableName = getUniqueNames(1)[0];
    connector.tableOperations().create(tableName, new NewTableConfiguration().withoutDefaultIterators());
    Table.ID tableId = Table.ID.of(connector.tableOperations().tableIdMap().get(tableName));
    SortedSet<Text> partitions = new TreeSet<>();
    // with some splits
    for (String s : "c,g,k,p,s,v".split(",")) partitions.add(new Text(s));
    connector.tableOperations().addSplits(tableName, partitions);
    BatchWriter bw = connector.createBatchWriter(tableName, new BatchWriterConfig());
    // create two files in each tablet
    String[] rows = "a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,w,x,y,z".split(",");
    for (String s : rows) {
        Mutation m = new Mutation(s);
        m.put("cf1", "cq1", "1");
        bw.addMutation(m);
        expected.add(s + ":cf1:cq1:1");
    }
    bw.flush();
    connector.tableOperations().flush(tableName, null, null, true);
    for (String s : rows) {
        Mutation m = new Mutation(s);
        m.put("cf1", "cq1", "2");
        bw.addMutation(m);
        expected.add(s + ":cf1:cq1:2");
    }
    bw.close();
    connector.tableOperations().flush(tableName, null, null, true);
    verifyData(expected, connector.createScanner(tableName, Authorizations.EMPTY));
    connector.tableOperations().offline(tableName, true);
    connector.securityOperations().grantTablePermission("root", MetadataTable.NAME, TablePermission.WRITE);
    try (Scanner metaScanner = connector.createScanner(MetadataTable.NAME, Authorizations.EMPTY)) {
        metaScanner.fetchColumnFamily(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME);
        metaScanner.setRange(new KeyExtent(tableId, null, null).toMetadataRange());
        BatchWriter mbw = connector.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig());
        for (Entry<Key, Value> entry : metaScanner) {
            String cq = entry.getKey().getColumnQualifier().toString();
            if (cq.startsWith(v1.toString())) {
                Path path = new Path(cq);
                String relPath = "/" + path.getParent().getName() + "/" + path.getName();
                Mutation fileMut = new Mutation(entry.getKey().getRow());
                fileMut.putDelete(entry.getKey().getColumnFamily(), entry.getKey().getColumnQualifier());
                fileMut.put(entry.getKey().getColumnFamily().toString(), relPath, entry.getValue().toString());
                mbw.addMutation(fileMut);
            }
        }
        mbw.close();
        connector.tableOperations().online(tableName, true);
        verifyData(expected, connector.createScanner(tableName, Authorizations.EMPTY));
        connector.tableOperations().compact(tableName, null, null, true, true);
        verifyData(expected, connector.createScanner(tableName, Authorizations.EMPTY));
        for (Entry<Key, Value> entry : metaScanner) {
            String cq = entry.getKey().getColumnQualifier().toString();
            Path path = new Path(cq);
            Assert.assertTrue("relative path not deleted " + path.toString(), path.depth() > 2);
        }
    }
}

Example 3 with KeyExtent

use of org.apache.accumulo.core.data.impl.KeyExtent in project accumulo by apache.

the class VolumeIT method verifyVolumesUsed.

private void verifyVolumesUsed(String tableName, boolean shouldExist, Path... paths) throws Exception {
    Connector conn = getConnector();
    List<String> expected = new ArrayList<>();
    for (int i = 0; i < 100; i++) {
        String row = String.format("%06d", i * 100 + 3);
        expected.add(row + ":cf1:cq1:1");
    }
    if (!conn.tableOperations().exists(tableName)) {
        Assert.assertFalse(shouldExist);
        writeData(tableName, conn);
        verifyData(expected, conn.createScanner(tableName, Authorizations.EMPTY));
        conn.tableOperations().flush(tableName, null, null, true);
    }
    verifyData(expected, conn.createScanner(tableName, Authorizations.EMPTY));
    Table.ID tableId = Table.ID.of(conn.tableOperations().tableIdMap().get(tableName));
    try (Scanner metaScanner = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY)) {
        MetadataSchema.TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.fetch(metaScanner);
        metaScanner.fetchColumnFamily(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME);
        metaScanner.setRange(new KeyExtent(tableId, null, null).toMetadataRange());
        int[] counts = new int[paths.length];
        outer: for (Entry<Key, Value> entry : metaScanner) {
            String cf = entry.getKey().getColumnFamily().toString();
            String cq = entry.getKey().getColumnQualifier().toString();
            String path;
            if (cf.equals(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME.toString()))
                path = cq;
            else
                path = entry.getValue().toString();
            for (int i = 0; i < paths.length; i++) {
                if (path.startsWith(paths[i].toString())) {
                    counts[i]++;
                    continue outer;
                }
            }
            Assert.fail("Unexpected volume " + path);
        }
        // keep retrying until WAL state information in ZooKeeper stabilizes or until test times out
        retry: while (true) {
            Instance i = conn.getInstance();
            ZooReaderWriter zk = new ZooReaderWriter(i.getZooKeepers(), i.getZooKeepersSessionTimeOut(), "");
            WalStateManager wals = new WalStateManager(i, zk);
            try {
                outer: for (Entry<Path, WalState> entry : wals.getAllState().entrySet()) {
                    for (Path path : paths) {
                        if (entry.getKey().toString().startsWith(path.toString())) {
                            continue outer;
                        }
                    }
                    log.warn("Unexpected volume " + entry.getKey() + " (" + entry.getValue() + ")");
                    continue retry;
                }
            } catch (WalMarkerException e) {
                Throwable cause = e.getCause();
                if (cause instanceof NoNodeException) {
                    // ignore WALs being cleaned up
                    continue retry;
                }
                throw e;
            }
            break;
        }
        // if a volume is chosen randomly for each tablet, then the probability that a volume will not be chosen for any tablet is ((num_volumes -
        // 1)/num_volumes)^num_tablets. For 100 tablets and 3 volumes the probability that only 2 volumes would be chosen is 2.46e-18
        int sum = 0;
        for (int count : counts) {
            Assert.assertTrue(count > 0);
            sum += count;
        }
        Assert.assertEquals(200, sum);
    }
}

Example 4 with KeyExtent

use of org.apache.accumulo.core.data.impl.KeyExtent in project accumulo by apache.

the class BulkImporter method assignMapFiles.

private Map<Path, List<KeyExtent>> assignMapFiles(String tableName, Map<Path, List<AssignmentInfo>> assignments, Map<KeyExtent, String> locations, int numThreads) {
    // group assignments by tablet
    Map<KeyExtent, List<PathSize>> assignmentsPerTablet = new TreeMap<>();
    for (Entry<Path, List<AssignmentInfo>> entry : assignments.entrySet()) {
        Path mapFile = entry.getKey();
        List<AssignmentInfo> tabletsToAssignMapFileTo = entry.getValue();
        for (AssignmentInfo ai : tabletsToAssignMapFileTo) {
            List<PathSize> mapFiles = assignmentsPerTablet.get(ai.ke);
            if (mapFiles == null) {
                mapFiles = new ArrayList<>();
                assignmentsPerTablet.put(ai.ke, mapFiles);
            }
            mapFiles.add(new PathSize(mapFile, ai.estSize));
        }
    }
    // group assignments by tabletserver
    Map<Path, List<KeyExtent>> assignmentFailures = Collections.synchronizedMap(new TreeMap<Path, List<KeyExtent>>());
    TreeMap<String, Map<KeyExtent, List<PathSize>>> assignmentsPerTabletServer = new TreeMap<>();
    for (Entry<KeyExtent, List<PathSize>> entry : assignmentsPerTablet.entrySet()) {
        KeyExtent ke = entry.getKey();
        String location = locations.get(ke);
        if (location == null) {
            for (PathSize pathSize : entry.getValue()) {
                synchronized (assignmentFailures) {
                    List<KeyExtent> failures = assignmentFailures.get(pathSize.path);
                    if (failures == null) {
                        failures = new ArrayList<>();
                        assignmentFailures.put(pathSize.path, failures);
                    }
                    failures.add(ke);
                }
            }
            log.warn("Could not assign {} map files to tablet {} because it had no location, will retry ...", entry.getValue().size(), ke);
            continue;
        }
        Map<KeyExtent, List<PathSize>> apt = assignmentsPerTabletServer.get(location);
        if (apt == null) {
            apt = new TreeMap<>();
            assignmentsPerTabletServer.put(location, apt);
        }
        apt.put(entry.getKey(), entry.getValue());
    }
    ExecutorService threadPool = Executors.newFixedThreadPool(numThreads, new NamingThreadFactory("submit"));
    for (Entry<String, Map<KeyExtent, List<PathSize>>> entry : assignmentsPerTabletServer.entrySet()) {
        String location = entry.getKey();
        threadPool.submit(new AssignmentTask(assignmentFailures, tableName, location, entry.getValue()));
    }
    threadPool.shutdown();
    while (!threadPool.isTerminated()) {
        try {
            threadPool.awaitTermination(60, TimeUnit.SECONDS);
        } catch (InterruptedException e) {
            log.error("Encountered InterruptedException while waiting for the thread pool to terminate.", e);
            throw new RuntimeException(e);
        }
    }
    return assignmentFailures;
}

Example 5 with KeyExtent

use of org.apache.accumulo.core.data.impl.KeyExtent in project accumulo by apache.

the class BulkImporter method assignMapFiles.

private List<KeyExtent> assignMapFiles(ClientContext context, HostAndPort location, Map<KeyExtent, List<PathSize>> assignmentsPerTablet) throws AccumuloException, AccumuloSecurityException {
    try {
        long timeInMillis = context.getConfiguration().getTimeInMillis(Property.TSERV_BULK_TIMEOUT);
        TabletClientService.Iface client = ThriftUtil.getTServerClient(location, context, timeInMillis);
        try {
            HashMap<KeyExtent, Map<String, org.apache.accumulo.core.data.thrift.MapFileInfo>> files = new HashMap<>();
            for (Entry<KeyExtent, List<PathSize>> entry : assignmentsPerTablet.entrySet()) {
                HashMap<String, org.apache.accumulo.core.data.thrift.MapFileInfo> tabletFiles = new HashMap<>();
                files.put(entry.getKey(), tabletFiles);
                for (PathSize pathSize : entry.getValue()) {
                    org.apache.accumulo.core.data.thrift.MapFileInfo mfi = new org.apache.accumulo.core.data.thrift.MapFileInfo(pathSize.estSize);
                    tabletFiles.put(pathSize.path.toString(), mfi);
                }
            }
            log.debug("Asking {} to bulk load {}", location, files);
            List<TKeyExtent> failures = client.bulkImport(Tracer.traceInfo(), context.rpcCreds(), tid, Translator.translate(files, Translators.KET), setTime);
            return Translator.translate(failures, Translators.TKET);
        } finally {
            ThriftUtil.returnClient((TServiceClient) client);
        }
    } catch (ThriftSecurityException e) {
        throw new AccumuloSecurityException(e.user, e.code, e);
    } catch (Throwable t) {
        log.error("Encountered unknown exception in assignMapFiles.", t);
        throw new AccumuloException(t);
    }
}