Examples with TableId org.apache.accumulo.core.data.TableId used on opensource projects

Example 76 with TableId

use of org.apache.accumulo.core.data.TableId in project accumulo by apache.

the class SplitRecoveryIT method test.

@Test
public void test() throws Exception {
    String tableName = getUniqueNames(1)[0];
    try (AccumuloClient client = Accumulo.newClient().from(getClientProps()).build()) {
        for (int tn = 0; tn < 2; tn++) {
            // create a table and put some data in it
            client.tableOperations().create(tableName);
            try (BatchWriter bw = client.createBatchWriter(tableName)) {
                bw.addMutation(m("a"));
                bw.addMutation(m("b"));
                bw.addMutation(m("c"));
            }
            // take the table offline
            client.tableOperations().offline(tableName);
            while (!isOffline(tableName, client)) sleepUninterruptibly(200, MILLISECONDS);
            // poke a partial split into the metadata table
            client.securityOperations().grantTablePermission(getAdminPrincipal(), MetadataTable.NAME, TablePermission.WRITE);
            TableId tableId = TableId.of(client.tableOperations().tableIdMap().get(tableName));
            KeyExtent extent = new KeyExtent(tableId, null, new Text("b"));
            Mutation m = TabletColumnFamily.createPrevRowMutation(extent);
            TabletColumnFamily.SPLIT_RATIO_COLUMN.put(m, new Value(Double.toString(0.5)));
            TabletColumnFamily.OLD_PREV_ROW_COLUMN.put(m, TabletColumnFamily.encodePrevEndRow(null));
            try (BatchWriter bw = client.createBatchWriter(MetadataTable.NAME)) {
                bw.addMutation(m);
                if (tn == 1) {
                    bw.flush();
                    try (Scanner scanner = client.createScanner(MetadataTable.NAME)) {
                        scanner.setRange(extent.toMetaRange());
                        scanner.fetchColumnFamily(DataFileColumnFamily.NAME);
                        KeyExtent extent2 = new KeyExtent(tableId, new Text("b"), null);
                        m = TabletColumnFamily.createPrevRowMutation(extent2);
                        ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value("t2"));
                        ServerColumnFamily.TIME_COLUMN.put(m, new Value("M0"));
                        for (Entry<Key, Value> entry : scanner) {
                            m.put(DataFileColumnFamily.NAME, entry.getKey().getColumnQualifier(), entry.getValue());
                        }
                        bw.addMutation(m);
                    }
                }
            }
            // bring the table online
            client.tableOperations().online(tableName);
            // verify the tablets went online
            try (Scanner scanner = client.createScanner(tableName)) {
                int i = 0;
                String[] expected = { "a", "b", "c" };
                for (Entry<Key, Value> entry : scanner) {
                    assertEquals(expected[i], entry.getKey().getRow().toString());
                    i++;
                }
                assertEquals(3, i);
                client.tableOperations().delete(tableName);
            }
        }
    }
}

Example 77 with TableId

use of org.apache.accumulo.core.data.TableId in project accumulo by apache.

the class MissingWalHeaderCompletesRecoveryIT method testPartialHeaderWalRecoveryCompletes.

@Test
public void testPartialHeaderWalRecoveryCompletes() throws Exception {
    try (AccumuloClient client = Accumulo.newClient().from(getClientProperties()).build()) {
        MiniAccumuloClusterImpl cluster = getCluster();
        FileSystem fs = getCluster().getFileSystem();
        // Fake out something that looks like host:port, it's irrelevant
        String fakeServer = "127.0.0.1:12345";
        File walogs = new File(cluster.getConfig().getAccumuloDir(), Constants.WAL_DIR);
        File walogServerDir = new File(walogs, fakeServer.replace(':', '+'));
        File partialHeaderWalog = new File(walogServerDir, UUID.randomUUID().toString());
        log.info("Created WAL with malformed header at {}", partialHeaderWalog.toURI());
        // Write half of the header
        FSDataOutputStream wal = fs.create(new Path(partialHeaderWalog.toURI()));
        wal.write(DfsLogger.LOG_FILE_HEADER_V4.getBytes(UTF_8), 0, DfsLogger.LOG_FILE_HEADER_V4.length() / 2);
        wal.close();
        assertTrue("root user did not have write permission to metadata table", client.securityOperations().hasTablePermission("root", MetadataTable.NAME, TablePermission.WRITE));
        String tableName = getUniqueNames(1)[0];
        client.tableOperations().create(tableName);
        TableId tableId = TableId.of(client.tableOperations().tableIdMap().get(tableName));
        assertNotNull("Table ID was null", tableId);
        LogEntry logEntry = new LogEntry(null, 0, partialHeaderWalog.toURI().toString());
        log.info("Taking {} offline", tableName);
        client.tableOperations().offline(tableName, true);
        log.info("{} is offline", tableName);
        Text row = TabletsSection.encodeRow(tableId, null);
        Mutation m = new Mutation(row);
        m.put(logEntry.getColumnFamily(), logEntry.getColumnQualifier(), logEntry.getValue());
        try (BatchWriter bw = client.createBatchWriter(MetadataTable.NAME)) {
            bw.addMutation(m);
        }
        log.info("Bringing {} online", tableName);
        client.tableOperations().online(tableName, true);
        log.info("{} is online", tableName);
        // otherwise the tablet will never come online and we won't be able to read it.
        try (Scanner s = client.createScanner(tableName, Authorizations.EMPTY)) {
            assertEquals(0, Iterables.size(s));
        }
    }
}

Example 78 with TableId

use of org.apache.accumulo.core.data.TableId in project accumulo by apache.

the class MissingWalHeaderCompletesRecoveryIT method testEmptyWalRecoveryCompletes.

@Test
public void testEmptyWalRecoveryCompletes() throws Exception {
    try (AccumuloClient client = Accumulo.newClient().from(getClientProperties()).build()) {
        MiniAccumuloClusterImpl cluster = getCluster();
        FileSystem fs = cluster.getFileSystem();
        // Fake out something that looks like host:port, it's irrelevant
        String fakeServer = "127.0.0.1:12345";
        File walogs = new File(cluster.getConfig().getAccumuloDir(), Constants.WAL_DIR);
        File walogServerDir = new File(walogs, fakeServer.replace(':', '+'));
        File emptyWalog = new File(walogServerDir, UUID.randomUUID().toString());
        log.info("Created empty WAL at {}", emptyWalog.toURI());
        fs.create(new Path(emptyWalog.toURI())).close();
        assertTrue("root user did not have write permission to metadata table", client.securityOperations().hasTablePermission("root", MetadataTable.NAME, TablePermission.WRITE));
        String tableName = getUniqueNames(1)[0];
        client.tableOperations().create(tableName);
        TableId tableId = TableId.of(client.tableOperations().tableIdMap().get(tableName));
        assertNotNull("Table ID was null", tableId);
        LogEntry logEntry = new LogEntry(new KeyExtent(tableId, null, null), 0, emptyWalog.toURI().toString());
        log.info("Taking {} offline", tableName);
        client.tableOperations().offline(tableName, true);
        log.info("{} is offline", tableName);
        Text row = TabletsSection.encodeRow(tableId, null);
        Mutation m = new Mutation(row);
        m.put(logEntry.getColumnFamily(), logEntry.getColumnQualifier(), logEntry.getValue());
        try (BatchWriter bw = client.createBatchWriter(MetadataTable.NAME)) {
            bw.addMutation(m);
        }
        log.info("Bringing {} online", tableName);
        client.tableOperations().online(tableName, true);
        log.info("{} is online", tableName);
        // otherwise the tablet will never come online and we won't be able to read it.
        try (Scanner s = client.createScanner(tableName, Authorizations.EMPTY)) {
            assertEquals(0, Iterables.size(s));
        }
    }
}

Example 79 with TableId

use of org.apache.accumulo.core.data.TableId in project accumulo by apache.

the class VolumeIT method verifyVolumesUsed.

private void verifyVolumesUsed(AccumuloClient client, String tableName, boolean shouldExist, Path... paths) throws Exception {
    if (!client.tableOperations().exists(tableName)) {
        assertFalse(shouldExist);
        writeData(tableName, client);
        verifyData(expected, client.createScanner(tableName, Authorizations.EMPTY));
        client.tableOperations().flush(tableName, null, null, true);
    }
    verifyData(expected, client.createScanner(tableName, Authorizations.EMPTY));
    TableId tableId = TableId.of(client.tableOperations().tableIdMap().get(tableName));
    try (Scanner metaScanner = client.createScanner(MetadataTable.NAME, Authorizations.EMPTY)) {
        metaScanner.fetchColumnFamily(DataFileColumnFamily.NAME);
        metaScanner.setRange(new KeyExtent(tableId, null, null).toMetaRange());
        int[] counts = new int[paths.length];
        outer: for (Entry<Key, Value> entry : metaScanner) {
            String path = entry.getKey().getColumnQualifier().toString();
            for (int i = 0; i < paths.length; i++) {
                if (path.startsWith(paths[i].toString())) {
                    counts[i]++;
                    continue outer;
                }
            }
            fail("Unexpected volume " + path);
        }
        // keep retrying until WAL state information in ZooKeeper stabilizes or until test times out
        retry: while (true) {
            WalStateManager wals = new WalStateManager(getServerContext());
            try {
                outer: for (Entry<Path, WalState> entry : wals.getAllState().entrySet()) {
                    for (Path path : paths) {
                        if (entry.getKey().toString().startsWith(path.toString())) {
                            continue outer;
                        }
                    }
                    log.warn("Unexpected volume " + entry.getKey() + " (" + entry.getValue() + ")");
                    continue retry;
                }
            } catch (WalMarkerException e) {
                Throwable cause = e.getCause();
                if (cause instanceof NoNodeException) {
                    // ignore WALs being cleaned up
                    continue retry;
                }
                throw e;
            }
            break;
        }
        // if a volume is chosen randomly for each tablet, then the probability that a volume will not
        // be chosen for any tablet is ((num_volumes -
        // 1)/num_volumes)^num_tablets. For 100 tablets and 3 volumes the probability that only 2
        // volumes would be chosen is 2.46e-18
        int sum = 0;
        for (int count : counts) {
            assertTrue(count > 0);
            sum += count;
        }
        assertEquals(100, sum);
    }
}

Example 80 with TableId

use of org.apache.accumulo.core.data.TableId in project accumulo by apache.

the class InMemoryMapIT method assertEquivalentMutate.

private void assertEquivalentMutate(List<Mutation> mutations) {
    InMemoryMap defaultMap = null;
    InMemoryMap nativeMapWrapper = null;
    InMemoryMap localityGroupMap = null;
    InMemoryMap localityGroupMapWithNative = null;
    try {
        Map<String, String> defaultMapConfig = new HashMap<>();
        defaultMapConfig.put(Property.TSERV_NATIVEMAP_ENABLED.getKey(), "false");
        defaultMapConfig.put(Property.TSERV_MEMDUMP_DIR.getKey(), tempFolder.newFolder().getAbsolutePath());
        defaultMapConfig.put(Property.TABLE_LOCALITY_GROUPS.getKey(), "");
        Map<String, String> nativeMapConfig = new HashMap<>();
        nativeMapConfig.put(Property.TSERV_NATIVEMAP_ENABLED.getKey(), "true");
        nativeMapConfig.put(Property.TSERV_MEMDUMP_DIR.getKey(), tempFolder.newFolder().getAbsolutePath());
        nativeMapConfig.put(Property.TABLE_LOCALITY_GROUPS.getKey(), "");
        Map<String, String> localityGroupConfig = new HashMap<>();
        localityGroupConfig.put(Property.TSERV_NATIVEMAP_ENABLED.getKey(), "false");
        localityGroupConfig.put(Property.TSERV_MEMDUMP_DIR.getKey(), tempFolder.newFolder().getAbsolutePath());
        Map<String, String> localityGroupNativeConfig = new HashMap<>();
        localityGroupNativeConfig.put(Property.TSERV_NATIVEMAP_ENABLED.getKey(), "true");
        localityGroupNativeConfig.put(Property.TSERV_MEMDUMP_DIR.getKey(), tempFolder.newFolder().getAbsolutePath());
        TableId testId = TableId.of("TEST");
        defaultMap = new InMemoryMap(new ConfigurationCopy(defaultMapConfig), getServerContext(), testId);
        nativeMapWrapper = new InMemoryMap(new ConfigurationCopy(nativeMapConfig), getServerContext(), testId);
        localityGroupMap = new InMemoryMap(updateConfigurationForLocalityGroups(new ConfigurationCopy(localityGroupConfig)), getServerContext(), testId);
        localityGroupMapWithNative = new InMemoryMap(updateConfigurationForLocalityGroups(new ConfigurationCopy(localityGroupNativeConfig)), getServerContext(), testId);
    } catch (Exception e) {
        log.error("Error getting new InMemoryMap ", e);
        fail(e.getMessage());
    }
    // ensure the maps are correct type
    assertEquals("Not a DefaultMap", InMemoryMap.TYPE_DEFAULT_MAP, defaultMap.getMapType());
    assertEquals("Not a NativeMapWrapper", InMemoryMap.TYPE_NATIVE_MAP_WRAPPER, nativeMapWrapper.getMapType());
    assertEquals("Not a LocalityGroupMap", InMemoryMap.TYPE_LOCALITY_GROUP_MAP, localityGroupMap.getMapType());
    assertEquals("Not a LocalityGroupMap with native", InMemoryMap.TYPE_LOCALITY_GROUP_MAP_NATIVE, localityGroupMapWithNative.getMapType());
    int count = 0;
    for (Mutation m : mutations) {
        count += m.size();
    }
    defaultMap.mutate(mutations, count);
    nativeMapWrapper.mutate(mutations, count);
    localityGroupMap.mutate(mutations, count);
    localityGroupMapWithNative.mutate(mutations, count);
    // let's use the transitive property to assert all four are equivalent
    assertMutatesEquivalent(mutations, defaultMap, nativeMapWrapper);
    assertMutatesEquivalent(mutations, defaultMap, localityGroupMap);
    assertMutatesEquivalent(mutations, defaultMap, localityGroupMapWithNative);
}