Examples with IntIndexedContainer com.carrotsearch.hppc.IntIndexedContainer used on opensource projects

Example 26 with IntIndexedContainer

use of com.carrotsearch.hppc.IntIndexedContainer in project crate by crate.

the class ShardCollectSource method getIterators.

private List<CompletableFuture<BatchIterator<Row>>> getIterators(CollectTask collectTask, RoutedCollectPhase collectPhase, boolean requiresScroll, Map<String, IntIndexedContainer> indexShards) {
    Metadata metadata = clusterService.state().metadata();
    List<CompletableFuture<BatchIterator<Row>>> iterators = new ArrayList<>();
    for (Map.Entry<String, IntIndexedContainer> entry : indexShards.entrySet()) {
        String indexName = entry.getKey();
        IndexMetadata indexMD = metadata.index(indexName);
        if (indexMD == null) {
            if (IndexParts.isPartitioned(indexName)) {
                continue;
            }
            throw new IndexNotFoundException(indexName);
        }
        Index index = indexMD.getIndex();
        try {
            indicesService.indexServiceSafe(index);
        } catch (IndexNotFoundException e) {
            if (IndexParts.isPartitioned(indexName)) {
                continue;
            }
            throw e;
        }
        for (IntCursor shardCursor : entry.getValue()) {
            ShardId shardId = new ShardId(index, shardCursor.value);
            try {
                ShardCollectorProvider shardCollectorProvider = getCollectorProviderSafe(shardId);
                CompletableFuture<BatchIterator<Row>> iterator = shardCollectorProvider.getFutureIterator(collectPhase, requiresScroll, collectTask);
                iterators.add(iterator);
            } catch (ShardNotFoundException | IllegalIndexShardStateException e) {
                // and the reader required in the fetchPhase would be missing.
                if (Symbols.containsColumn(collectPhase.toCollect(), DocSysColumns.FETCHID)) {
                    throw e;
                }
                iterators.add(remoteCollectorFactory.createCollector(shardId, collectPhase, collectTask, shardCollectorProviderFactory));
            } catch (IndexNotFoundException e) {
                // Prevent wrapping this to not break retry-detection
                throw e;
            } catch (Throwable t) {
                Exceptions.rethrowRuntimeException(t);
            }
        }
    }
    return iterators;
}

Example 27 with IntIndexedContainer

use of com.carrotsearch.hppc.IntIndexedContainer in project crate by crate.

the class SystemCollectSource method getIterator.

@Override
public CompletableFuture<BatchIterator<Row>> getIterator(TransactionContext txnCtx, CollectPhase phase, CollectTask collectTask, boolean supportMoveToStart) {
    RoutedCollectPhase collectPhase = (RoutedCollectPhase) phase;
    Map<String, Map<String, IntIndexedContainer>> locations = collectPhase.routing().locations();
    String table = Iterables.getOnlyElement(locations.get(clusterService.localNode().getId()).keySet());
    RelationName relationName = RelationName.fromIndexName(table);
    StaticTableDefinition<?> tableDefinition = tableDefinition(relationName);
    User user = requireNonNull(userLookup.findUser(txnCtx.sessionSettings().userName()), "User who invoked a statement must exist");
    return CompletableFuture.completedFuture(CollectingBatchIterator.newInstance(() -> {
    }, // If data is already local, then `CollectingBatchIterator` takes care of kill handling.
    t -> {
    }, () -> tableDefinition.retrieveRecords(txnCtx, user).thenApply(records -> recordsToRows(collectPhase, collectTask.txnCtx(), tableDefinition.getReferenceResolver(), supportMoveToStart, records)), tableDefinition.involvesIO()));
}

Example 28 with IntIndexedContainer

use of com.carrotsearch.hppc.IntIndexedContainer in project crate by crate.

the class SchemasITest method testShardsTable.

@Test
public void testShardsTable() throws Exception {
    execute("create table t2 (id int primary key) clustered into 4 shards with(number_of_replicas=0)");
    execute("create table t3 (id int primary key) clustered into 8 shards with(number_of_replicas=0)");
    ensureYellow();
    TableInfo ti = schemas.getTableInfo(new RelationName("sys", "shards"));
    ClusterService clusterService = clusterService();
    Routing routing = ti.getRouting(clusterService.state(), routingProvider, null, null, SessionContext.systemSessionContext());
    Set<String> tables = new HashSet<>();
    Set<String> expectedTables = Set.of(getFqn("t2"), getFqn("t3"));
    int numShards = 0;
    for (Map.Entry<String, Map<String, IntIndexedContainer>> nodeEntry : routing.locations().entrySet()) {
        for (Map.Entry<String, IntIndexedContainer> indexEntry : nodeEntry.getValue().entrySet()) {
            tables.add(indexEntry.getKey());
            numShards += indexEntry.getValue().size();
        }
    }
    assertThat(numShards, is(12));
    assertThat(tables, is(expectedTables));
}

Example 29 with IntIndexedContainer

use of com.carrotsearch.hppc.IntIndexedContainer in project crate by crate.

the class SchemasITest method testDocTable.

@Test
public void testDocTable() {
    execute("create table t1 (" + "id int primary key, " + "name string, " + "CONSTRAINT not_miguel CHECK (name != 'miguel'), " + "details object(dynamic) as (size byte, created timestamp with time zone)" + ") clustered into 10 shards with (number_of_replicas=1)");
    ensureYellow();
    DocTableInfo ti = schemas.getTableInfo(new RelationName(sqlExecutor.getCurrentSchema(), "t1"));
    assertThat(ti.ident().name(), is("t1"));
    assertThat(ti.columns().size(), is(3));
    assertThat(ti.primaryKey().size(), is(1));
    assertThat(ti.primaryKey().get(0), is(new ColumnIdent("id")));
    assertThat(ti.clusteredBy(), is(new ColumnIdent("id")));
    List<CheckConstraint<Symbol>> checkConstraints = ti.checkConstraints();
    assertEquals(1, checkConstraints.size());
    assertEquals(checkConstraints.get(0).name(), "not_miguel");
    assertThat(checkConstraints.get(0).expressionStr(), equalTo("\"name\" <> 'miguel'"));
    ClusterService clusterService = clusterService();
    Routing routing = ti.getRouting(clusterService.state(), routingProvider, WhereClause.MATCH_ALL, RoutingProvider.ShardSelection.ANY, SessionContext.systemSessionContext());
    Set<String> nodes = routing.nodes();
    // for the rare case
    assertThat(nodes.size(), isOneOf(1, 2));
    // where all shards are on 1 node
    int numShards = 0;
    for (Map.Entry<String, Map<String, IntIndexedContainer>> nodeEntry : routing.locations().entrySet()) {
        for (Map.Entry<String, IntIndexedContainer> indexEntry : nodeEntry.getValue().entrySet()) {
            assertThat(indexEntry.getKey(), is(getFqn("t1")));
            numShards += indexEntry.getValue().size();
        }
    }
    assertThat(numShards, is(10));
}

Example 30 with IntIndexedContainer

use of com.carrotsearch.hppc.IntIndexedContainer in project crate by crate.

the class PlanPrinter method xContentSafeRoutingLocations.

/**
 * Converts the shardId's of each node->table from a {@link IntIndexedContainer} to a list of Integers as custom
 * classes are not supported by the {@link org.elasticsearch.common.xcontent.XContentBuilder}.
 */
private static Map<String, Map<String, List<Integer>>> xContentSafeRoutingLocations(Map<String, Map<String, IntIndexedContainer>> locations) {
    HashMap<String, Map<String, List<Integer>>> safeLocations = new HashMap<>(locations.size(), 1f);
    for (Map.Entry<String, Map<String, IntIndexedContainer>> nodeEntry : locations.entrySet()) {
        HashMap<String, List<Integer>> tableShards = new HashMap<>(nodeEntry.getValue().size(), 1f);
        for (Map.Entry<String, IntIndexedContainer> tableEntry : nodeEntry.getValue().entrySet()) {
            ArrayList<Integer> shardList = new ArrayList<>(tableEntry.getValue().size());
            for (IntCursor cursor : tableEntry.getValue()) {
                shardList.add(cursor.value);
            }
            // ensure a deterministic shard list by sorting it (important for test assertions but maybe also for apps)
            shardList.sort(Integer::compareTo);
            tableShards.put(tableEntry.getKey(), shardList);
        }
        safeLocations.put(nodeEntry.getKey(), tableShards);
    }
    return safeLocations;
}