Search in sources :

Example 1 with Location

use of io.dingodb.meta.Location in project dingo by dingodb.

the class DingoJobVisitor method visit.

@Override
public Collection<Output> visit(@Nonnull DingoExchange rel) {
    Collection<Output> inputs = dingo(rel.getInput()).accept(this);
    List<Output> outputs = new LinkedList<>();
    TupleSchema schema = TupleSchema.fromRelDataType(rel.getRowType());
    for (Output input : inputs) {
        Task task = input.getTask();
        Location target = input.getTargetLocation();
        if (target == null) {
            target = Services.META.currentLocation();
        }
        if (!target.equals(task.getLocation())) {
            Id id = idGenerator.get();
            Id receiveId = idGenerator.get();
            SendOperator send = new SendOperator(target.getHost(), target.getPort(), receiveId, schema);
            send.setId(id);
            task.putOperator(send);
            input.setLink(send.getInput(0));
            ReceiveOperator receive = new ReceiveOperator(task.getHost(), task.getLocation().getPort(), schema);
            receive.setId(receiveId);
            receive.getSoleOutput().copyHint(input);
            Task rcvTask = job.getOrCreate(target);
            rcvTask.putOperator(receive);
            outputs.addAll(receive.getOutputs());
        } else {
            outputs.add(input);
        }
    }
    return outputs;
}
Also used : ReceiveOperator(io.dingodb.exec.operator.ReceiveOperator) Task(io.dingodb.exec.base.Task) Output(io.dingodb.exec.base.Output) TableId(io.dingodb.common.table.TableId) Id(io.dingodb.exec.base.Id) SendOperator(io.dingodb.exec.operator.SendOperator) LinkedList(java.util.LinkedList) TupleSchema(io.dingodb.common.table.TupleSchema) Location(io.dingodb.meta.Location)

Example 2 with Location

use of io.dingodb.meta.Location in project dingo by dingodb.

the class DingoJobVisitor method visit.

@Override
public Collection<Output> visit(@Nonnull DingoDistributedValues rel) {
    List<Output> outputs = new LinkedList<>();
    String tableName = getSimpleName(rel.getTable());
    final Map<String, Location> partLocations = Services.META.getPartLocations(tableName);
    final PartitionStrategy ps = new SimpleHashStrategy(partLocations.size());
    final TableDefinition td = Services.META.getTableDefinition(tableName);
    Map<String, List<Object[]>> partMap = ps.partTuples(rel.getValues(), td.getKeyMapping());
    for (Map.Entry<String, List<Object[]>> entry : partMap.entrySet()) {
        Object partId = entry.getKey();
        ValuesOperator operator = new ValuesOperator(entry.getValue());
        operator.setId(idGenerator.get());
        OutputHint hint = new OutputHint();
        hint.setPartId(partId);
        Location location = partLocations.get(partId);
        hint.setLocation(location);
        operator.getSoleOutput().setHint(hint);
        Task task = job.getOrCreate(location);
        task.putOperator(operator);
        outputs.addAll(operator.getOutputs());
    }
    return outputs;
}
Also used : Task(io.dingodb.exec.base.Task) LinkedList(java.util.LinkedList) OutputHint(io.dingodb.exec.base.OutputHint) Output(io.dingodb.exec.base.Output) TableDefinition(io.dingodb.common.table.TableDefinition) List(java.util.List) ArrayList(java.util.ArrayList) LinkedList(java.util.LinkedList) SimpleHashStrategy(io.dingodb.exec.partition.SimpleHashStrategy) ValuesOperator(io.dingodb.exec.operator.ValuesOperator) Map(java.util.Map) Location(io.dingodb.meta.Location) PartitionStrategy(io.dingodb.exec.partition.PartitionStrategy)

Example 3 with Location

use of io.dingodb.meta.Location in project dingo by dingodb.

the class DingoJobVisitor method visit.

@Override
public Collection<Output> visit(@Nonnull DingoPartition rel) {
    Collection<Output> inputs = dingo(rel.getInput()).accept(this);
    String tableName = getSimpleName(rel.getTable());
    List<Output> outputs = new LinkedList<>();
    final Map<String, Location> partLocations = Services.META.getPartLocations(tableName);
    final TableDefinition td = Services.META.getTableDefinition(tableName);
    final PartitionStrategy ps = new SimpleHashStrategy(partLocations.size());
    for (Output input : inputs) {
        Task task = input.getTask();
        PartitionOperator operator = new PartitionOperator(ps, td.getKeyMapping());
        operator.setId(idGenerator.get());
        operator.createOutputs(tableName, partLocations);
        task.putOperator(operator);
        input.setLink(operator.getInput(0));
        outputs.addAll(operator.getOutputs());
    }
    return outputs;
}
Also used : Task(io.dingodb.exec.base.Task) PartitionOperator(io.dingodb.exec.operator.PartitionOperator) Output(io.dingodb.exec.base.Output) TableDefinition(io.dingodb.common.table.TableDefinition) SimpleHashStrategy(io.dingodb.exec.partition.SimpleHashStrategy) LinkedList(java.util.LinkedList) Location(io.dingodb.meta.Location) PartitionStrategy(io.dingodb.exec.partition.PartitionStrategy)

Example 4 with Location

use of io.dingodb.meta.Location in project dingo by dingodb.

the class DingoJobVisitor method visit.

@Override
public Collection<Output> visit(@Nonnull DingoPartScan rel) {
    String tableName = getSimpleName(rel.getTable());
    TableDefinition td = Services.META.getTableDefinition(tableName);
    Map<String, Location> parts = Services.META.getPartLocations(tableName);
    List<Output> outputs = new ArrayList<>(parts.size());
    TableId tableId = new TableId(Services.META.getTableKey(tableName));
    String filterStr = null;
    if (rel.getFilter() != null) {
        filterStr = RexConverter.convert(rel.getFilter()).toString();
    }
    for (Map.Entry<String, Location> entry : parts.entrySet()) {
        final Object partId = entry.getKey();
        PartScanOperator operator = new PartScanOperator(tableId, entry.getKey(), td.getTupleSchema(), td.getKeyMapping(), filterStr, rel.getSelection());
        operator.setId(idGenerator.get());
        Task task = job.getOrCreate(entry.getValue());
        task.putOperator(operator);
        operator.getSoleOutput().setHint(OutputHint.of(tableName, partId));
        outputs.addAll(operator.getOutputs());
    }
    return outputs;
}
Also used : TableId(io.dingodb.common.table.TableId) Task(io.dingodb.exec.base.Task) ArrayList(java.util.ArrayList) PartScanOperator(io.dingodb.exec.operator.PartScanOperator) Output(io.dingodb.exec.base.Output) TableDefinition(io.dingodb.common.table.TableDefinition) Map(java.util.Map) Location(io.dingodb.meta.Location)

Example 5 with Location

use of io.dingodb.meta.Location in project dingo by dingodb.

the class DingoJobVisitor method visit.

@Override
public Collection<Output> visit(@Nonnull DingoGetByKeys rel) {
    String tableName = getSimpleName(rel.getTable());
    final Map<String, Location> partLocations = Services.META.getPartLocations(tableName);
    final TableDefinition td = Services.META.getTableDefinition(tableName);
    final PartitionStrategy ps = new SimpleHashStrategy(partLocations.size());
    final TableId tableId = new TableId(Services.META.getTableKey(tableName));
    Map<String, List<Object[]>> partMap = ps.partKeyTuples(rel.getKeyTuples());
    List<Output> outputs = new LinkedList<>();
    for (Map.Entry<String, List<Object[]>> entry : partMap.entrySet()) {
        final Object partId = entry.getKey();
        GetByKeysOperator operator = new GetByKeysOperator(tableId, partId, td.getTupleSchema(), td.getKeyMapping(), entry.getValue(), rel.getSelection());
        operator.setId(idGenerator.get());
        Task task = job.getOrCreate(partLocations.get(entry.getKey()));
        task.putOperator(operator);
        operator.getSoleOutput().setHint(OutputHint.of(tableName, partId));
        outputs.addAll(operator.getOutputs());
    }
    return outputs;
}
Also used : TableId(io.dingodb.common.table.TableId) Task(io.dingodb.exec.base.Task) LinkedList(java.util.LinkedList) GetByKeysOperator(io.dingodb.exec.operator.GetByKeysOperator) Output(io.dingodb.exec.base.Output) TableDefinition(io.dingodb.common.table.TableDefinition) List(java.util.List) ArrayList(java.util.ArrayList) LinkedList(java.util.LinkedList) SimpleHashStrategy(io.dingodb.exec.partition.SimpleHashStrategy) Map(java.util.Map) Location(io.dingodb.meta.Location) PartitionStrategy(io.dingodb.exec.partition.PartitionStrategy)

Aggregations

Location (io.dingodb.meta.Location)9 Output (io.dingodb.exec.base.Output)6 Task (io.dingodb.exec.base.Task)6 Map (java.util.Map)6 TableDefinition (io.dingodb.common.table.TableDefinition)4 TableId (io.dingodb.common.table.TableId)4 LinkedList (java.util.LinkedList)4 PartitionStrategy (io.dingodb.exec.partition.PartitionStrategy)3 SimpleHashStrategy (io.dingodb.exec.partition.SimpleHashStrategy)3 ArrayList (java.util.ArrayList)3 OutputHint (io.dingodb.exec.base.OutputHint)2 List (java.util.List)2 JsonProcessingException (com.fasterxml.jackson.core.JsonProcessingException)1 ImmutableMap (com.google.common.collect.ImmutableMap)1 TupleSchema (io.dingodb.common.table.TupleSchema)1 Id (io.dingodb.exec.base.Id)1 GetByKeysOperator (io.dingodb.exec.operator.GetByKeysOperator)1 PartScanOperator (io.dingodb.exec.operator.PartScanOperator)1 PartitionOperator (io.dingodb.exec.operator.PartitionOperator)1 ReceiveOperator (io.dingodb.exec.operator.ReceiveOperator)1