Examples with CalculatePartitioner uk.gov.gchq.gaffer.parquetstore.operation.handler.utilities.CalculatePartitioner used on opensource projects

Example 1 with CalculatePartitioner

use of uk.gov.gchq.gaffer.parquetstore.operation.handler.utilities.CalculatePartitioner in project Gaffer by gchq.

the class AddElementsFromRDD method calculateAndWritePartitioner.

/**
 * Calculates the new graph partitioner and writes it to file.
 *
 * @throws OperationException if an {@link IOException} is thrown
 */
private void calculateAndWritePartitioner() throws OperationException {
    // Create new graph partitioner
    LOGGER.info("Calculating new GraphPartitioner");
    final GraphPartitioner newPartitioner;
    try {
        newPartitioner = new CalculatePartitioner(new Path(getSortedAggregatedDirectory(true, true)), store.getSchema(), fs).call();
    } catch (final IOException e) {
        throw new OperationException("IOException calculating new graph partitioner", e);
    }
    LOGGER.info("New GraphPartitioner has partitions for {} groups, {} reversed edge groups", newPartitioner.getGroups().size(), newPartitioner.getGroupsForReversedEdges().size());
    // Write out graph partitioner
    Path newGraphPartitionerPath = null;
    try {
        newGraphPartitionerPath = new Path(getSortedAggregatedDirectory(true, true) + "graphPartitioner");
        final FSDataOutputStream stream = fs.create(newGraphPartitionerPath);
        LOGGER.info("Writing graph partitioner to {}", newGraphPartitionerPath);
        new GraphPartitionerSerialiser().write(newPartitioner, stream);
        stream.close();
    } catch (final IOException e) {
        throw new OperationException("IOException writing out graph partitioner to " + newGraphPartitionerPath, e);
    }
}

Example 2 with CalculatePartitioner

use of uk.gov.gchq.gaffer.parquetstore.operation.handler.utilities.CalculatePartitioner in project Gaffer by gchq.

the class ParquetStore method loadGraphPartitioner.

private void loadGraphPartitioner() throws StoreException {
    final String dataDir = getDataDir();
    try {
        if (fs.exists(new Path(dataDir))) {
            this.currentSnapshot = getLatestSnapshot(dataDir);
            LOGGER.info("Setting currentSnapshot to {}", this.currentSnapshot);
            final Path path = getGraphPartitionerPath();
            if (!fs.exists(path)) {
                LOGGER.info("Graph partitioner does not exist in {} so creating it", path);
                final GraphPartitioner partitioner = new CalculatePartitioner(new Path(dataDir + "/" + getSnapshotPath(this.currentSnapshot)), getSchema(), fs).call();
                LOGGER.info("Writing graph partitioner to {}", path);
                final FSDataOutputStream stream = fs.create(path);
                new GraphPartitionerSerialiser().write(partitioner, stream);
                stream.close();
            }
            LOGGER.info("Loading graph partitioner from path {}", path);
            loadGraphPartitioner(path);
        } else {
            throw new StoreException("Data directory " + dataDir + " does not exist - store is in an inconsistent state");
        }
    } catch (final IOException e) {
        throw new StoreException(e.getMessage(), e);
    }
}