Examples with IndexWriter io.prestosql.spi.heuristicindex.IndexWriter used on opensource projects

Example 1 with IndexWriter

use of io.prestosql.spi.heuristicindex.IndexWriter in project hetu-core by openlookeng.

the class CreateIndexOperator method finish.

@Override
public void finish() {
    // i.e. index is PERSISTING , FINISHED_PERSISTING or FINISHED
    if (state != State.NEEDS_INPUT) {
        return;
    }
    // start PERSISTING
    // if this operator is responsible for PERSISTING an IndexWriter it will call persist()
    // otherwise the operator will simply go from PERSISTING to FINISHED_PERSISTING without calling persist()
    state = State.PERSISTING;
    // mark current operator as finished
    finished.put(this, true);
    // wait for all operators to have their finish() method called, i.e. there's no more inputs expected
    while (finished.containsValue(false)) {
        try {
            TimeUnit.MILLISECONDS.sleep(50);
        } catch (InterruptedException e) {
            throw new RuntimeException("CreateIndexOperator unexpectedly interrupted while waiting for all operators to finish: ", e);
        }
    }
    // persist index to disk if this operator is responsible for persisting a writer
    try {
        Iterator<Map.Entry<String, IndexWriter>> iterator = levelWriter.entrySet().iterator();
        while (iterator.hasNext()) {
            Map.Entry<String, IndexWriter> entry = iterator.next();
            if (persistBy.get(entry.getValue()) == this) {
                String writerKey = entry.getKey();
                entry.getValue().persist();
                // remove reference to writer once persisted so it can be GCed
                iterator.remove();
                LOG.debug("Writer for %s has finished persisting. Remaining: %d", writerKey, levelWriter.size());
            }
        }
    } catch (IOException e) {
        throw new UncheckedIOException("Persisting index failed: " + e.getMessage(), e);
    }
    synchronized (levelWriter) {
        // All writers have finished persisting
        if (levelWriter.isEmpty()) {
            LOG.debug("Writing index record by %s", this);
            if (persistBy.isEmpty()) {
                // table scan is empty. no data scanned from table. addInput() has never been called.
                throw new IllegalStateException("The table is empty. No index will be created.");
            }
            // update metadata
            IndexClient indexClient = heuristicIndexerManager.getIndexClient();
            try {
                createIndexMetadata.setIndexSize(heuristicIndexerManager.getIndexClient().getIndexSize(createIndexMetadata.getIndexName()));
                IndexClient.RecordStatus recordStatus = indexClient.lookUpIndexRecord(createIndexMetadata);
                LOG.debug("Current record status: %s", recordStatus);
                switch(recordStatus) {
                    case SAME_NAME:
                    case IN_PROGRESS_SAME_NAME:
                    case IN_PROGRESS_SAME_CONTENT:
                    case IN_PROGRESS_SAME_INDEX_PART_CONFLICT:
                    case IN_PROGRESS_SAME_INDEX_PART_CAN_MERGE:
                        indexClient.deleteIndexRecord(createIndexMetadata.getIndexName(), Collections.emptyList());
                        indexClient.addIndexRecord(createIndexMetadata);
                        break;
                    case NOT_FOUND:
                    case SAME_INDEX_PART_CAN_MERGE:
                        indexClient.addIndexRecord(createIndexMetadata);
                        break;
                    default:
                }
            } catch (IOException e) {
                throw new UncheckedIOException("Unable to update index records: ", e);
            }
        }
    }
    state = State.FINISHED_PERSISTING;
}

Example 2 with IndexWriter

use of io.prestosql.spi.heuristicindex.IndexWriter in project hetu-core by openlookeng.

the class UpdateIndexOperator method finish.

@Override
public void finish() {
    // i.e. index is PERSISTING , FINISHED_PERSISTING or FINISHED
    if (state != UpdateIndexOperator.State.NEEDS_INPUT) {
        return;
    }
    // start PERSISTING
    // if this operator is responsible for PERSISTING an IndexWriter it will call persist()
    // otherwise the operator will simply go from PERSISTING to FINISHED_PERSISTING without calling persist()
    state = UpdateIndexOperator.State.PERSISTING;
    // mark current operator as finished
    finished.put(this, true);
    // wait for all operators to have their finish() method called, i.e. there's no more inputs expected
    while (finished.containsValue(false)) {
        try {
            TimeUnit.MILLISECONDS.sleep(50);
        } catch (InterruptedException e) {
            throw new RuntimeException("UpdateIndexOperator unexpectedly interrupted while waiting for all operators to finish: ", e);
        }
    }
    // persist index to disk if this operator is responsible for persisting a writer
    try {
        Iterator<Map.Entry<String, IndexWriter>> iterator = levelWriter.entrySet().iterator();
        while (iterator.hasNext()) {
            Map.Entry<String, IndexWriter> entry = iterator.next();
            if (persistBy.get(entry.getValue()) == this) {
                // A partition/table index doesn't need to be updated as none of the orc files in the partition have a newer modified time
                if (createIndexMetadata.getCreateLevel() == CreateIndexMetadata.Level.STRIPE || !(pathToModifiedTime.containsKey(entry.getKey()) && indexLevelToMaxModifiedTime.containsKey(entry.getKey()) && indexLevelToMaxModifiedTime.get(entry.getKey()) <= Long.parseLong(pathToModifiedTime.get(entry.getKey())))) {
                    entry.getValue().persist();
                }
                String writerKey = entry.getKey();
                // remove reference to writer once persisted so it can be GCed
                iterator.remove();
                LOG.debug("Writer for %s has finished persisting. Remaining: %d", writerKey, levelWriter.size());
            }
        }
    } catch (IOException e) {
        throw new UncheckedIOException("Persisting index failed: " + e.getMessage(), e);
    }
    synchronized (levelWriter) {
        // All writers have finished persisting
        if (levelWriter.isEmpty()) {
            LOG.debug("Writing index record by %s", this);
            // update metadata
            IndexClient indexClient = heuristicIndexerManager.getIndexClient();
            try {
                createIndexMetadata.setIndexSize(heuristicIndexerManager.getIndexClient().getIndexSize(createIndexMetadata.getIndexName()));
                IndexClient.RecordStatus recordStatus = indexClient.lookUpIndexRecord(createIndexMetadata);
                LOG.debug("Current record status: %s", recordStatus);
                CreateIndexMetadata updatedCreateIndexMetadata = new CreateIndexMetadata(createIndexMetadata.getIndexName(), createIndexMetadata.getTableName(), createIndexMetadata.getIndexType(), createIndexMetadata.getIndexSize(), createIndexMetadata.getIndexColumns(), createIndexMetadata.getPartitions(), createIndexMetadata.getProperties(), createIndexMetadata.getUser(), createIndexMetadata.getCreateLevel());
                indexClient.deleteIndexRecord(updatedCreateIndexMetadata.getIndexName(), Collections.emptyList());
                indexClient.addIndexRecord(updatedCreateIndexMetadata);
            } catch (IOException e) {
                throw new UncheckedIOException("Unable to update index records: ", e);
            }
        }
    }
    state = UpdateIndexOperator.State.FINISHED_PERSISTING;
}