Search in sources :

Example 61 with Optional

use of java.util.Optional in project neo4j by neo4j.

the class StoreMigrator method moveMigratedFiles.

@Override
public void moveMigratedFiles(File migrationDir, File storeDir, String versionToUpgradeFrom, String versionToUpgradeTo) throws IOException {
    // Move the migrated ones into the store directory
    StoreFile.fileOperation(MOVE, fileSystem, migrationDir, storeDir, StoreFile.currentStoreFiles(), // allow to skip non existent source files
    true, // allow to overwrite target files
    ExistingTargetStrategy.OVERWRITE, StoreFileType.values());
    // move the files with the page cache.
    try {
        Iterable<FileHandle> fileHandles = pageCache.streamFilesRecursive(migrationDir)::iterator;
        for (FileHandle fh : fileHandles) {
            Predicate<StoreFile> predicate = storeFile -> storeFile.fileName(StoreFileType.STORE).equals(fh.getFile().getName());
            if (StreamSupport.stream(StoreFile.currentStoreFiles().spliterator(), false).anyMatch(predicate)) {
                final Optional<PagedFile> optionalPagedFile = pageCache.getExistingMapping(fh.getFile());
                if (optionalPagedFile.isPresent()) {
                    optionalPagedFile.get().close();
                }
                fh.rename(new File(storeDir, fh.getFile().getName()), StandardCopyOption.REPLACE_EXISTING);
            }
        }
    } catch (NoSuchFileException e) {
    //This means that we had no files only present in the page cache, this is fine.
    }
    RecordFormats oldFormat = selectForVersion(versionToUpgradeFrom);
    RecordFormats newFormat = selectForVersion(versionToUpgradeTo);
    boolean movingAwayFromVersionTrailers = oldFormat.hasCapability(VERSION_TRAILERS) && !newFormat.hasCapability(VERSION_TRAILERS);
    if (movingAwayFromVersionTrailers) {
        StoreFile.removeTrailers(versionToUpgradeFrom, fileSystem, storeDir, pageCache.pageSize());
    }
    File neoStore = new File(storeDir, MetaDataStore.DEFAULT_NAME);
    long logVersion = MetaDataStore.getRecord(pageCache, neoStore, Position.LOG_VERSION);
    long lastCommittedTx = MetaDataStore.getRecord(pageCache, neoStore, Position.LAST_TRANSACTION_ID);
    // update or add upgrade id and time and other necessary neostore records
    updateOrAddNeoStoreFieldsAsPartOfMigration(migrationDir, storeDir, versionToUpgradeTo);
    // delete old logs
    legacyLogs.deleteUnusedLogFiles(storeDir);
    if (movingAwayFromVersionTrailers) {
        // write a check point in the log in order to make recovery work in the newer version
        new StoreMigratorCheckPointer(storeDir, fileSystem).checkPoint(logVersion, lastCommittedTx);
    }
}
Also used : PropertyDeduplicator(org.neo4j.kernel.impl.storemigration.legacystore.v21.propertydeduplication.PropertyDeduplicator) Arrays(java.util.Arrays) PageCursor(org.neo4j.io.pagecache.PageCursor) NullLogProvider(org.neo4j.logging.NullLogProvider) StringUtils(org.apache.commons.lang3.StringUtils) UNKNOWN_TX_COMMIT_TIMESTAMP(org.neo4j.kernel.impl.transaction.log.TransactionIdStore.UNKNOWN_TX_COMMIT_TIMESTAMP) COPY(org.neo4j.kernel.impl.storemigration.FileOperation.COPY) SchemaIndexProvider(org.neo4j.kernel.api.index.SchemaIndexProvider) CountsComputer(org.neo4j.kernel.impl.store.CountsComputer) Arrays.asList(java.util.Arrays.asList) BASE_TX_COMMIT_TIMESTAMP(org.neo4j.kernel.impl.transaction.log.TransactionIdStore.BASE_TX_COMMIT_TIMESTAMP) Position(org.neo4j.kernel.impl.store.MetaDataStore.Position) Lifespan(org.neo4j.kernel.lifecycle.Lifespan) RecordCursors(org.neo4j.kernel.impl.store.RecordCursors) SilentMigrationProgressMonitor(org.neo4j.kernel.impl.storemigration.monitoring.SilentMigrationProgressMonitor) StandardCharsets(java.nio.charset.StandardCharsets) MetaDataRecordFormat(org.neo4j.kernel.impl.store.format.standard.MetaDataRecordFormat) DELETE(org.neo4j.kernel.impl.storemigration.FileOperation.DELETE) Stream(java.util.stream.Stream) ExecutionSupervisors.withDynamicProcessorAssignment(org.neo4j.unsafe.impl.batchimport.staging.ExecutionSupervisors.withDynamicProcessorAssignment) StoreType(org.neo4j.kernel.impl.store.StoreType) BatchImporter(org.neo4j.unsafe.impl.batchimport.BatchImporter) NodeStore(org.neo4j.kernel.impl.store.NodeStore) StoreFactory(org.neo4j.kernel.impl.store.StoreFactory) Collectors(org.neo4j.unsafe.impl.batchimport.input.Collectors) PagedFile(org.neo4j.io.pagecache.PagedFile) VERSION_TRAILERS(org.neo4j.kernel.impl.store.format.Capability.VERSION_TRAILERS) FormatFamily(org.neo4j.kernel.impl.store.format.FormatFamily) Supplier(java.util.function.Supplier) UNKNOWN_TX_CHECKSUM(org.neo4j.kernel.impl.transaction.log.TransactionIdStore.UNKNOWN_TX_CHECKSUM) BufferedOutputStream(java.io.BufferedOutputStream) StandardCopyOption(java.nio.file.StandardCopyOption) ArrayList(java.util.ArrayList) TransactionId(org.neo4j.kernel.impl.store.TransactionId) StandardV2_2(org.neo4j.kernel.impl.store.format.standard.StandardV2_2) LogPosition(org.neo4j.kernel.impl.transaction.log.LogPosition) StandardV2_1(org.neo4j.kernel.impl.store.format.standard.StandardV2_1) RelationshipStore(org.neo4j.kernel.impl.store.RelationshipStore) StandardV2_0(org.neo4j.kernel.impl.store.format.standard.StandardV2_0) IdMappers(org.neo4j.unsafe.impl.batchimport.cache.idmapping.IdMappers) BiConsumer(java.util.function.BiConsumer) IdGenerators(org.neo4j.unsafe.impl.batchimport.cache.idmapping.IdGenerators) StreamSupport(java.util.stream.StreamSupport) DirectRecordStoreMigrator(org.neo4j.kernel.impl.storemigration.DirectRecordStoreMigrator) CoarseBoundedProgressExecutionMonitor(org.neo4j.unsafe.impl.batchimport.staging.CoarseBoundedProgressExecutionMonitor) StoreFile(org.neo4j.kernel.impl.storemigration.StoreFile) FileOutputStream(java.io.FileOutputStream) RecordFormatSelector.selectForVersion(org.neo4j.kernel.impl.store.format.RecordFormatSelector.selectForVersion) IOException(java.io.IOException) NodeRecordFormat(org.neo4j.kernel.impl.store.format.standard.NodeRecordFormat) LegacyLogs(org.neo4j.kernel.impl.storemigration.legacylogs.LegacyLogs) File(java.io.File) Iterables(org.neo4j.helpers.collection.Iterables) ParallelBatchImporter(org.neo4j.unsafe.impl.batchimport.ParallelBatchImporter) NodeRecord(org.neo4j.kernel.impl.store.record.NodeRecord) Configuration(org.neo4j.unsafe.impl.batchimport.Configuration) BufferedReader(java.io.BufferedReader) MigrationProgressMonitor(org.neo4j.kernel.impl.storemigration.monitoring.MigrationProgressMonitor) RelationshipRecord(org.neo4j.kernel.impl.store.record.RelationshipRecord) StoreUpgrader(org.neo4j.kernel.impl.storemigration.StoreUpgrader) NoSuchFileException(java.nio.file.NoSuchFileException) BASE_TX_LOG_VERSION(org.neo4j.kernel.impl.transaction.log.TransactionIdStore.BASE_TX_LOG_VERSION) BASE_TX_CHECKSUM(org.neo4j.kernel.impl.transaction.log.TransactionIdStore.BASE_TX_CHECKSUM) PageCache(org.neo4j.io.pagecache.PageCache) InputNode(org.neo4j.unsafe.impl.batchimport.input.InputNode) Predicate(java.util.function.Predicate) Collection(java.util.Collection) StandardOpenOption(java.nio.file.StandardOpenOption) MOVE(org.neo4j.kernel.impl.storemigration.FileOperation.MOVE) LogService(org.neo4j.kernel.impl.logging.LogService) List(java.util.List) StoreFileType(org.neo4j.kernel.impl.storemigration.StoreFileType) MetaDataStore(org.neo4j.kernel.impl.store.MetaDataStore) Writer(java.io.Writer) Optional(java.util.Optional) Inputs(org.neo4j.unsafe.impl.batchimport.input.Inputs) InputIterable(org.neo4j.unsafe.impl.batchimport.InputIterable) InputRelationship(org.neo4j.unsafe.impl.batchimport.input.InputRelationship) TransactionIdStore(org.neo4j.kernel.impl.transaction.log.TransactionIdStore) InputEntity(org.neo4j.unsafe.impl.batchimport.input.InputEntity) DEFAULT_NAME(org.neo4j.kernel.impl.store.MetaDataStore.DEFAULT_NAME) RecordFormats(org.neo4j.kernel.impl.store.format.RecordFormats) CustomIOConfigValidator(org.neo4j.kernel.impl.util.CustomIOConfigValidator) AdditionalInitialIds(org.neo4j.unsafe.impl.batchimport.AdditionalInitialIds) NeoStores(org.neo4j.kernel.impl.store.NeoStores) StorePropertyCursor(org.neo4j.kernel.impl.api.store.StorePropertyCursor) CountsTracker(org.neo4j.kernel.impl.store.counts.CountsTracker) ExistingTargetStrategy(org.neo4j.kernel.impl.storemigration.ExistingTargetStrategy) PhysicalLogFiles(org.neo4j.kernel.impl.transaction.log.PhysicalLogFiles) RelationshipRecordFormat(org.neo4j.kernel.impl.store.format.standard.RelationshipRecordFormat) OutputStream(java.io.OutputStream) Config(org.neo4j.kernel.configuration.Config) ReadOnlyIdGeneratorFactory(org.neo4j.kernel.impl.store.id.ReadOnlyIdGeneratorFactory) LockService(org.neo4j.kernel.impl.locking.LockService) FileHandle(org.neo4j.io.pagecache.FileHandle) CapabilityType(org.neo4j.kernel.impl.store.format.CapabilityType) PrimitiveRecord(org.neo4j.kernel.impl.store.record.PrimitiveRecord) FIELD_NOT_PRESENT(org.neo4j.kernel.impl.store.format.standard.MetaDataRecordFormat.FIELD_NOT_PRESENT) StoreMigratorCheckPointer(org.neo4j.kernel.impl.storemigration.StoreMigratorCheckPointer) ExecutionMonitor(org.neo4j.unsafe.impl.batchimport.staging.ExecutionMonitor) BASE_TX_LOG_BYTE_OFFSET(org.neo4j.kernel.impl.transaction.log.TransactionIdStore.BASE_TX_LOG_BYTE_OFFSET) FileSystemAbstraction(org.neo4j.io.fs.FileSystemAbstraction) PagedFile(org.neo4j.io.pagecache.PagedFile) FileHandle(org.neo4j.io.pagecache.FileHandle) NoSuchFileException(java.nio.file.NoSuchFileException) RecordFormats(org.neo4j.kernel.impl.store.format.RecordFormats) StoreFile(org.neo4j.kernel.impl.storemigration.StoreFile) PagedFile(org.neo4j.io.pagecache.PagedFile) StoreFile(org.neo4j.kernel.impl.storemigration.StoreFile) File(java.io.File) StoreMigratorCheckPointer(org.neo4j.kernel.impl.storemigration.StoreMigratorCheckPointer)

Example 62 with Optional

use of java.util.Optional in project ninja by ninjaframework.

the class TemplateEngineFreemarkerTest method before.

@Before
public void before() throws Exception {
    //Setup that allows to to execute invoke(...) in a very minimal version.
    when(ninjaProperties.getWithDefault(FREEMARKER_CONFIGURATION_FILE_SUFFIX, ".ftl.html")).thenReturn(".ftl.html");
    templateEngineFreemarker = new TemplateEngineFreemarker(messages, lang, logger, templateEngineHelper, templateEngineManager, templateEngineFreemarkerReverseRouteMethod, templateEngineFreemarkerAssetsAtMethod, templateEngineFreemarkerWebJarsAtMethod, ninjaProperties);
    when(lang.getLanguage(any(Context.class), any(Optional.class))).thenReturn(Optional.<String>empty());
    Session session = Mockito.mock(Session.class);
    when(session.isEmpty()).thenReturn(true);
    when(context.getSession()).thenReturn(session);
    when(context.getRoute()).thenReturn(route);
    when(lang.getLocaleFromStringOrDefault(any(Optional.class))).thenReturn(Locale.ENGLISH);
    FlashScope flashScope = Mockito.mock(FlashScope.class);
    Map<String, String> flashScopeData = new HashMap<>();
    when(flashScope.getCurrentFlashCookieData()).thenReturn(flashScopeData);
    when(context.getFlashScope()).thenReturn(flashScope);
    when(templateEngineHelper.getTemplateForResult(any(Route.class), any(Result.class), Mockito.anyString())).thenReturn("views/template.ftl.html");
    writer = new StringWriter();
    ResponseStreams responseStreams = mock(ResponseStreams.class);
    when(context.finalizeHeaders(any(Result.class))).thenReturn(responseStreams);
    when(responseStreams.getWriter()).thenReturn(writer);
}
Also used : Context(ninja.Context) Optional(java.util.Optional) StringWriter(java.io.StringWriter) ResponseStreams(ninja.utils.ResponseStreams) HashMap(java.util.HashMap) FlashScope(ninja.session.FlashScope) Route(ninja.Route) Session(ninja.session.Session) Result(ninja.Result) Before(org.junit.Before)

Example 63 with Optional

use of java.util.Optional in project neo4j by neo4j.

the class ServerPoliciesPlugin method readEndpoints.

private List<Endpoint> readEndpoints(CoreTopology coreTopology, ReadReplicaTopology rrTopology, Policy policy) {
    Set<ServerInfo> possibleReaders = rrTopology.members().entrySet().stream().map(entry -> new ServerInfo(entry.getValue().connectors().boltAddress(), entry.getKey(), entry.getValue().groups())).collect(Collectors.toSet());
    if (allowReadsOnFollowers || possibleReaders.size() == 0) {
        Set<MemberId> validCores = coreTopology.members().keySet();
        try {
            MemberId leader = leaderLocator.getLeader();
            validCores = validCores.stream().filter(memberId -> !memberId.equals(leader)).collect(Collectors.toSet());
        } catch (NoLeaderFoundException ignored) {
        // we might end up using the leader for reading during this ttl, should be fine in general
        }
        for (MemberId validCore : validCores) {
            Optional<CoreServerInfo> coreServerInfo = coreTopology.find(validCore);
            if (coreServerInfo.isPresent()) {
                CoreServerInfo serverInfo = coreServerInfo.get();
                possibleReaders.add(new ServerInfo(serverInfo.connectors().boltAddress(), validCore, serverInfo.groups()));
            }
        }
    }
    Set<ServerInfo> readers = policy.apply(possibleReaders);
    return readers.stream().map(r -> Endpoint.read(r.boltAddress())).collect(Collectors.toList());
}
Also used : ReadReplicaTopology(org.neo4j.causalclustering.discovery.ReadReplicaTopology) Service(org.neo4j.helpers.Service) LoadBalancingResult(org.neo4j.causalclustering.load_balancing.LoadBalancingResult) Log(org.neo4j.logging.Log) CausalClusteringSettings(org.neo4j.causalclustering.core.CausalClusteringSettings) Util.asList(org.neo4j.causalclustering.load_balancing.Util.asList) LogProvider(org.neo4j.logging.LogProvider) FilteringPolicyLoader.load(org.neo4j.causalclustering.load_balancing.plugins.server_policies.FilteringPolicyLoader.load) Map(java.util.Map) MemberId(org.neo4j.causalclustering.identity.MemberId) Endpoint(org.neo4j.causalclustering.load_balancing.Endpoint) CoreServerInfo(org.neo4j.causalclustering.discovery.CoreServerInfo) LeaderLocator(org.neo4j.causalclustering.core.consensus.LeaderLocator) InvalidSettingException(org.neo4j.graphdb.config.InvalidSettingException) CoreTopology(org.neo4j.causalclustering.discovery.CoreTopology) LoadBalancingPlugin(org.neo4j.causalclustering.load_balancing.LoadBalancingPlugin) Config(org.neo4j.kernel.configuration.Config) Collections.emptyList(java.util.Collections.emptyList) TopologyService(org.neo4j.causalclustering.discovery.TopologyService) Set(java.util.Set) Collectors(java.util.stream.Collectors) List(java.util.List) NoLeaderFoundException(org.neo4j.causalclustering.core.consensus.NoLeaderFoundException) Optional(java.util.Optional) Util.extractBoltAddress(org.neo4j.causalclustering.load_balancing.Util.extractBoltAddress) MemberId(org.neo4j.causalclustering.identity.MemberId) NoLeaderFoundException(org.neo4j.causalclustering.core.consensus.NoLeaderFoundException) CoreServerInfo(org.neo4j.causalclustering.discovery.CoreServerInfo) CoreServerInfo(org.neo4j.causalclustering.discovery.CoreServerInfo)

Example 64 with Optional

use of java.util.Optional in project presto by prestodb.

the class HiveMetadata method buildTableObject.

private static Table buildTableObject(String queryId, String schemaName, String tableName, String tableOwner, List<HiveColumnHandle> columnHandles, HiveStorageFormat hiveStorageFormat, List<String> partitionedBy, Optional<HiveBucketProperty> bucketProperty, Map<String, String> additionalTableParameters, Path targetPath, boolean external, String prestoVersion) {
    Map<String, HiveColumnHandle> columnHandlesByName = Maps.uniqueIndex(columnHandles, HiveColumnHandle::getName);
    List<Column> partitionColumns = partitionedBy.stream().map(columnHandlesByName::get).map(column -> new Column(column.getName(), column.getHiveType(), column.getComment())).collect(toList());
    Set<String> partitionColumnNames = ImmutableSet.copyOf(partitionedBy);
    ImmutableList.Builder<Column> columns = ImmutableList.builder();
    for (HiveColumnHandle columnHandle : columnHandles) {
        String name = columnHandle.getName();
        HiveType type = columnHandle.getHiveType();
        if (!partitionColumnNames.contains(name)) {
            verify(!columnHandle.isPartitionKey(), "Column handles are not consistent with partitioned by property");
            columns.add(new Column(name, type, columnHandle.getComment()));
        } else {
            verify(columnHandle.isPartitionKey(), "Column handles are not consistent with partitioned by property");
        }
    }
    ImmutableMap.Builder<String, String> tableParameters = ImmutableMap.<String, String>builder().put("comment", "Created by Presto").put(PRESTO_VERSION_NAME, prestoVersion).put(PRESTO_QUERY_ID_NAME, queryId).putAll(additionalTableParameters);
    if (external) {
        tableParameters.put("EXTERNAL", "TRUE");
    }
    Table.Builder tableBuilder = Table.builder().setDatabaseName(schemaName).setTableName(tableName).setOwner(tableOwner).setTableType((external ? EXTERNAL_TABLE : MANAGED_TABLE).name()).setDataColumns(columns.build()).setPartitionColumns(partitionColumns).setParameters(tableParameters.build());
    tableBuilder.getStorageBuilder().setStorageFormat(fromHiveStorageFormat(hiveStorageFormat)).setBucketProperty(bucketProperty).setLocation(targetPath.toString());
    return tableBuilder.build();
}
Also used : DateTimeZone(org.joda.time.DateTimeZone) TypeManager(com.facebook.presto.spi.type.TypeManager) HiveTableProperties.getPartitionedBy(com.facebook.presto.hive.HiveTableProperties.getPartitionedBy) Map(java.util.Map) INVALID_TABLE_PROPERTY(com.facebook.presto.spi.StandardErrorCode.INVALID_TABLE_PROPERTY) HivePartitionManager.extractPartitionKeyValues(com.facebook.presto.hive.HivePartitionManager.extractPartitionKeyValues) WriteMode(com.facebook.presto.hive.metastore.SemiTransactionalHiveMetastore.WriteMode) Verify(com.google.common.base.Verify) StorageFormat(com.facebook.presto.hive.metastore.StorageFormat) HIVE_INVALID_METADATA(com.facebook.presto.hive.HiveErrorCode.HIVE_INVALID_METADATA) PrincipalPrivileges(com.facebook.presto.hive.metastore.PrincipalPrivileges) PATH_COLUMN_NAME(com.facebook.presto.hive.HiveColumnHandle.PATH_COLUMN_NAME) Set(java.util.Set) HivePrivilegeInfo.toHivePrivilege(com.facebook.presto.hive.metastore.HivePrivilegeInfo.toHivePrivilege) SemiTransactionalHiveMetastore(com.facebook.presto.hive.metastore.SemiTransactionalHiveMetastore) MANAGED_TABLE(org.apache.hadoop.hive.metastore.TableType.MANAGED_TABLE) ConnectorSession(com.facebook.presto.spi.ConnectorSession) Domain(com.facebook.presto.spi.predicate.Domain) HiveColumnHandle.updateRowIdHandle(com.facebook.presto.hive.HiveColumnHandle.updateRowIdHandle) HivePrivilegeInfo(com.facebook.presto.hive.metastore.HivePrivilegeInfo) MoreObjects.toStringHelper(com.google.common.base.MoreObjects.toStringHelper) Iterables(com.google.common.collect.Iterables) Table(com.facebook.presto.hive.metastore.Table) Slice(io.airlift.slice.Slice) Database(com.facebook.presto.hive.metastore.Database) REGULAR(com.facebook.presto.hive.HiveColumnHandle.ColumnType.REGULAR) BUCKET_COLUMN_NAME(com.facebook.presto.hive.HiveColumnHandle.BUCKET_COLUMN_NAME) HiveUtil.columnExtraInfo(com.facebook.presto.hive.HiveUtil.columnExtraInfo) HiveTableProperties.getBucketProperty(com.facebook.presto.hive.HiveTableProperties.getBucketProperty) ConnectorOutputTableHandle(com.facebook.presto.spi.ConnectorOutputTableHandle) ArrayList(java.util.ArrayList) OptionalLong(java.util.OptionalLong) DIRECT_TO_TARGET_EXISTING_DIRECTORY(com.facebook.presto.hive.metastore.SemiTransactionalHiveMetastore.WriteMode.DIRECT_TO_TARGET_EXISTING_DIRECTORY) Type(com.facebook.presto.spi.type.Type) HiveUtil.encodeViewData(com.facebook.presto.hive.HiveUtil.encodeViewData) ImmutableMultimap(com.google.common.collect.ImmutableMultimap) HiveTableProperties.getHiveStorageFormat(com.facebook.presto.hive.HiveTableProperties.getHiveStorageFormat) HiveWriteUtils.checkTableIsWritable(com.facebook.presto.hive.HiveWriteUtils.checkTableIsWritable) ConnectorTableMetadata(com.facebook.presto.spi.ConnectorTableMetadata) PRESTO_VIEW_FLAG(com.facebook.presto.hive.HiveUtil.PRESTO_VIEW_FLAG) Properties(java.util.Properties) HiveWriteUtils.initializeSerializer(com.facebook.presto.hive.HiveWriteUtils.initializeSerializer) HIVE_CONCURRENT_MODIFICATION_DETECTED(com.facebook.presto.hive.HiveErrorCode.HIVE_CONCURRENT_MODIFICATION_DETECTED) Constraint(com.facebook.presto.spi.Constraint) IOException(java.io.IOException) ConnectorTableLayoutResult(com.facebook.presto.spi.ConnectorTableLayoutResult) HivePrivilege(com.facebook.presto.hive.metastore.HivePrivilegeInfo.HivePrivilege) SchemaTablePrefix(com.facebook.presto.spi.SchemaTablePrefix) ColumnHandle(com.facebook.presto.spi.ColumnHandle) ConnectorNewTableLayout(com.facebook.presto.spi.ConnectorNewTableLayout) TableType(org.apache.hadoop.hive.metastore.TableType) HiveUtil.hiveColumnHandles(com.facebook.presto.hive.HiveUtil.hiveColumnHandles) DIRECT_TO_TARGET_NEW_DIRECTORY(com.facebook.presto.hive.metastore.SemiTransactionalHiveMetastore.WriteMode.DIRECT_TO_TARGET_NEW_DIRECTORY) ConnectorViewDefinition(com.facebook.presto.spi.ConnectorViewDefinition) ViewNotFoundException(com.facebook.presto.spi.ViewNotFoundException) HiveTableProperties.getExternalLocation(com.facebook.presto.hive.HiveTableProperties.getExternalLocation) EXTERNAL_TABLE(org.apache.hadoop.hive.metastore.TableType.EXTERNAL_TABLE) HiveType.toHiveType(com.facebook.presto.hive.HiveType.toHiveType) STAGE_AND_MOVE_TO_TARGET_DIRECTORY(com.facebook.presto.hive.metastore.SemiTransactionalHiveMetastore.WriteMode.STAGE_AND_MOVE_TO_TARGET_DIRECTORY) Privilege(com.facebook.presto.spi.security.Privilege) FileSinkOperator(org.apache.hadoop.hive.ql.exec.FileSinkOperator) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) SchemaTableName(com.facebook.presto.spi.SchemaTableName) HiveSessionProperties.isBucketExecutionEnabled(com.facebook.presto.hive.HiveSessionProperties.isBucketExecutionEnabled) Collectors.toMap(java.util.stream.Collectors.toMap) Iterables.concat(com.google.common.collect.Iterables.concat) HIVE_COLUMN_ORDER_MISMATCH(com.facebook.presto.hive.HiveErrorCode.HIVE_COLUMN_ORDER_MISMATCH) BUCKETED_BY_PROPERTY(com.facebook.presto.hive.HiveTableProperties.BUCKETED_BY_PROPERTY) Path(org.apache.hadoop.fs.Path) DiscretePredicates(com.facebook.presto.spi.DiscretePredicates) NullableValue(com.facebook.presto.spi.predicate.NullableValue) Collectors.toSet(java.util.stream.Collectors.toSet) HIVE_UNSUPPORTED_FORMAT(com.facebook.presto.hive.HiveErrorCode.HIVE_UNSUPPORTED_FORMAT) ImmutableSet(com.google.common.collect.ImmutableSet) HIVE_UNKNOWN_ERROR(com.facebook.presto.hive.HiveErrorCode.HIVE_UNKNOWN_ERROR) ImmutableMap(com.google.common.collect.ImmutableMap) Predicate(java.util.function.Predicate) INVALID_SCHEMA_PROPERTY(com.facebook.presto.spi.StandardErrorCode.INVALID_SCHEMA_PROPERTY) Collections.emptyList(java.util.Collections.emptyList) Collection(java.util.Collection) SCHEMA_NOT_EMPTY(com.facebook.presto.spi.StandardErrorCode.SCHEMA_NOT_EMPTY) Collectors(java.util.stream.Collectors) Sets(com.google.common.collect.Sets) String.format(java.lang.String.format) STORAGE_FORMAT_PROPERTY(com.facebook.presto.hive.HiveTableProperties.STORAGE_FORMAT_PROPERTY) TupleDomain(com.facebook.presto.spi.predicate.TupleDomain) HIVE_STRING(com.facebook.presto.hive.HiveType.HIVE_STRING) List(java.util.List) ColumnMetadata(com.facebook.presto.spi.ColumnMetadata) NOT_SUPPORTED(com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED) USER(com.facebook.presto.hive.metastore.PrincipalType.USER) Optional(java.util.Optional) HIVE_TIMEZONE_MISMATCH(com.facebook.presto.hive.HiveErrorCode.HIVE_TIMEZONE_MISMATCH) PARTITION_KEY(com.facebook.presto.hive.HiveColumnHandle.ColumnType.PARTITION_KEY) HiveUtil.schemaTableName(com.facebook.presto.hive.HiveUtil.schemaTableName) HIDDEN(com.facebook.presto.hive.HiveColumnHandle.ColumnType.HIDDEN) JsonCodec(io.airlift.json.JsonCodec) TupleDomain.withColumnDomains(com.facebook.presto.spi.predicate.TupleDomain.withColumnDomains) ConnectorMetadata(com.facebook.presto.spi.connector.ConnectorMetadata) Column(com.facebook.presto.hive.metastore.Column) ConnectorTableLayoutHandle(com.facebook.presto.spi.ConnectorTableLayoutHandle) HIVE_WRITER_CLOSE_ERROR(com.facebook.presto.hive.HiveErrorCode.HIVE_WRITER_CLOSE_ERROR) ConnectorTableHandle(com.facebook.presto.spi.ConnectorTableHandle) PrestoException(com.facebook.presto.spi.PrestoException) Function(java.util.function.Function) Partition(com.facebook.presto.hive.metastore.Partition) ImmutableList(com.google.common.collect.ImmutableList) PARTITIONED_BY_PROPERTY(com.facebook.presto.hive.HiveTableProperties.PARTITIONED_BY_PROPERTY) Verify.verify(com.google.common.base.Verify.verify) Objects.requireNonNull(java.util.Objects.requireNonNull) HiveUtil.toPartitionValues(com.facebook.presto.hive.HiveUtil.toPartitionValues) HiveWriteUtils.isWritableType(com.facebook.presto.hive.HiveWriteUtils.isWritableType) ConnectorNodePartitioning(com.facebook.presto.spi.ConnectorNodePartitioning) ConnectorInsertTableHandle(com.facebook.presto.spi.ConnectorInsertTableHandle) ConnectorOutputMetadata(com.facebook.presto.spi.connector.ConnectorOutputMetadata) ConnectorTableLayout(com.facebook.presto.spi.ConnectorTableLayout) EXTERNAL_LOCATION_PROPERTY(com.facebook.presto.hive.HiveTableProperties.EXTERNAL_LOCATION_PROPERTY) VIEW_STORAGE_FORMAT(com.facebook.presto.hive.metastore.StorageFormat.VIEW_STORAGE_FORMAT) Maps(com.google.common.collect.Maps) JobConf(org.apache.hadoop.mapred.JobConf) Collectors.toList(java.util.stream.Collectors.toList) TableNotFoundException(com.facebook.presto.spi.TableNotFoundException) BUCKET_COUNT_PROPERTY(com.facebook.presto.hive.HiveTableProperties.BUCKET_COUNT_PROPERTY) StorageFormat.fromHiveStorageFormat(com.facebook.presto.hive.metastore.StorageFormat.fromHiveStorageFormat) HiveUtil.decodeViewData(com.facebook.presto.hive.HiveUtil.decodeViewData) VisibleForTesting(com.google.common.annotations.VisibleForTesting) MetastoreUtil.getHiveSchema(com.facebook.presto.hive.metastore.MetastoreUtil.getHiveSchema) Table(com.facebook.presto.hive.metastore.Table) ImmutableList(com.google.common.collect.ImmutableList) ImmutableMap(com.google.common.collect.ImmutableMap) Column(com.facebook.presto.hive.metastore.Column) HiveType.toHiveType(com.facebook.presto.hive.HiveType.toHiveType)

Example 65 with Optional

use of java.util.Optional in project presto by prestodb.

the class HiveMetadata method beginInsert.

@Override
public HiveInsertTableHandle beginInsert(ConnectorSession session, ConnectorTableHandle tableHandle) {
    verifyJvmTimeZone();
    SchemaTableName tableName = schemaTableName(tableHandle);
    Optional<Table> table = metastore.getTable(tableName.getSchemaName(), tableName.getTableName());
    if (!table.isPresent()) {
        throw new TableNotFoundException(tableName);
    }
    checkTableIsWritable(table.get());
    for (Column column : table.get().getDataColumns()) {
        if (!isWritableType(column.getType())) {
            throw new PrestoException(NOT_SUPPORTED, format("Inserting into Hive table %s.%s with column type %s not supported", table.get().getDatabaseName(), table.get().getTableName(), column.getType()));
        }
    }
    List<HiveColumnHandle> handles = hiveColumnHandles(connectorId, table.get()).stream().filter(columnHandle -> !columnHandle.isHidden()).collect(toList());
    HiveStorageFormat tableStorageFormat = extractHiveStorageFormat(table.get());
    LocationHandle locationHandle = locationService.forExistingTable(metastore, session.getUser(), session.getQueryId(), table.get());
    HiveInsertTableHandle result = new HiveInsertTableHandle(connectorId, tableName.getSchemaName(), tableName.getTableName(), handles, session.getQueryId(), metastore.generatePageSinkMetadata(tableName), locationHandle, table.get().getStorage().getBucketProperty(), tableStorageFormat, respectTableFormat ? tableStorageFormat : defaultStorageFormat);
    Optional<Path> writePathRoot = locationService.writePathRoot(locationHandle);
    Path targetPathRoot = locationService.targetPathRoot(locationHandle);
    if (writePathRoot.isPresent()) {
        WriteMode mode = writePathRoot.get().equals(targetPathRoot) ? DIRECT_TO_TARGET_NEW_DIRECTORY : STAGE_AND_MOVE_TO_TARGET_DIRECTORY;
        metastore.declareIntentionToWrite(session, mode, writePathRoot.get(), result.getFilePrefix(), tableName);
    } else {
        metastore.declareIntentionToWrite(session, DIRECT_TO_TARGET_EXISTING_DIRECTORY, targetPathRoot, result.getFilePrefix(), tableName);
    }
    return result;
}
Also used : DateTimeZone(org.joda.time.DateTimeZone) TypeManager(com.facebook.presto.spi.type.TypeManager) HiveTableProperties.getPartitionedBy(com.facebook.presto.hive.HiveTableProperties.getPartitionedBy) Map(java.util.Map) INVALID_TABLE_PROPERTY(com.facebook.presto.spi.StandardErrorCode.INVALID_TABLE_PROPERTY) HivePartitionManager.extractPartitionKeyValues(com.facebook.presto.hive.HivePartitionManager.extractPartitionKeyValues) WriteMode(com.facebook.presto.hive.metastore.SemiTransactionalHiveMetastore.WriteMode) Verify(com.google.common.base.Verify) StorageFormat(com.facebook.presto.hive.metastore.StorageFormat) HIVE_INVALID_METADATA(com.facebook.presto.hive.HiveErrorCode.HIVE_INVALID_METADATA) PrincipalPrivileges(com.facebook.presto.hive.metastore.PrincipalPrivileges) PATH_COLUMN_NAME(com.facebook.presto.hive.HiveColumnHandle.PATH_COLUMN_NAME) Set(java.util.Set) HivePrivilegeInfo.toHivePrivilege(com.facebook.presto.hive.metastore.HivePrivilegeInfo.toHivePrivilege) SemiTransactionalHiveMetastore(com.facebook.presto.hive.metastore.SemiTransactionalHiveMetastore) MANAGED_TABLE(org.apache.hadoop.hive.metastore.TableType.MANAGED_TABLE) ConnectorSession(com.facebook.presto.spi.ConnectorSession) Domain(com.facebook.presto.spi.predicate.Domain) HiveColumnHandle.updateRowIdHandle(com.facebook.presto.hive.HiveColumnHandle.updateRowIdHandle) HivePrivilegeInfo(com.facebook.presto.hive.metastore.HivePrivilegeInfo) MoreObjects.toStringHelper(com.google.common.base.MoreObjects.toStringHelper) Iterables(com.google.common.collect.Iterables) Table(com.facebook.presto.hive.metastore.Table) Slice(io.airlift.slice.Slice) Database(com.facebook.presto.hive.metastore.Database) REGULAR(com.facebook.presto.hive.HiveColumnHandle.ColumnType.REGULAR) BUCKET_COLUMN_NAME(com.facebook.presto.hive.HiveColumnHandle.BUCKET_COLUMN_NAME) HiveUtil.columnExtraInfo(com.facebook.presto.hive.HiveUtil.columnExtraInfo) HiveTableProperties.getBucketProperty(com.facebook.presto.hive.HiveTableProperties.getBucketProperty) ConnectorOutputTableHandle(com.facebook.presto.spi.ConnectorOutputTableHandle) ArrayList(java.util.ArrayList) OptionalLong(java.util.OptionalLong) DIRECT_TO_TARGET_EXISTING_DIRECTORY(com.facebook.presto.hive.metastore.SemiTransactionalHiveMetastore.WriteMode.DIRECT_TO_TARGET_EXISTING_DIRECTORY) Type(com.facebook.presto.spi.type.Type) HiveUtil.encodeViewData(com.facebook.presto.hive.HiveUtil.encodeViewData) ImmutableMultimap(com.google.common.collect.ImmutableMultimap) HiveTableProperties.getHiveStorageFormat(com.facebook.presto.hive.HiveTableProperties.getHiveStorageFormat) HiveWriteUtils.checkTableIsWritable(com.facebook.presto.hive.HiveWriteUtils.checkTableIsWritable) ConnectorTableMetadata(com.facebook.presto.spi.ConnectorTableMetadata) PRESTO_VIEW_FLAG(com.facebook.presto.hive.HiveUtil.PRESTO_VIEW_FLAG) Properties(java.util.Properties) HiveWriteUtils.initializeSerializer(com.facebook.presto.hive.HiveWriteUtils.initializeSerializer) HIVE_CONCURRENT_MODIFICATION_DETECTED(com.facebook.presto.hive.HiveErrorCode.HIVE_CONCURRENT_MODIFICATION_DETECTED) Constraint(com.facebook.presto.spi.Constraint) IOException(java.io.IOException) ConnectorTableLayoutResult(com.facebook.presto.spi.ConnectorTableLayoutResult) HivePrivilege(com.facebook.presto.hive.metastore.HivePrivilegeInfo.HivePrivilege) SchemaTablePrefix(com.facebook.presto.spi.SchemaTablePrefix) ColumnHandle(com.facebook.presto.spi.ColumnHandle) ConnectorNewTableLayout(com.facebook.presto.spi.ConnectorNewTableLayout) TableType(org.apache.hadoop.hive.metastore.TableType) HiveUtil.hiveColumnHandles(com.facebook.presto.hive.HiveUtil.hiveColumnHandles) DIRECT_TO_TARGET_NEW_DIRECTORY(com.facebook.presto.hive.metastore.SemiTransactionalHiveMetastore.WriteMode.DIRECT_TO_TARGET_NEW_DIRECTORY) ConnectorViewDefinition(com.facebook.presto.spi.ConnectorViewDefinition) ViewNotFoundException(com.facebook.presto.spi.ViewNotFoundException) HiveTableProperties.getExternalLocation(com.facebook.presto.hive.HiveTableProperties.getExternalLocation) EXTERNAL_TABLE(org.apache.hadoop.hive.metastore.TableType.EXTERNAL_TABLE) HiveType.toHiveType(com.facebook.presto.hive.HiveType.toHiveType) STAGE_AND_MOVE_TO_TARGET_DIRECTORY(com.facebook.presto.hive.metastore.SemiTransactionalHiveMetastore.WriteMode.STAGE_AND_MOVE_TO_TARGET_DIRECTORY) Privilege(com.facebook.presto.spi.security.Privilege) FileSinkOperator(org.apache.hadoop.hive.ql.exec.FileSinkOperator) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) SchemaTableName(com.facebook.presto.spi.SchemaTableName) HiveSessionProperties.isBucketExecutionEnabled(com.facebook.presto.hive.HiveSessionProperties.isBucketExecutionEnabled) Collectors.toMap(java.util.stream.Collectors.toMap) Iterables.concat(com.google.common.collect.Iterables.concat) HIVE_COLUMN_ORDER_MISMATCH(com.facebook.presto.hive.HiveErrorCode.HIVE_COLUMN_ORDER_MISMATCH) BUCKETED_BY_PROPERTY(com.facebook.presto.hive.HiveTableProperties.BUCKETED_BY_PROPERTY) Path(org.apache.hadoop.fs.Path) DiscretePredicates(com.facebook.presto.spi.DiscretePredicates) NullableValue(com.facebook.presto.spi.predicate.NullableValue) Collectors.toSet(java.util.stream.Collectors.toSet) HIVE_UNSUPPORTED_FORMAT(com.facebook.presto.hive.HiveErrorCode.HIVE_UNSUPPORTED_FORMAT) ImmutableSet(com.google.common.collect.ImmutableSet) HIVE_UNKNOWN_ERROR(com.facebook.presto.hive.HiveErrorCode.HIVE_UNKNOWN_ERROR) ImmutableMap(com.google.common.collect.ImmutableMap) Predicate(java.util.function.Predicate) INVALID_SCHEMA_PROPERTY(com.facebook.presto.spi.StandardErrorCode.INVALID_SCHEMA_PROPERTY) Collections.emptyList(java.util.Collections.emptyList) Collection(java.util.Collection) SCHEMA_NOT_EMPTY(com.facebook.presto.spi.StandardErrorCode.SCHEMA_NOT_EMPTY) Collectors(java.util.stream.Collectors) Sets(com.google.common.collect.Sets) String.format(java.lang.String.format) STORAGE_FORMAT_PROPERTY(com.facebook.presto.hive.HiveTableProperties.STORAGE_FORMAT_PROPERTY) TupleDomain(com.facebook.presto.spi.predicate.TupleDomain) HIVE_STRING(com.facebook.presto.hive.HiveType.HIVE_STRING) List(java.util.List) ColumnMetadata(com.facebook.presto.spi.ColumnMetadata) NOT_SUPPORTED(com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED) USER(com.facebook.presto.hive.metastore.PrincipalType.USER) Optional(java.util.Optional) HIVE_TIMEZONE_MISMATCH(com.facebook.presto.hive.HiveErrorCode.HIVE_TIMEZONE_MISMATCH) PARTITION_KEY(com.facebook.presto.hive.HiveColumnHandle.ColumnType.PARTITION_KEY) HiveUtil.schemaTableName(com.facebook.presto.hive.HiveUtil.schemaTableName) HIDDEN(com.facebook.presto.hive.HiveColumnHandle.ColumnType.HIDDEN) JsonCodec(io.airlift.json.JsonCodec) TupleDomain.withColumnDomains(com.facebook.presto.spi.predicate.TupleDomain.withColumnDomains) ConnectorMetadata(com.facebook.presto.spi.connector.ConnectorMetadata) Column(com.facebook.presto.hive.metastore.Column) ConnectorTableLayoutHandle(com.facebook.presto.spi.ConnectorTableLayoutHandle) HIVE_WRITER_CLOSE_ERROR(com.facebook.presto.hive.HiveErrorCode.HIVE_WRITER_CLOSE_ERROR) ConnectorTableHandle(com.facebook.presto.spi.ConnectorTableHandle) PrestoException(com.facebook.presto.spi.PrestoException) Function(java.util.function.Function) Partition(com.facebook.presto.hive.metastore.Partition) ImmutableList(com.google.common.collect.ImmutableList) PARTITIONED_BY_PROPERTY(com.facebook.presto.hive.HiveTableProperties.PARTITIONED_BY_PROPERTY) Verify.verify(com.google.common.base.Verify.verify) Objects.requireNonNull(java.util.Objects.requireNonNull) HiveUtil.toPartitionValues(com.facebook.presto.hive.HiveUtil.toPartitionValues) HiveWriteUtils.isWritableType(com.facebook.presto.hive.HiveWriteUtils.isWritableType) ConnectorNodePartitioning(com.facebook.presto.spi.ConnectorNodePartitioning) ConnectorInsertTableHandle(com.facebook.presto.spi.ConnectorInsertTableHandle) ConnectorOutputMetadata(com.facebook.presto.spi.connector.ConnectorOutputMetadata) ConnectorTableLayout(com.facebook.presto.spi.ConnectorTableLayout) EXTERNAL_LOCATION_PROPERTY(com.facebook.presto.hive.HiveTableProperties.EXTERNAL_LOCATION_PROPERTY) VIEW_STORAGE_FORMAT(com.facebook.presto.hive.metastore.StorageFormat.VIEW_STORAGE_FORMAT) Maps(com.google.common.collect.Maps) JobConf(org.apache.hadoop.mapred.JobConf) Collectors.toList(java.util.stream.Collectors.toList) TableNotFoundException(com.facebook.presto.spi.TableNotFoundException) BUCKET_COUNT_PROPERTY(com.facebook.presto.hive.HiveTableProperties.BUCKET_COUNT_PROPERTY) StorageFormat.fromHiveStorageFormat(com.facebook.presto.hive.metastore.StorageFormat.fromHiveStorageFormat) HiveUtil.decodeViewData(com.facebook.presto.hive.HiveUtil.decodeViewData) VisibleForTesting(com.google.common.annotations.VisibleForTesting) MetastoreUtil.getHiveSchema(com.facebook.presto.hive.metastore.MetastoreUtil.getHiveSchema) Path(org.apache.hadoop.fs.Path) Table(com.facebook.presto.hive.metastore.Table) PrestoException(com.facebook.presto.spi.PrestoException) SchemaTableName(com.facebook.presto.spi.SchemaTableName) TableNotFoundException(com.facebook.presto.spi.TableNotFoundException) Column(com.facebook.presto.hive.metastore.Column) HiveTableProperties.getHiveStorageFormat(com.facebook.presto.hive.HiveTableProperties.getHiveStorageFormat) StorageFormat.fromHiveStorageFormat(com.facebook.presto.hive.metastore.StorageFormat.fromHiveStorageFormat) WriteMode(com.facebook.presto.hive.metastore.SemiTransactionalHiveMetastore.WriteMode)

Aggregations

Optional (java.util.Optional)414 List (java.util.List)208 Map (java.util.Map)138 ArrayList (java.util.ArrayList)107 Set (java.util.Set)95 Collectors (java.util.stream.Collectors)92 IOException (java.io.IOException)85 ImmutableList (com.google.common.collect.ImmutableList)82 ImmutableMap (com.google.common.collect.ImmutableMap)75 Test (org.junit.Test)69 ImmutableSet (com.google.common.collect.ImmutableSet)65 HashMap (java.util.HashMap)65 Path (java.nio.file.Path)55 Logger (org.slf4j.Logger)54 LoggerFactory (org.slf4j.LoggerFactory)53 Collections (java.util.Collections)51 Collection (java.util.Collection)48 Arrays (java.util.Arrays)42 BuildTarget (com.facebook.buck.model.BuildTarget)40 HashSet (java.util.HashSet)37