Package org.apache.cassandra.spark.bulkwriter

Class BulkSparkConf

  • All Implemented Interfaces:
    java.io.Serializable
    public class BulkSparkConf
extends java.lang.Object
implements java.io.Serializable
    See Also:
    Serialized Form

    • Field Detail

      • DEFAULT_HTTP_CONNECTION_TIMEOUT

        public static final int DEFAULT_HTTP_CONNECTION_TIMEOUT
        See Also:
        Constant Field Values

      • DEFAULT_HTTP_RESPONSE_TIMEOUT

        public static final int DEFAULT_HTTP_RESPONSE_TIMEOUT
        See Also:
        Constant Field Values

      • DEFAULT_HTTP_MAX_CONNECTIONS

        public static final int DEFAULT_HTTP_MAX_CONNECTIONS
        See Also:
        Constant Field Values

      • DEFAULT_SIDECAR_REQUEST_RETRIES

        public static final int DEFAULT_SIDECAR_REQUEST_RETRIES
        See Also:
        Constant Field Values

      • DEFAULT_SIDECAR_REQUEST_RETRY_DELAY_MILLIS

        public static final long DEFAULT_SIDECAR_REQUEST_RETRY_DELAY_MILLIS

      • DEFAULT_SIDECAR_REQUEST_MAX_RETRY_DELAY_MILLIS

        public static final long DEFAULT_SIDECAR_REQUEST_MAX_RETRY_DELAY_MILLIS

      • DEFAULT_SIDECAR_REQUEST_TIMEOUT_SECONDS

        public static final int DEFAULT_SIDECAR_REQUEST_TIMEOUT_SECONDS
        See Also:
        Constant Field Values

      • DEFAULT_COMMIT_BATCH_SIZE

        public static final int DEFAULT_COMMIT_BATCH_SIZE
        See Also:
        Constant Field Values

      • DEFAULT_RING_RETRY_COUNT

        public static final int DEFAULT_RING_RETRY_COUNT
        See Also:
        Constant Field Values

      • DEFAULT_SSTABLE_DATA_SIZE_IN_MIB

        public static final int DEFAULT_SSTABLE_DATA_SIZE_IN_MIB
        See Also:
        Constant Field Values

      • DEFAULT_STORAGE_CLIENT_KEEP_ALIVE_SECONDS

        public static final long DEFAULT_STORAGE_CLIENT_KEEP_ALIVE_SECONDS
        See Also:
        Constant Field Values

      • DEFAULT_STORAGE_CLIENT_CONCURRENCY

        public static final int DEFAULT_STORAGE_CLIENT_CONCURRENCY

      • DEFAULT_STORAGE_CLIENT_MAX_CHUNK_SIZE_IN_BYTES

        public static final int DEFAULT_STORAGE_CLIENT_MAX_CHUNK_SIZE_IN_BYTES
        See Also:
        Constant Field Values

      • CASSANDRA_VERSION

        public static final java.lang.String CASSANDRA_VERSION
        See Also:
        Constant Field Values

      • HTTP_MAX_CONNECTIONS

        public static final java.lang.String HTTP_MAX_CONNECTIONS
        See Also:
        Constant Field Values

      • HTTP_RESPONSE_TIMEOUT

        public static final java.lang.String HTTP_RESPONSE_TIMEOUT
        See Also:
        Constant Field Values

      • HTTP_CONNECTION_TIMEOUT

        public static final java.lang.String HTTP_CONNECTION_TIMEOUT
        See Also:
        Constant Field Values

      • SIDECAR_REQUEST_RETRIES

        public static final java.lang.String SIDECAR_REQUEST_RETRIES
        See Also:
        Constant Field Values

      • SIDECAR_REQUEST_RETRY_DELAY_MILLIS

        public static final java.lang.String SIDECAR_REQUEST_RETRY_DELAY_MILLIS
        See Also:
        Constant Field Values

      • SIDECAR_REQUEST_MAX_RETRY_DELAY_MILLIS

        public static final java.lang.String SIDECAR_REQUEST_MAX_RETRY_DELAY_MILLIS
        See Also:
        Constant Field Values

      • SIDECAR_REQUEST_TIMEOUT_SECONDS

        public static final java.lang.String SIDECAR_REQUEST_TIMEOUT_SECONDS
        See Also:
        Constant Field Values

      • RING_RETRY_COUNT

        public static final java.lang.String RING_RETRY_COUNT
        See Also:
        Constant Field Values

      • IMPORT_COORDINATOR_TIMEOUT_MULTIPLIER

        public static final java.lang.String IMPORT_COORDINATOR_TIMEOUT_MULTIPLIER
        See Also:
        Constant Field Values

      • MINIMUM_JOB_KEEP_ALIVE_MINUTES

        public static final int MINIMUM_JOB_KEEP_ALIVE_MINUTES
        See Also:
        Constant Field Values

      • keyspace

        public final java.lang.String keyspace

      • table

        public final java.lang.String table

      • localDC

        public final java.lang.String localDC

      • numberSplits

        public final java.lang.Integer numberSplits

      • sstableDataSizeInMiB

        public final java.lang.Integer sstableDataSizeInMiB

      • commitBatchSize

        public final int commitBatchSize

      • skipExtendedVerify

        public final boolean skipExtendedVerify

      • writeMode

        public final WriteMode writeMode

      • commitThreadsPerInstance

        public final int commitThreadsPerInstance

      • importCoordinatorTimeoutMultiplier

        public final double importCoordinatorTimeoutMultiplier

      • quoteIdentifiers

        public boolean quoteIdentifiers

      • keystorePassword

        protected final java.lang.String keystorePassword

      • keystorePath

        protected final java.lang.String keystorePath

      • keystoreBase64Encoded

        protected final java.lang.String keystoreBase64Encoded

      • keystoreType

        protected final java.lang.String keystoreType

      • cassandraRole

        protected final java.lang.String cassandraRole

      • truststorePassword

        protected final java.lang.String truststorePassword

      • truststorePath

        protected final java.lang.String truststorePath

      • truststoreBase64Encoded

        protected final java.lang.String truststoreBase64Encoded

      • truststoreType

        protected final java.lang.String truststoreType

      • ttl

        protected final java.lang.String ttl

      • timestamp

        protected final java.lang.String timestamp

      • conf

        protected final org.apache.spark.SparkConf conf

      • effectiveSidecarPort

        protected final int effectiveSidecarPort

      • userProvidedSidecarPort

        protected final int userProvidedSidecarPort

      • jobKeepAliveMinutes

        protected final int jobKeepAliveMinutes

      • jobTimeoutSeconds

        protected final long jobTimeoutSeconds

      • configuredJobId

        protected final java.lang.String configuredJobId

      • useOpenSsl

        protected boolean useOpenSsl

      • ringRetryCount

        protected int ringRetryCount

      • coordinatedWriteConfJson

        protected final java.lang.String coordinatedWriteConfJson

    • Constructor Detail

      • BulkSparkConf

        public BulkSparkConf​(org.apache.spark.SparkConf conf,
                     java.util.Map<java.lang.String,​java.lang.String> options)

      • BulkSparkConf

        public BulkSparkConf​(org.apache.spark.SparkConf conf,
                     java.util.Map<java.lang.String,​java.lang.String> options,
                     @Nullable
                     org.slf4j.Logger logger)

    • Method Detail

      • digestAlgorithmSupplierFromOptions

        @NotNull
protected DigestAlgorithmSupplier digestAlgorithmSupplierFromOptions​(DataTransport dataTransport,
                                                                     java.util.Map<java.lang.String,​java.lang.String> options)
        Returns the supplier for the digest algorithm from the configured options.
        Parameters:
        options - a key-value map with options for the bulk write job
        Returns:
        the configured DigestAlgorithmSupplier

      • resolveSSTableDataSizeInMiB

        public static int resolveSSTableDataSizeInMiB​(java.util.Map<java.lang.String,​java.lang.String> options)

      • resolveSidecarContactPoints

        public static java.lang.String resolveSidecarContactPoints​(java.util.Map<java.lang.String,​java.lang.String> options)

      • buildSidecarContactPoints

        protected java.util.Set<o.a.c.sidecar.client.shaded.client.SidecarInstance> buildSidecarContactPoints()

      • isCoordinatedWriteConfigured

        public boolean isCoordinatedWriteConfigured()

      • buildCoordinatedWriteConf

        @Nullable
protected CoordinatedWriteConf buildCoordinatedWriteConf​(DataTransport dataTransport,
                                                         @Nullable
                                                         org.slf4j.Logger logger)

      • validateEnvironment

        protected void validateEnvironment()
                            throws java.lang.RuntimeException
        Throws:
        java.lang.RuntimeException

      • validateSslConfiguration

        public void validateSslConfiguration()
        Validates the SSL configuration present and throws an exception if it is incorrect
        Throws:
        java.lang.NullPointerException - if the mTLS KeyStore password is provided, but both file path and base64 string are missing; or if either mTLS TrustStore file path or base64 string is provided, but the password is missing
        java.lang.IllegalArgumentException - if the mTLS TrustStore password is provided, but both file path and base64 string are missing

      • getUserProvidedSidecarPort

        public int getUserProvidedSidecarPort()

      • getEffectiveSidecarPort

        public int getEffectiveSidecarPort()

      • getTrustStorePath

        protected java.lang.String getTrustStorePath()

      • getTTLOptions

        public TTLOption getTTLOptions()

      • getTruststoreBase64Encoded

        protected java.lang.String getTruststoreBase64Encoded()

      • getTrustStoreTypeOrDefault

        public java.lang.String getTrustStoreTypeOrDefault()

      • getKeyStorePath

        protected java.lang.String getKeyStorePath()

      • getKeystoreBase64Encoded

        protected java.lang.String getKeystoreBase64Encoded()

      • getKeyStore

        public java.io.InputStream getKeyStore()

      • getCassandraRole

        @Nullable
public java.lang.String getCassandraRole()

      • getTrustStore

        @Nullable
public java.io.InputStream getTrustStore()

      • getKeyStoreFromPath

        protected java.io.InputStream getKeyStoreFromPath​(java.lang.String keyStorePath)

      • getKeyStoreFromBase64EncodedString

        protected java.io.InputStream getKeyStoreFromBase64EncodedString​(java.lang.String keyStoreBase64Encoded)

      • getTrustStorePasswordOrDefault

        public java.lang.String getTrustStorePasswordOrDefault()

      • getKeyStoreTypeOrDefault

        public java.lang.String getKeyStoreTypeOrDefault()

      • getKeyStorePassword

        public java.lang.String getKeyStorePassword()

      • getConfiguredTrustStorePassword

        public java.lang.String getConfiguredTrustStorePassword()

      • getConfiguredKeyStorePassword

        public java.lang.String getConfiguredKeyStorePassword()

      • getSidecarRequestRetries

        public int getSidecarRequestRetries()

      • getSidecarRequestRetryDelayMillis

        public long getSidecarRequestRetryDelayMillis()

      • getSidecarRequestMaxRetryDelayMillis

        public long getSidecarRequestMaxRetryDelayMillis()

      • getSidecarRequestTimeoutSeconds

        public int getSidecarRequestTimeoutSeconds()

      • getHttpConnectionTimeoutMs

        public int getHttpConnectionTimeoutMs()

      • getHttpResponseTimeoutMs

        public int getHttpResponseTimeoutMs()

      • getMaxHttpConnections

        public int getMaxHttpConnections()

      • getSkipClean

        public boolean getSkipClean()

      • getCores

        public java.lang.Integer getCores()

      • getJobKeepAliveMinutes

        public int getJobKeepAliveMinutes()

      • getJobTimeoutSeconds

        public long getJobTimeoutSeconds()

      • getDouble

        protected double getDouble​(java.lang.String settingName,
                           double defaultValue)

      • getInt

        protected int getInt​(java.lang.String settingName,
                     int defaultValue)

      • getOptionalInt

        protected java.util.Optional<java.lang.Integer> getOptionalInt​(java.lang.String settingName)

      • getLong

        protected long getLong​(java.lang.String settingName,
                       long defaultValue)

      • getBoolean

        protected boolean getBoolean​(java.lang.String settingName,
                             boolean defaultValue)

      • getSettingNameOrDeprecatedName

        protected java.lang.String getSettingNameOrDeprecatedName​(java.lang.String settingName)

      • getDeprecatedSettingPrefixes

        @NotNull
protected java.util.List<java.lang.String> getDeprecatedSettingPrefixes()

      • setupSparkConf

        public static void setupSparkConf​(org.apache.spark.SparkConf conf,
                                  boolean addKryoRegistrator)
        The SBW utilizes Cassandra libraries to generate SSTables. Under JDK11, this library needs additional JVM options to be set for the executors for some backward-compatibility reasons. This method will add the appropriate JVM options. Additionally, we set up the SBW KryoRegistrator here rather than requiring the end-user to call 2 static methods to set up the appropriate settings for the job.
        Parameters:
        conf - the Spark Configuration to set up
        addKryoRegistrator - passs true if your application hasn't separately added the bulk-specific Kryo registrator, false if you have set it up separately (see the usage docs for more details)

      • getSparkConf

        public org.apache.spark.SparkConf getSparkConf()

      • getUseOpenSsl

        public boolean getUseOpenSsl()

      • getRingRetryCount

        public int getRingRetryCount()

      • hasKeystoreAndKeystorePassword

        public boolean hasKeystoreAndKeystorePassword()

      • hasTruststoreAndTruststorePassword

        public boolean hasTruststoreAndTruststorePassword()