OpParams

Instance Constructors

new OpParams()
new OpParams(stageParams: Map[String, Map[String, Any]], readerParams: Map[String, ReaderParams], modelLocation: Option[String], writeLocation: Option[String], metricsLocation: Option[String], metricsCompress: Option[Boolean], metricsCodec: Option[String], batchDurationSecs: Option[Int], awaitTerminationTimeoutSecs: Option[Int], customTagName: Option[String], customTagValue: Option[String], logStageMetrics: Option[Boolean], collectStageMetrics: Option[Boolean], customParams: Map[String, Any], alternateReaderParams: Map[String, ReaderParams])

stageParams

a map of parameters to inject into stages. Format is Map(StageSimpleName -> Map(ParameterName -> Value)). Allows changing parameters away from defaults (and only defaults) at the level of stage type (all stages of the same type will get the same setting. Note: Will NOT override parameter values that have been set previously in code OR with a previous set of parameters.

readerParams

a map of parameters to inject into readers. Format is Map(ReaderType -> Map(ParameterName -> Value)). In order to use these parameters the read method in the reader must be overwritten to specifically take and use these parameters.

modelLocation

location to save model to or read model from

writeLocation

location to write out any data generated by flow

metricsLocation

location to write out any metrics generated by flow

metricsCompress

should compress metrics file

metricsCodec

compress with the supplied codec

batchDurationSecs

the time interval at which streaming data will be divided into batches

awaitTerminationTimeoutSecs

the time to await until streaming context termination

customTagName

tag name printed on log lines (e.g., by com.salesforce.op.utils.spark.OpSparkListener)

customTagValue

the value for the tag printed on log lines (e.g., by com.salesforce.op.utils.spark.OpSparkListener)

logStageMetrics

if com.salesforce.op.utils.spark.OpSparkListener should log metrics for every stage. Note: can increase logging significantly if app has too many stages.

collectStageMetrics

if com.salesforce.op.utils.spark.OpSparkListener should collect metrics for every stage. Note: can increase memory usage on the driver if app has too many stages.

customParams

any custom parameters

alternateReaderParams

a map of parameters to inject into readers - other than the main reader being called in the workflow run Format is Map(ReaderType -> Map(ParameterName -> Value)). In order to use these parameters the read method in the reader must be overwritten to specifically take and use these parameters.

Value Members

final def !=(arg0: Any): Boolean

Definition Classes

AnyRef → Any
final def ##(): Int

Definition Classes

AnyRef → Any
final def ==(arg0: Any): Boolean

Definition Classes

AnyRef → Any
val alternateReaderParams: Map[String, ReaderParams]

a map of parameters to inject into readers - other than the main reader being called in the workflow run Format is Map(ReaderType -> Map(ParameterName -> Value)).

a map of parameters to inject into readers - other than the main reader being called in the workflow run Format is Map(ReaderType -> Map(ParameterName -> Value)). In order to use these parameters the read method in the reader must be overwritten to specifically take and use these parameters.
final def asInstanceOf[T0]: T0

Definition Classes

Any
val awaitTerminationTimeoutSecs: Option[Int]

the time to await until streaming context termination
val batchDurationSecs: Option[Int]

the time interval at which streaming data will be divided into batches
def clone(): AnyRef

Attributes

protected[java.lang]

Definition Classes

AnyRef

Annotations

@throws( ... )
val collectStageMetrics: Option[Boolean]

if com.salesforce.op.utils.spark.OpSparkListener should collect metrics for every stage.

if com.salesforce.op.utils.spark.OpSparkListener should collect metrics for every stage. Note: can increase memory usage on the driver if app has too many stages.
def copy(stageParams: Map[String, Map[String, Any]] = this.stageParams, readerParams: Map[String, ReaderParams] = this.readerParams, modelLocation: Option[String] = this.modelLocation, writeLocation: Option[String] = this.writeLocation, metricsLocation: Option[String] = this.metricsLocation, metricsCompress: Option[Boolean] = this.metricsCompress, metricsCodec: Option[String] = this.metricsCodec, batchDurationSecs: Option[Int] = this.batchDurationSecs, awaitTerminationTimeoutSecs: Option[Int] = this.awaitTerminationTimeoutSecs, customTagName: Option[String] = this.customTagName, customTagValue: Option[String] = this.customTagValue, logStageMetrics: Option[Boolean] = this.logStageMetrics, collectStageMetrics: Option[Boolean] = this.collectStageMetrics, customParams: Map[String, Any] = this.customParams, alternateReaderParams: Map[String, ReaderParams] = this.alternateReaderParams): OpParams
val customParams: Map[String, Any]

any custom parameters
val customTagName: Option[String]

tag name printed on log lines (e.g., by com.salesforce.op.utils.spark.OpSparkListener)
val customTagValue: Option[String]

the value for the tag printed on log lines (e.g., by com.salesforce.op.utils.spark.OpSparkListener)
final def eq(arg0: AnyRef): Boolean

Definition Classes

AnyRef
def equals(arg0: Any): Boolean

Definition Classes

AnyRef → Any
def finalize(): Unit

Attributes

protected[java.lang]

Definition Classes

AnyRef

Annotations

@throws( classOf[java.lang.Throwable] )
final def getClass(): Class[_]

Definition Classes

AnyRef → Any
def hashCode(): Int

Definition Classes

AnyRef → Any
final def isInstanceOf[T0]: Boolean

Definition Classes

Any
val logStageMetrics: Option[Boolean]

if com.salesforce.op.utils.spark.OpSparkListener should log metrics for every stage.

if com.salesforce.op.utils.spark.OpSparkListener should log metrics for every stage. Note: can increase logging significantly if app has too many stages.
val metricsCodec: Option[String]

compress with the supplied codec
val metricsCompress: Option[Boolean]

should compress metrics file
val metricsLocation: Option[String]

location to write out any metrics generated by flow
val modelLocation: Option[String]

location to save model to or read model from
final def ne(arg0: AnyRef): Boolean

Definition Classes

AnyRef
final def notify(): Unit

Definition Classes

AnyRef
final def notifyAll(): Unit

Definition Classes

AnyRef
val readerParams: Map[String, ReaderParams]

a map of parameters to inject into readers.

a map of parameters to inject into readers. Format is Map(ReaderType -> Map(ParameterName -> Value)). In order to use these parameters the read method in the reader must be overwritten to specifically take and use these parameters.
val stageParams: Map[String, Map[String, Any]]

a map of parameters to inject into stages.

a map of parameters to inject into stages. Format is Map(StageSimpleName -> Map(ParameterName -> Value)). Allows changing parameters away from defaults (and only defaults) at the level of stage type (all stages of the same type will get the same setting. Note: Will NOT override parameter values that have been set previously in code OR with a previous set of parameters.
def switchReaderParams(): OpParams

Switch the reader params with the alternate reader params and return a new params instance this is necessary because the readers will always try to use readerParams so if you want to use the alternate reader params they need to be moved to readerParams

Switch the reader params with the alternate reader params and return a new params instance this is necessary because the readers will always try to use readerParams so if you want to use the alternate reader params they need to be moved to readerParams

returns

a new params instance
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes

AnyRef
def toJson(pretty: Boolean = true): String

Write this instance to json string

Write this instance to json string

pretty

should pretty print

returns

json string of the instance

Definition Classes

JsonLike
def toString(): String

This instance json string

This instance json string

returns

json string of the instance

Definition Classes

JsonLike → AnyRef → Any
final def wait(): Unit

Definition Classes

AnyRef

Annotations

@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes

AnyRef

Annotations

@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes

AnyRef

Annotations

@throws( ... )
def withValues(readLocations: Map[String, String] = Map.empty, writeLocation: Option[String] = None, modelLocation: Option[String] = None, metricsLocation: Option[String] = None, batchDurationSecs: Option[Int] = None, awaitTerminationTimeoutSecs: Option[Int] = None, alternateReadLocations: Map[String, String] = Map.empty): OpParams

Copy op params with new specified values

Copy op params with new specified values

readLocations

read locations

writeLocation

write locations

modelLocation

model location

metricsLocation

metrics location

batchDurationSecs

the time interval at which streaming data will be divided into batches

awaitTerminationTimeoutSecs

the time to await until streaming context termination

alternateReadLocations

read locations for readers other than the main reader in the workflow

returns

a copy of params with new values
val writeLocation: Option[String]

location to write out any data generated by flow

Related Docs: object OpParams | package op

final class OpParams extends JsonLike

Instance Constructors

new OpParams()

Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

val alternateReaderParams: Map[String, ReaderParams]

final def asInstanceOf[T0]: T0

val awaitTerminationTimeoutSecs: Option[Int]

val batchDurationSecs: Option[Int]

def clone(): AnyRef

val collectStageMetrics: Option[Boolean]

val customParams: Map[String, Any]

val customTagName: Option[String]

val customTagValue: Option[String]

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def finalize(): Unit

final def getClass(): Class[_]

def hashCode(): Int

final def isInstanceOf[T0]: Boolean

val logStageMetrics: Option[Boolean]

val metricsCodec: Option[String]

val metricsCompress: Option[Boolean]

val metricsLocation: Option[String]

val modelLocation: Option[String]

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

val readerParams: Map[String, ReaderParams]

val stageParams: Map[String, Map[String, Any]]

def switchReaderParams(): OpParams

final def synchronized[T0](arg0: ⇒ T0): T0

def toJson(pretty: Boolean = true): String

def toString(): String

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

val writeLocation: Option[String]

Inherited from JsonLike

Inherited from Serializable

Inherited from Serializable

Inherited from AnyRef

Inherited from Any

Ungrouped