OpApp

Instance Constructors

new OpApp()

Abstract Value Members

abstract def run(runType: OpWorkflowRunType, opParams: OpParams)(implicit spark: SparkSession, streaming: StreamingContext): Unit

The main function to run your OpWorkflow.

The main function to run your OpWorkflow. The easiest way is to create an OpWorkflowRunner and run it.

runType

run type

opParams

parameters injected at runtime

spark

spark session which runs the workflow

streaming

spark streaming context which runs the workflow

Concrete Value Members

final def !=(arg0: Any): Boolean

Definition Classes

AnyRef → Any
final def ##(): Int

Definition Classes

AnyRef → Any
final def ==(arg0: Any): Boolean

Definition Classes

AnyRef → Any
def appName: String

Application name (gets the value of 'spark.app.name' parameter).
final def asInstanceOf[T0]: T0

Definition Classes

Any
def clone(): AnyRef

Attributes

protected[java.lang]

Definition Classes

AnyRef

Annotations

@throws( ... )
def defaultAppName: String

Default application name - to be used if 'spark.app.name' parameter is not set.
final def eq(arg0: AnyRef): Boolean

Definition Classes

AnyRef
def equals(arg0: Any): Boolean

Definition Classes

AnyRef → Any
def finalize(): Unit

Attributes

protected[java.lang]

Definition Classes

AnyRef

Annotations

@throws( classOf[java.lang.Throwable] )
final def getClass(): Class[_]

Definition Classes

AnyRef → Any
def hashCode(): Int

Definition Classes

AnyRef → Any
final def isInstanceOf[T0]: Boolean

Definition Classes

Any
def kryoRegistrator: Class[_ <: OpKryoRegistrator]

Kryo registrar to use when creating a SparkConf.

Kryo registrar to use when creating a SparkConf.

First create your own registrator by extending the OpKryoRegistrator and then register your new classes by overriding OpKryoRegistrator.registerCustomClasses.

Then override this method to set your registrator with Spark.
def main(args: Array[String]): Unit

The main method - loads the params and runs the workflow according to parameter settings.

The main method - loads the params and runs the workflow according to parameter settings.

args

command line args to be parsed into OpWorkflowRunnerConfig
final def ne(arg0: AnyRef): Boolean

Definition Classes

AnyRef
final def notify(): Unit

Definition Classes

AnyRef
final def notifyAll(): Unit

Definition Classes

AnyRef
def parseArgs(args: Array[String]): (OpWorkflowRunType, OpParams)

Parse command line arguments as OpParams.

Parse command line arguments as OpParams.

args

command line arguments

returns

run type and OpParams
def sparkConf: SparkConf

Configuration for a Spark application.

Configuration for a Spark application. Used to set various Spark parameters as key-value pairs.

returns

SparkConf
def sparkSession: SparkSession

Gets/creates a Spark Session.
def sparkStreamingContext(batchDuration: Duration): StreamingContext

Gets/creates a Spark Streaming Context.

Gets/creates a Spark Streaming Context.

batchDuration

the time interval at which streaming data will be divided into batches
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes

AnyRef
def toString(): String

Definition Classes

AnyRef → Any
final def wait(): Unit

Definition Classes

AnyRef

Annotations

@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes

AnyRef

Annotations

@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes

AnyRef

Annotations

@throws( ... )

Related Doc: package op

abstract class OpApp extends AnyRef

Instance Constructors

new OpApp()

Abstract Value Members

abstract def run(runType: OpWorkflowRunType, opParams: OpParams)(implicit spark: SparkSession, streaming: StreamingContext): Unit

Concrete Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

def appName: String

final def asInstanceOf[T0]: T0

def clone(): AnyRef

def defaultAppName: String

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def finalize(): Unit

final def getClass(): Class[_]

def hashCode(): Int

final def isInstanceOf[T0]: Boolean

def kryoRegistrator: Class[_ <: OpKryoRegistrator]

def main(args: Array[String]): Unit

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

def parseArgs(args: Array[String]): (OpWorkflowRunType, OpParams)

def sparkConf: SparkConf

def sparkSession: SparkSession

def sparkStreamingContext(batchDuration: Duration): StreamingContext

final def synchronized[T0](arg0: ⇒ T0): T0

def toString(): String

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

Inherited from AnyRef

Inherited from Any

Ungrouped