OpSparkListener

Instance Constructors

new OpSparkListener(appName: String, appId: String, runType: String, customTagName: Option[String], customTagValue: Option[String], logStageMetrics: Boolean, collectStageMetrics: Boolean)

appName

application name

appId

application id

runType

OpWorkflowRunType

customTagName

tag name printed on log lines

customTagValue

the value for the tag printed on log lines

logStageMetrics

should log metrics for every stage Note: can increase logging significantly if app has too many stages.

collectStageMetrics

should collect metrics for every stage Note: can increase memory usage on the driver if app has too many stages.

Value Members

final def !=(arg0: Any): Boolean

Definition Classes

AnyRef → Any
final def ##(): Int

Definition Classes

AnyRef → Any
final def ==(arg0: Any): Boolean

Definition Classes

AnyRef → Any
val appId: String

application id
val appName: String

application name
final def asInstanceOf[T0]: T0

Definition Classes

Any
def clone(): AnyRef

Attributes

protected[java.lang]

Definition Classes

AnyRef

Annotations

@throws( ... )
val collectStageMetrics: Boolean

should collect metrics for every stage Note: can increase memory usage on the driver if app has too many stages.
val customTagName: Option[String]

tag name printed on log lines
val customTagValue: Option[String]

the value for the tag printed on log lines
final def eq(arg0: AnyRef): Boolean

Definition Classes

AnyRef
def equals(arg0: Any): Boolean

Definition Classes

AnyRef → Any
def finalize(): Unit

Attributes

protected[java.lang]

Definition Classes

AnyRef

Annotations

@throws( classOf[java.lang.Throwable] )
final def getClass(): Class[_]

Definition Classes

AnyRef → Any
def hashCode(): Int

Definition Classes

AnyRef → Any
final def isInstanceOf[T0]: Boolean

Definition Classes

Any
var jobGroup: String

Attributes

protected
val logPrefix: String
val logStageMetrics: Boolean

should log metrics for every stage Note: can increase logging significantly if app has too many stages.
def metrics: AppMetrics

All the metrics computed by the spark listener
final def ne(arg0: AnyRef): Boolean

Definition Classes

AnyRef
final def notify(): Unit

Definition Classes

AnyRef
final def notifyAll(): Unit

Definition Classes

AnyRef
def onApplicationEnd(applicationEnd: SparkListenerApplicationEnd): Unit

Definition Classes

OpSparkListener → SparkListener → SparkListenerInterface
def onApplicationStart(applicationStart: SparkListenerApplicationStart): Unit

Definition Classes

OpSparkListener → SparkListener → SparkListenerInterface
def onBlockManagerAdded(blockManagerAdded: SparkListenerBlockManagerAdded): Unit

Definition Classes

SparkListener → SparkListenerInterface
def onBlockManagerRemoved(blockManagerRemoved: SparkListenerBlockManagerRemoved): Unit

Definition Classes

SparkListener → SparkListenerInterface
def onBlockUpdated(blockUpdated: SparkListenerBlockUpdated): Unit

Definition Classes

SparkListener → SparkListenerInterface
def onEnvironmentUpdate(environmentUpdate: SparkListenerEnvironmentUpdate): Unit

Definition Classes

SparkListener → SparkListenerInterface
def onExecutorAdded(executorAdded: SparkListenerExecutorAdded): Unit

Definition Classes

SparkListener → SparkListenerInterface
def onExecutorBlacklisted(executorBlacklisted: SparkListenerExecutorBlacklisted): Unit

Definition Classes

SparkListener → SparkListenerInterface
def onExecutorBlacklistedForStage(executorBlacklistedForStage: SparkListenerExecutorBlacklistedForStage): Unit

Definition Classes

SparkListener → SparkListenerInterface
def onExecutorMetricsUpdate(executorMetricsUpdate: SparkListenerExecutorMetricsUpdate): Unit

Definition Classes

SparkListener → SparkListenerInterface
def onExecutorRemoved(executorRemoved: SparkListenerExecutorRemoved): Unit

Definition Classes

SparkListener → SparkListenerInterface
def onExecutorUnblacklisted(executorUnblacklisted: SparkListenerExecutorUnblacklisted): Unit

Definition Classes

SparkListener → SparkListenerInterface
def onJobEnd(jobEnd: SparkListenerJobEnd): Unit

Definition Classes

OpSparkListener → SparkListener → SparkListenerInterface
def onJobStart(jobStart: SparkListenerJobStart): Unit

Definition Classes

OpSparkListener → SparkListener → SparkListenerInterface
def onNodeBlacklisted(nodeBlacklisted: SparkListenerNodeBlacklisted): Unit

Definition Classes

SparkListener → SparkListenerInterface
def onNodeBlacklistedForStage(nodeBlacklistedForStage: SparkListenerNodeBlacklistedForStage): Unit

Definition Classes

SparkListener → SparkListenerInterface
def onNodeUnblacklisted(nodeUnblacklisted: SparkListenerNodeUnblacklisted): Unit

Definition Classes

SparkListener → SparkListenerInterface
def onOtherEvent(event: SparkListenerEvent): Unit

Definition Classes

SparkListener → SparkListenerInterface
def onSpeculativeTaskSubmitted(speculativeTask: SparkListenerSpeculativeTaskSubmitted): Unit

Definition Classes

SparkListener → SparkListenerInterface
def onStageCompleted(stageCompleted: SparkListenerStageCompleted): Unit

Definition Classes

OpSparkListener → SparkListener → SparkListenerInterface
def onStageSubmitted(stageSubmitted: SparkListenerStageSubmitted): Unit

Definition Classes

SparkListener → SparkListenerInterface
def onTaskEnd(taskEnd: SparkListenerTaskEnd): Unit

Definition Classes

SparkListener → SparkListenerInterface
def onTaskGettingResult(taskGettingResult: SparkListenerTaskGettingResult): Unit

Definition Classes

SparkListener → SparkListenerInterface
def onTaskStart(taskStart: SparkListenerTaskStart): Unit

Definition Classes

SparkListener → SparkListenerInterface
def onUnpersistRDD(unpersistRDD: SparkListenerUnpersistRDD): Unit

Definition Classes

SparkListener → SparkListenerInterface
val runType: String

OpWorkflowRunType
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes

AnyRef
def toString(): String

Definition Classes

AnyRef → Any
final def wait(): Unit

Definition Classes

AnyRef

Annotations

@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes

AnyRef

Annotations

@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes

AnyRef

Annotations

@throws( ... )

Related Docs: object OpSparkListener | package spark

class OpSparkListener extends SparkListener

Instance Constructors

new OpSparkListener(appName: String, appId: String, runType: String, customTagName: Option[String], customTagValue: Option[String], logStageMetrics: Boolean, collectStageMetrics: Boolean)

Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

val appId: String

val appName: String

final def asInstanceOf[T0]: T0

def clone(): AnyRef

val collectStageMetrics: Boolean

val customTagName: Option[String]

val customTagValue: Option[String]

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def finalize(): Unit

final def getClass(): Class[_]

def hashCode(): Int

final def isInstanceOf[T0]: Boolean

var jobGroup: String

val logPrefix: String

val logStageMetrics: Boolean

def metrics: AppMetrics

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

def onApplicationEnd(applicationEnd: SparkListenerApplicationEnd): Unit

def onApplicationStart(applicationStart: SparkListenerApplicationStart): Unit

def onBlockManagerAdded(blockManagerAdded: SparkListenerBlockManagerAdded): Unit

def onBlockManagerRemoved(blockManagerRemoved: SparkListenerBlockManagerRemoved): Unit

def onBlockUpdated(blockUpdated: SparkListenerBlockUpdated): Unit

def onEnvironmentUpdate(environmentUpdate: SparkListenerEnvironmentUpdate): Unit

def onExecutorAdded(executorAdded: SparkListenerExecutorAdded): Unit

def onExecutorBlacklisted(executorBlacklisted: SparkListenerExecutorBlacklisted): Unit

def onExecutorBlacklistedForStage(executorBlacklistedForStage: SparkListenerExecutorBlacklistedForStage): Unit

def onExecutorMetricsUpdate(executorMetricsUpdate: SparkListenerExecutorMetricsUpdate): Unit

def onExecutorRemoved(executorRemoved: SparkListenerExecutorRemoved): Unit

def onExecutorUnblacklisted(executorUnblacklisted: SparkListenerExecutorUnblacklisted): Unit

def onJobEnd(jobEnd: SparkListenerJobEnd): Unit

def onJobStart(jobStart: SparkListenerJobStart): Unit

def onNodeBlacklisted(nodeBlacklisted: SparkListenerNodeBlacklisted): Unit

def onNodeBlacklistedForStage(nodeBlacklistedForStage: SparkListenerNodeBlacklistedForStage): Unit

def onNodeUnblacklisted(nodeUnblacklisted: SparkListenerNodeUnblacklisted): Unit

def onOtherEvent(event: SparkListenerEvent): Unit

def onSpeculativeTaskSubmitted(speculativeTask: SparkListenerSpeculativeTaskSubmitted): Unit

def onStageCompleted(stageCompleted: SparkListenerStageCompleted): Unit

def onStageSubmitted(stageSubmitted: SparkListenerStageSubmitted): Unit

def onTaskEnd(taskEnd: SparkListenerTaskEnd): Unit

def onTaskGettingResult(taskGettingResult: SparkListenerTaskGettingResult): Unit

def onTaskStart(taskStart: SparkListenerTaskStart): Unit

def onUnpersistRDD(unpersistRDD: SparkListenerUnpersistRDD): Unit

val runType: String

final def synchronized[T0](arg0: ⇒ T0): T0

def toString(): String

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

Inherited from SparkListener

Inherited from SparkListenerInterface

Inherited from AnyRef

Inherited from Any

Ungrouped