Reader

Abstract Value Members

abstract def generateDataFrame(rawFeatures: Array[OPFeature], opParams: OpParams)(implicit spark: SparkSession): DataFrame

Generate the dataframe that will be used in the OpPipeline calling this method

Generate the dataframe that will be used in the OpPipeline calling this method

rawFeatures

features to generate from the dataset read in by this reader

opParams

op parameters

spark

spark instance to do the reading and conversion from RDD to Dataframe

returns

A dataframe containing columns with all of the raw input features expected by the pipeline
abstract def subReaders: Seq[DataReader[_]]

All the reader's sub readers (used in joins)

All the reader's sub readers (used in joins)

returns

sub readers
implicit abstract val wtt: scala.reflect.api.JavaUniverse.WeakTypeTag[T]

Reader type tag

Reader type tag

Definition Classes

ReaderType

Concrete Value Members

final def !=(arg0: Any): Boolean

Definition Classes

AnyRef → Any
final def ##(): Int

Definition Classes

AnyRef → Any
final def ==(arg0: Any): Boolean

Definition Classes

AnyRef → Any
final def asInstanceOf[T0]: T0

Definition Classes

Any
def clone(): AnyRef

Attributes

protected[java.lang]

Definition Classes

AnyRef

Annotations

@throws( ... )
final def eq(arg0: AnyRef): Boolean

Definition Classes

AnyRef
def equals(arg0: Any): Boolean

Definition Classes

AnyRef → Any
def finalize(): Unit

Attributes

protected[java.lang]

Definition Classes

AnyRef

Annotations

@throws( classOf[java.lang.Throwable] )
final def fullTypeName: String

Full reader input type name

Full reader input type name

returns

full input type name

Definition Classes

ReaderType
final def getClass(): Class[_]

Definition Classes

AnyRef → Any
def getGenStage[I](f: OPFeature): FeatureGeneratorStage[I, _ <: FeatureType]

Attributes

protected[com.salesforce.op]
final def getReaderParams(opParams: OpParams): Option[ReaderParams]

Default method for extracting this reader's parameters from readerParams in OpParams

Default method for extracting this reader's parameters from readerParams in OpParams

opParams

contains map of reader type to ReaderParams instances

returns

ReaderParams instance if it exists

Definition Classes

ReaderType
def hashCode(): Int

Definition Classes

AnyRef → Any
final def innerJoin[U](other: DataReader[U], joinKeys: JoinKeys = JoinKeys()): JoinedDataReader[T, U]

Inner join

Inner join

U

Type of data read by right data reader

other

reader from right side of join

joinKeys

join keys to use

returns

joined reader
final def isInstanceOf[T0]: Boolean

Definition Classes

Any
final def join[U](other: DataReader[U], joinType: JoinType, joinKeys: JoinKeys = JoinKeys()): JoinedDataReader[T, U]

Join readers

Join readers

U

Type of data read by right data reader

other

reader from right side of join

joinType

type of join to perform

joinKeys

join keys to use

returns

joined reader

Attributes

protected
final def leftOuterJoin[U](other: DataReader[U], joinKeys: JoinKeys = JoinKeys()): JoinedDataReader[T, U]

Left Outer join

Left Outer join

U

Type of data read by right data reader

other

reader from right side of join

joinKeys

join keys to use

returns

joined reader
final def ne(arg0: AnyRef): Boolean

Definition Classes

AnyRef
final def notify(): Unit

Definition Classes

AnyRef
final def notifyAll(): Unit

Definition Classes

AnyRef
final def outerJoin[U](other: DataReader[U], joinKeys: JoinKeys = JoinKeys()): JoinedDataReader[T, U]

Outer join

Outer join

U

Type of data read by right data reader

other

reader from right side of join

joinKeys

join keys to use

returns

joined reader
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes

AnyRef
def toString(): String

Definition Classes

AnyRef → Any
final def typeName: String

Short reader input type name

Short reader input type name

returns

short reader input type name

Definition Classes

ReaderType
final def wait(): Unit

Definition Classes

AnyRef

Annotations

@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes

AnyRef

Annotations

@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes

AnyRef

Annotations

@throws( ... )

Related Doc: package readers

trait Reader[T] extends ReaderType[T]

Abstract Value Members

abstract def generateDataFrame(rawFeatures: Array[OPFeature], opParams: OpParams)(implicit spark: SparkSession): DataFrame

abstract def subReaders: Seq[DataReader[_]]

implicit abstract val wtt: scala.reflect.api.JavaUniverse.WeakTypeTag[T]

Concrete Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

final def asInstanceOf[T0]: T0

def clone(): AnyRef

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def finalize(): Unit

final def fullTypeName: String

final def getClass(): Class[_]

def getGenStage[I](f: OPFeature): FeatureGeneratorStage[I, _ <: FeatureType]

final def getReaderParams(opParams: OpParams): Option[ReaderParams]

def hashCode(): Int

final def innerJoin[U](other: DataReader[U], joinKeys: JoinKeys = JoinKeys()): JoinedDataReader[T, U]

final def isInstanceOf[T0]: Boolean

final def join[U](other: DataReader[U], joinType: JoinType, joinKeys: JoinKeys = JoinKeys()): JoinedDataReader[T, U]

final def leftOuterJoin[U](other: DataReader[U], joinKeys: JoinKeys = JoinKeys()): JoinedDataReader[T, U]

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

final def outerJoin[U](other: DataReader[U], joinKeys: JoinKeys = JoinKeys()): JoinedDataReader[T, U]

final def synchronized[T0](arg0: ⇒ T0): T0

def toString(): String

final def typeName: String

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

Inherited from ReaderType[T]

Inherited from Serializable

Inherited from Serializable

Inherited from AnyRef

Inherited from Any

Ungrouped