RealVectorizer

Instance Constructors

new RealVectorizer(uid: String = UID[RealVectorizer[_]], operationName: String = "vecReal")(implicit tti: scala.reflect.api.JavaUniverse.TypeTag[T], ttiv: scala.reflect.api.JavaUniverse.TypeTag[Option[Double]])

uid

uid for instance

Type Members

final type InputFeatures = Array[FeatureLike[T]]

Input Features type

Input Features type

Definition Classes

OpPipelineStageN → OpPipelineStage → InputParams
final type OutputFeatures = FeatureLike[OPVector]

Definition Classes

OpPipelineStage → OpPipelineStageBase

Value Members

final def !=(arg0: Any): Boolean

Definition Classes

AnyRef → Any
final def ##(): Int

Definition Classes

AnyRef → Any
final def $[T](param: Param[T]): T

Attributes

protected

Definition Classes

Params
final def ==(arg0: Any): Boolean

Definition Classes

AnyRef → Any
final def asInstanceOf[T0]: T0

Definition Classes

Any
implicit def booleanToDouble(v: Boolean): Double

Definition Classes

VectorizerDefaults
final def checkInputLength(features: Array[_]): Boolean

Checks the input length

Checks the input length

features

input features

returns

true is input size as expected, false otherwise

Definition Classes

OpPipelineStageN → InputParams
final def checkSerializable: Try[Unit]

Check if the stage is serializable

Check if the stage is serializable

returns

Failure if not serializable

Definition Classes

SequenceEstimator → OpPipelineStageBase
final def clear(param: Param[_]): RealVectorizer.this.type

Definition Classes

Params
def clone(): AnyRef

Attributes

protected[java.lang]

Definition Classes

AnyRef

Annotations

@throws( ... )
final def copy(extra: ParamMap): RealVectorizer.this.type

This method is used to make a copy of the instance with new parameters in several methods in spark internals Default will find the constructor and make a copy for any class (AS LONG AS ALL CONSTRUCTOR PARAMS ARE VALS, this is why type tags are written as implicit vals in base classes).

This method is used to make a copy of the instance with new parameters in several methods in spark internals Default will find the constructor and make a copy for any class (AS LONG AS ALL CONSTRUCTOR PARAMS ARE VALS, this is why type tags are written as implicit vals in base classes).

Note: that the convention in spark is to have the uid be a constructor argument, so that copies will share a uid with the original (developers should follow this convention).

extra

new parameters want to add to instance

returns

a new instance with the same uid

Definition Classes

OpPipelineStageBase → Params
def copyValues[T <: Params](to: T, extra: ParamMap): T

Attributes

protected

Definition Classes

Params
final def defaultCopy[T <: Params](extra: ParamMap): T

Attributes

protected

Definition Classes

Params
final def eq(arg0: AnyRef): Boolean

Definition Classes

AnyRef
def equals(arg0: Any): Boolean

Definition Classes

AnyRef → Any
def explainParam(param: Param[_]): String

Definition Classes

Params
def explainParams(): String

Definition Classes

Params
final def extractParamMap(): ParamMap

Definition Classes

Params
final def extractParamMap(extra: ParamMap): ParamMap

Definition Classes

Params
final val fillValue: DoubleParam
def finalize(): Unit

Attributes

protected[java.lang]

Definition Classes

AnyRef

Annotations

@throws( classOf[java.lang.Throwable] )
def fit(dataset: Dataset[_]): SequenceModel[T, OPVector]

Spark operation on dataset to produce Dataset for constructor fit function and then turn output function into a Model

Spark operation on dataset to produce Dataset for constructor fit function and then turn output function into a Model

dataset

input data for this stage

returns

a fitted model that will perform the transformation specified by the function defined in constructor fit

Definition Classes

SequenceEstimator → Estimator
def fit(dataset: Dataset[_], paramMaps: Array[ParamMap]): Seq[SequenceModel[T, OPVector]]

Definition Classes

Estimator

Annotations

@Since( "2.0.0" )
def fit(dataset: Dataset[_], paramMap: ParamMap): SequenceModel[T, OPVector]

Definition Classes

Estimator

Annotations

@Since( "2.0.0" )
def fit(dataset: Dataset[_], firstParamPair: ParamPair[_], otherParamPairs: ParamPair[_]*): SequenceModel[T, OPVector]

Definition Classes

Estimator

Annotations

@Since( "2.0.0" ) @varargs()
def fitFn(dataset: Dataset[Seq[Option[Double]]]): SequenceModel[T, OPVector]

Function that fits the sequence model

Function that fits the sequence model

Definition Classes

RealVectorizer → SequenceEstimator
final def get[T](param: Param[T]): Option[T]

Definition Classes

Params
final def getClass(): Class[_]

Definition Classes

AnyRef → Any
final def getDefault[T](param: Param[T]): Option[T]

Definition Classes

Params
final def getInputFeature[T <: FeatureType](i: Int): Option[FeatureLike[T]]

Gets an input feature Note: this method IS NOT safe to use outside the driver, please use getTransientFeature method instead

Gets an input feature Note: this method IS NOT safe to use outside the driver, please use getTransientFeature method instead

returns

array of features

Definition Classes

InputParams

Exceptions thrown

NoSuchElementException if the features are not set

RuntimeException in case one of the features is null
final def getInputFeatures(): Array[OPFeature]

Gets the input features Note: this method IS NOT safe to use outside the driver, please use getTransientFeatures method instead

Gets the input features Note: this method IS NOT safe to use outside the driver, please use getTransientFeatures method instead

returns

array of features

Definition Classes

InputParams

Exceptions thrown

NoSuchElementException if the features are not set

RuntimeException in case one of the features is null
final def getInputSchema(): StructType

Definition Classes

OpPipelineStageParams
final def getMetadata(): Metadata

Definition Classes

OpPipelineStageParams
final def getOrDefault[T](param: Param[T]): T

Definition Classes

Params
def getOutput(): FeatureLike[OPVector]

Output features that will be created by this stage

Output features that will be created by this stage

returns

feature of type OutputFeatures

Definition Classes

HasOut → OpPipelineStageBase
final def getOutputFeatureName: String

Name of output feature (i.e.

Name of output feature (i.e. column created by this stage)

Definition Classes

OpPipelineStage
def getParam(paramName: String): Param[Any]

Definition Classes

Params
final def getTransientFeature(i: Int): Option[TransientFeature]

Gets an input feature at index i

Gets an input feature at index i

i

input index

returns

maybe an input feature

Definition Classes

InputParams
final def getTransientFeatures(): Array[TransientFeature]

Gets the input Features

Gets the input Features

returns

input features

Definition Classes

InputParams
final def hasDefault[T](param: Param[T]): Boolean

Definition Classes

Params
def hasParam(paramName: String): Boolean

Definition Classes

Params
def hashCode(): Int

Definition Classes

AnyRef → Any
final def inN: Array[TransientFeature]

Attributes

protected

Definition Classes

HasInN
def initializeLogIfNecessary(isInterpreter: Boolean, silent: Boolean): Boolean

Attributes

protected

Definition Classes

Logging
def initializeLogIfNecessary(isInterpreter: Boolean): Unit

Attributes

protected

Definition Classes

Logging
final def inputAsArray(in: InputFeatures): Array[OPFeature]

Function to convert InputFeatures to an Array of FeatureLike

Function to convert InputFeatures to an Array of FeatureLike

returns

an Array of FeatureLike

Definition Classes

OpPipelineStageN → InputParams
final def isDefined(param: Param[_]): Boolean

Definition Classes

Params
final def isInstanceOf[T0]: Boolean

Definition Classes

Any
final def isSet(param: Param[_]): Boolean

Definition Classes

Params
def isTraceEnabled(): Boolean

Attributes

protected

Definition Classes

Logging
def log: Logger

Attributes

protected

Definition Classes

Logging
def logDebug(msg: ⇒ String, throwable: Throwable): Unit

Attributes

protected

Definition Classes

Logging
def logDebug(msg: ⇒ String): Unit

Attributes

protected

Definition Classes

Logging
def logError(msg: ⇒ String, throwable: Throwable): Unit

Attributes

protected

Definition Classes

Logging
def logError(msg: ⇒ String): Unit

Attributes

protected

Definition Classes

Logging
def logInfo(msg: ⇒ String, throwable: Throwable): Unit

Attributes

protected

Definition Classes

Logging
def logInfo(msg: ⇒ String): Unit

Attributes

protected

Definition Classes

Logging
def logName: String

Attributes

protected

Definition Classes

Logging
def logTrace(msg: ⇒ String, throwable: Throwable): Unit

Attributes

protected

Definition Classes

Logging
def logTrace(msg: ⇒ String): Unit

Attributes

protected

Definition Classes

Logging
def logWarning(msg: ⇒ String, throwable: Throwable): Unit

Attributes

protected

Definition Classes

Logging
def logWarning(msg: ⇒ String): Unit

Attributes

protected

Definition Classes

Logging
final def ne(arg0: AnyRef): Boolean

Definition Classes

AnyRef
final def notify(): Unit

Definition Classes

AnyRef
final def notifyAll(): Unit

Definition Classes

AnyRef
def onGetMetadata(): Unit

Function to be called on getMetadata

Function to be called on getMetadata

Attributes

protected

Definition Classes

OpPipelineStageParams
def onSetInput(): Unit

Function to be called on setInput

Function to be called on setInput

Definition Classes

VectorizerDefaults → InputParams
val operationName: String

unique name of the operation this stage performs

unique name of the operation this stage performs

Definition Classes

SequenceEstimator → OpPipelineStageBase
final def outputAsArray(out: OutputFeatures): Array[OPFeature]

Function to convert OutputFeatures to an Array of FeatureLike

Function to convert OutputFeatures to an Array of FeatureLike

returns

an Array of FeatureLike

Definition Classes

OpPipelineStage → OpPipelineStageBase
def outputFeatureUid: String

Attributes

protected[com.salesforce.op]

Definition Classes

OpPipelineStageN → OpPipelineStage
def outputIsResponse: Boolean

Should output feature be a response? Yes, if any of the input features are.

Should output feature be a response? Yes, if any of the input features are.

returns

true if the the output feature should be a response

Definition Classes

OpPipelineStage
def outputVectorMeta: OpVectorMetadata

Get the metadata describing the output vector

Get the metadata describing the output vector

This does not trigger onGetMetadata()

returns

Metadata of output vector

Attributes

protected

Definition Classes

VectorizerDefaults
lazy val params: Array[Param[_]]

Definition Classes

Params
def save(path: String): Unit

Definition Classes

MLWritable

Annotations

@Since( "1.6.0" ) @throws( ... )
val seqIConvert: FeatureTypeSparkConverter[T]

Definition Classes

SequenceEstimator
implicit val seqIEncoder: Encoder[Seq[T.Value]]

Definition Classes

SequenceEstimator
final def set(paramPair: ParamPair[_]): RealVectorizer.this.type

Attributes

protected

Definition Classes

Params
final def set(param: String, value: Any): RealVectorizer.this.type

Attributes

protected

Definition Classes

Params
final def set[T](param: Param[T], value: T): RealVectorizer.this.type

Definition Classes

Params
final def setDefault(paramPairs: ParamPair[_]*): RealVectorizer.this.type

Attributes

protected

Definition Classes

Params
final def setDefault[T](param: Param[T], value: T): RealVectorizer.this.type

Attributes

protected

Definition Classes

Params
def setFillWithConstant(value: Double): RealVectorizer.this.type
def setFillWithMean: RealVectorizer.this.type
final def setInput(features: FeatureLike[T]*): RealVectorizer.this.type

Definition Classes

OpPipelineStageN
final def setInput(features: InputFeatures): RealVectorizer.this.type

Input features that will be used by the stage

Input features that will be used by the stage

returns

feature of type InputFeatures

Definition Classes

OpPipelineStageBase
final def setInputFeatures[S <: OPFeature](features: Array[S]): RealVectorizer.this.type

Sets input features

Sets input features

S

feature like type

features

array of input features

returns

this stage

Attributes

protected

Definition Classes

InputParams
final def setMetadata(m: Metadata): RealVectorizer.this.type

Definition Classes

OpPipelineStageParams
def setOutputFeatureName(name: String): RealVectorizer.this.type

Definition Classes

OpPipelineStage
def setTrackNulls(v: Boolean): RealVectorizer.this.type

Option to keep track of values that were missing

Option to keep track of values that were missing

Definition Classes

TrackNullsParam
final def stageName: String

Stage unique name consisting of the stage operation name and uid

Stage unique name consisting of the stage operation name and uid

returns

stage name

Definition Classes

OpPipelineStageBase
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes

AnyRef
def toString(): String

Definition Classes

Identifiable → AnyRef → Any
final val trackNulls: BooleanParam

Definition Classes

TrackNullsParam
final def transformSchema(schema: StructType): StructType

This function translates the input and output features into spark schema checks and changes that will occur on the underlying data frame

This function translates the input and output features into spark schema checks and changes that will occur on the underlying data frame

schema

schema of the input data frame

returns

a new schema with the output features added

Definition Classes

OpPipelineStageBase
def transformSchema(schema: StructType, logging: Boolean): StructType

Attributes

protected

Definition Classes

PipelineStage

Annotations

@DeveloperApi()
implicit val tti: scala.reflect.api.JavaUniverse.TypeTag[T]

type tag for input

type tag for input

Definition Classes

SequenceEstimator
implicit val ttiv: scala.reflect.api.JavaUniverse.TypeTag[T.Value]

type tag for input value

type tag for input value

Definition Classes

SequenceEstimator
implicit val tto: scala.reflect.api.JavaUniverse.TypeTag[OPVector]

type tag for input

type tag for input

Definition Classes

SequenceEstimator → HasOut
implicit val ttov: scala.reflect.api.JavaUniverse.TypeTag[Value]

type tag for output value

type tag for output value

Definition Classes

SequenceEstimator → HasOut
val uid: String

uid for instance

uid for instance

Definition Classes

SequenceEstimator → Identifiable
def vectorMetadataFromInputFeatures: OpVectorMetadata

Compute the output vector metadata only from the input features.

Compute the output vector metadata only from the input features. Vectorizers use this to derive the full vector, including pivot columns or indicator features.

returns

Vector metadata from input features

Attributes

protected

Definition Classes

VectorizerDefaults
def vectorMetadataWithNullIndicators: OpVectorMetadata

Attributes

protected

Definition Classes

VectorizerDefaults
def vectorOutputName: String

Get the name of the output vector

Get the name of the output vector

returns

Output vector name as a string

Attributes

protected

Definition Classes

VectorizerDefaults
final def wait(): Unit

Definition Classes

AnyRef

Annotations

@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes

AnyRef

Annotations

@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes

AnyRef

Annotations

@throws( ... )
final val withConstant: BooleanParam
final def write: MLWriter

Definition Classes

OpPipelineStageBase → MLWritable

Related Doc: package feature

class RealVectorizer[T <: Real] extends SequenceEstimator[T, OPVector] with VectorizerDefaults with TrackNullsParam

Instance Constructors

new RealVectorizer(uid: String = UID[RealVectorizer[_]], operationName: String = "vecReal")(implicit tti: scala.reflect.api.JavaUniverse.TypeTag[T], ttiv: scala.reflect.api.JavaUniverse.TypeTag[Option[Double]])

Type Members

final type InputFeatures = Array[FeatureLike[T]]

final type OutputFeatures = FeatureLike[OPVector]

Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def $[T](param: Param[T]): T

final def ==(arg0: Any): Boolean

final def asInstanceOf[T0]: T0

implicit def booleanToDouble(v: Boolean): Double

final def checkInputLength(features: Array[_]): Boolean

final def checkSerializable: Try[Unit]

final def clear(param: Param[_]): RealVectorizer.this.type

def clone(): AnyRef

final def copy(extra: ParamMap): RealVectorizer.this.type

def copyValues[T <: Params](to: T, extra: ParamMap): T

final def defaultCopy[T <: Params](extra: ParamMap): T

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def explainParam(param: Param[_]): String

def explainParams(): String

final def extractParamMap(): ParamMap

final def extractParamMap(extra: ParamMap): ParamMap

final val fillValue: DoubleParam

def finalize(): Unit

def fit(dataset: Dataset[_]): SequenceModel[T, OPVector]

def fit(dataset: Dataset[_], paramMaps: Array[ParamMap]): Seq[SequenceModel[T, OPVector]]

def fit(dataset: Dataset[_], paramMap: ParamMap): SequenceModel[T, OPVector]

def fit(dataset: Dataset[_], firstParamPair: ParamPair[_], otherParamPairs: ParamPair[_]*): SequenceModel[T, OPVector]

def fitFn(dataset: Dataset[Seq[Option[Double]]]): SequenceModel[T, OPVector]

final def get[T](param: Param[T]): Option[T]

final def getClass(): Class[_]

final def getDefault[T](param: Param[T]): Option[T]

final def getInputFeature[T <: FeatureType](i: Int): Option[FeatureLike[T]]

final def getInputFeatures(): Array[OPFeature]

final def getInputSchema(): StructType

final def getMetadata(): Metadata

final def getOrDefault[T](param: Param[T]): T

def getOutput(): FeatureLike[OPVector]

final def getOutputFeatureName: String

def getParam(paramName: String): Param[Any]

final def getTransientFeature(i: Int): Option[TransientFeature]

final def getTransientFeatures(): Array[TransientFeature]

final def hasDefault[T](param: Param[T]): Boolean

def hasParam(paramName: String): Boolean

def hashCode(): Int

final def inN: Array[TransientFeature]

def initializeLogIfNecessary(isInterpreter: Boolean, silent: Boolean): Boolean

def initializeLogIfNecessary(isInterpreter: Boolean): Unit

final def inputAsArray(in: InputFeatures): Array[OPFeature]

final def isDefined(param: Param[_]): Boolean

final def isInstanceOf[T0]: Boolean

final def isSet(param: Param[_]): Boolean

def isTraceEnabled(): Boolean

def log: Logger

def logDebug(msg: ⇒ String, throwable: Throwable): Unit

def logDebug(msg: ⇒ String): Unit

def logError(msg: ⇒ String, throwable: Throwable): Unit

def logError(msg: ⇒ String): Unit

def logInfo(msg: ⇒ String, throwable: Throwable): Unit

def logInfo(msg: ⇒ String): Unit

def logName: String

def logTrace(msg: ⇒ String, throwable: Throwable): Unit

def logTrace(msg: ⇒ String): Unit

def logWarning(msg: ⇒ String, throwable: Throwable): Unit

def logWarning(msg: ⇒ String): Unit

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

def onGetMetadata(): Unit

def onSetInput(): Unit

val operationName: String

final def outputAsArray(out: OutputFeatures): Array[OPFeature]

def outputFeatureUid: String

def outputIsResponse: Boolean