FilterMap

Instance Constructors

new FilterMap(uid: String = UID[FilterMap[_]])(implicit tti: scala.reflect.api.JavaUniverse.TypeTag[I])

uid

uid for instance

tti

type tag for input

Type Members

final type InputFeatures = FeatureLike[I]

Input Features type

Input Features type

Definition Classes

OpPipelineStage1 → OpPipelineStage → InputParams
type KeyValue = (String) ⇒ Any

Feature name (key) -> value lookup, e.g Row, Map etc.

Feature name (key) -> value lookup, e.g Row, Map etc.

Definition Classes

OpTransformer
final type OutputFeatures = FeatureLike[I]

Definition Classes

OpPipelineStage → OpPipelineStageBase

Value Members

final def !=(arg0: Any): Boolean

Definition Classes

AnyRef → Any
final def ##(): Int

Definition Classes

AnyRef → Any
final def $[T](param: Param[T]): T

Attributes

protected

Definition Classes

Params
final def ==(arg0: Any): Boolean

Definition Classes

AnyRef → Any
final val allowListKeys: StringArrayParam

Definition Classes

MapPivotParams
final def asInstanceOf[T0]: T0

Definition Classes

Any
final val blockListKeys: StringArrayParam

Definition Classes

MapPivotParams
final def checkInputLength(features: Array[_]): Boolean

Checks the input length

Checks the input length

features

input features

returns

true is input size as expected, false otherwise

Definition Classes

OpPipelineStage1 → InputParams
final def checkSerializable: Try[Unit]

Check if the stage is serializable

Check if the stage is serializable

returns

Failure if not serializable

Definition Classes

OpTransformer1 → OpPipelineStageBase
final val cleanKeys: BooleanParam

Definition Classes

MapPivotParams
def cleanMap[V](m: Map[String, V], shouldCleanKey: Boolean, shouldCleanValue: Boolean): Map[String, V]

Definition Classes

CleanTextMapFun
final val cleanText: BooleanParam

Definition Classes

TextParams
def cleanTextFn(s: String, shouldClean: Boolean): String

Definition Classes

CleanTextFun
final def clear(param: Param[_]): FilterMap.this.type

Definition Classes

Params
def clone(): AnyRef

Attributes

protected[java.lang]

Definition Classes

AnyRef

Annotations

@throws( ... )
final def copy(extra: ParamMap): FilterMap.this.type

This method is used to make a copy of the instance with new parameters in several methods in spark internals Default will find the constructor and make a copy for any class (AS LONG AS ALL CONSTRUCTOR PARAMS ARE VALS, this is why type tags are written as implicit vals in base classes).

This method is used to make a copy of the instance with new parameters in several methods in spark internals Default will find the constructor and make a copy for any class (AS LONG AS ALL CONSTRUCTOR PARAMS ARE VALS, this is why type tags are written as implicit vals in base classes).

Note: that the convention in spark is to have the uid be a constructor argument, so that copies will share a uid with the original (developers should follow this convention).

extra

new parameters want to add to instance

returns

a new instance with the same uid

Definition Classes

OpPipelineStageBase → Params
def copyValues[T <: Params](to: T, extra: ParamMap): T

Attributes

protected

Definition Classes

Params
final def defaultCopy[T <: Params](extra: ParamMap): T

Attributes

protected

Definition Classes

Params
final def eq(arg0: AnyRef): Boolean

Definition Classes

AnyRef
def equals(arg0: Any): Boolean

Definition Classes

AnyRef → Any
def explainParam(param: Param[_]): String

Definition Classes

Params
def explainParams(): String

Definition Classes

Params
final def extractParamMap(): ParamMap

Definition Classes

Params
final def extractParamMap(extra: ParamMap): ParamMap

Definition Classes

Params
def filterKeys[V](m: Map[String, V], shouldCleanKey: Boolean, shouldCleanValue: Boolean): Map[String, V]

Attributes

protected

Definition Classes

MapPivotParams
def finalize(): Unit

Attributes

protected[java.lang]

Definition Classes

AnyRef

Annotations

@throws( classOf[java.lang.Throwable] )
final def get[T](param: Param[T]): Option[T]

Definition Classes

Params
final def getClass(): Class[_]

Definition Classes

AnyRef → Any
final def getDefault[T](param: Param[T]): Option[T]

Definition Classes

Params
final def getInputFeature[T <: FeatureType](i: Int): Option[FeatureLike[T]]

Gets an input feature Note: this method IS NOT safe to use outside the driver, please use getTransientFeature method instead

Gets an input feature Note: this method IS NOT safe to use outside the driver, please use getTransientFeature method instead

returns

array of features

Definition Classes

InputParams

Exceptions thrown

NoSuchElementException if the features are not set

RuntimeException in case one of the features is null
final def getInputFeatures(): Array[OPFeature]

Gets the input features Note: this method IS NOT safe to use outside the driver, please use getTransientFeatures method instead

Gets the input features Note: this method IS NOT safe to use outside the driver, please use getTransientFeatures method instead

returns

array of features

Definition Classes

InputParams

Exceptions thrown

NoSuchElementException if the features are not set

RuntimeException in case one of the features is null
final def getInputSchema(): StructType

Definition Classes

OpPipelineStageParams
final def getMetadata(): Metadata

Definition Classes

OpPipelineStageParams
final def getOrDefault[T](param: Param[T]): T

Definition Classes

Params
def getOutput(): FeatureLike[I]

Output features that will be created by this stage

Output features that will be created by this stage

returns

feature of type OutputFeatures

Definition Classes

HasOut → OpPipelineStageBase
final def getOutputFeatureName: String

Name of output feature (i.e.

Name of output feature (i.e. column created by this stage)

Definition Classes

OpPipelineStage
def getParam(paramName: String): Param[Any]

Definition Classes

Params
final def getTransientFeature(i: Int): Option[TransientFeature]

Gets an input feature at index i

Gets an input feature at index i

i

input index

returns

maybe an input feature

Definition Classes

InputParams
final def getTransientFeatures(): Array[TransientFeature]

Gets the input Features

Gets the input Features

returns

input features

Definition Classes

InputParams
final def hasDefault[T](param: Param[T]): Boolean

Definition Classes

Params
def hasParam(paramName: String): Boolean

Definition Classes

Params
def hashCode(): Int

Definition Classes

AnyRef → Any
final def in1: TransientFeature

Attributes

protected

Definition Classes

HasIn1
def initializeLogIfNecessary(isInterpreter: Boolean, silent: Boolean): Boolean

Attributes

protected

Definition Classes

Logging
def initializeLogIfNecessary(isInterpreter: Boolean): Unit

Attributes

protected

Definition Classes

Logging
final def inputAsArray(in: InputFeatures): Array[OPFeature]

Function to convert InputFeatures to an Array of FeatureLike

Function to convert InputFeatures to an Array of FeatureLike

returns

an Array of FeatureLike

Definition Classes

OpPipelineStage1 → InputParams
final def isDefined(param: Param[_]): Boolean

Definition Classes

Params
final def isInstanceOf[T0]: Boolean

Definition Classes

Any
final def isSet(param: Param[_]): Boolean

Definition Classes

Params
def isTraceEnabled(): Boolean

Attributes

protected

Definition Classes

Logging
def log: Logger

Attributes

protected

Definition Classes

Logging
def logDebug(msg: ⇒ String, throwable: Throwable): Unit

Attributes

protected

Definition Classes

Logging
def logDebug(msg: ⇒ String): Unit

Attributes

protected

Definition Classes

Logging
def logError(msg: ⇒ String, throwable: Throwable): Unit

Attributes

protected

Definition Classes

Logging
def logError(msg: ⇒ String): Unit

Attributes

protected

Definition Classes

Logging
def logInfo(msg: ⇒ String, throwable: Throwable): Unit

Attributes

protected

Definition Classes

Logging
def logInfo(msg: ⇒ String): Unit

Attributes

protected

Definition Classes

Logging
def logName: String

Attributes

protected

Definition Classes

Logging
def logTrace(msg: ⇒ String, throwable: Throwable): Unit

Attributes

protected

Definition Classes

Logging
def logTrace(msg: ⇒ String): Unit

Attributes

protected

Definition Classes

Logging
def logWarning(msg: ⇒ String, throwable: Throwable): Unit

Attributes

protected

Definition Classes

Logging
def logWarning(msg: ⇒ String): Unit

Attributes

protected

Definition Classes

Logging
final def ne(arg0: AnyRef): Boolean

Definition Classes

AnyRef
final def notify(): Unit

Definition Classes

AnyRef
final def notifyAll(): Unit

Definition Classes

AnyRef
def onGetMetadata(): Unit

Function to be called on getMetadata

Function to be called on getMetadata

Attributes

protected

Definition Classes

OpPipelineStageParams
def onSetInput(): Unit

Function to be called on setInput

Function to be called on setInput

Attributes

protected

Definition Classes

InputParams
val operationName: String

unique name of the operation this stage performs

unique name of the operation this stage performs

Definition Classes

UnaryTransformer → OpPipelineStageBase
final def outputAsArray(out: OutputFeatures): Array[OPFeature]

Function to convert OutputFeatures to an Array of FeatureLike

Function to convert OutputFeatures to an Array of FeatureLike

returns

an Array of FeatureLike

Definition Classes

OpPipelineStage → OpPipelineStageBase
def outputFeatureUid: String

Attributes

protected[com.salesforce.op]

Definition Classes

OpPipelineStage1 → OpPipelineStage
def outputIsResponse: Boolean

Should output feature be a response? Yes, if any of the input features are.

Should output feature be a response? Yes, if any of the input features are.

returns

true if the the output feature should be a response

Definition Classes

OpPipelineStage
lazy val params: Array[Param[_]]

Definition Classes

Params
def save(path: String): Unit

Definition Classes

MLWritable

Annotations

@Since( "1.6.0" ) @throws( ... )
final def set(paramPair: ParamPair[_]): FilterMap.this.type

Attributes

protected

Definition Classes

Params
final def set(param: String, value: Any): FilterMap.this.type

Attributes

protected

Definition Classes

Params
final def set[T](param: Param[T], value: T): FilterMap.this.type

Definition Classes

Params
final def setAllowListKeys(keys: Array[String]): FilterMap.this.type

Definition Classes

MapPivotParams
final def setBlockListKeys(keys: Array[String]): FilterMap.this.type

Definition Classes

MapPivotParams
def setCleanKeys(clean: Boolean): FilterMap.this.type

Definition Classes

MapPivotParams
def setCleanText(clean: Boolean): FilterMap.this.type

Definition Classes

TextParams
final def setDefault(paramPairs: ParamPair[_]*): FilterMap.this.type

Attributes

protected

Definition Classes

Params
final def setDefault[T](param: Param[T], value: T): FilterMap.this.type

Attributes

protected

Definition Classes

Params
final def setInput(features: InputFeatures): FilterMap.this.type

Input features that will be used by the stage

Input features that will be used by the stage

returns

feature of type InputFeatures

Definition Classes

OpPipelineStageBase
final def setInputFeatures[S <: OPFeature](features: Array[S]): FilterMap.this.type

Sets input features

Sets input features

S

feature like type

features

array of input features

returns

this stage

Attributes

protected

Definition Classes

InputParams
final def setMetadata(m: Metadata): FilterMap.this.type

Definition Classes

OpPipelineStageParams
def setOutputFeatureName(name: String): FilterMap.this.type

Definition Classes

OpPipelineStage
final def stageName: String

Stage unique name consisting of the stage operation name and uid

Stage unique name consisting of the stage operation name and uid

returns

stage name

Definition Classes

OpPipelineStageBase
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes

AnyRef
def toString(): String

Definition Classes

Identifiable → AnyRef → Any
def transform(dataset: Dataset[_]): DataFrame

Spark operation on dataset to produce new output feature column using defined function

Spark operation on dataset to produce new output feature column using defined function

dataset

input data for this stage

returns

a new dataset containing a column for the transformed feature

Definition Classes

OpTransformer1 → Transformer
def transform(dataset: Dataset[_], paramMap: ParamMap): DataFrame

Definition Classes

Transformer

Annotations

@Since( "2.0.0" )
def transform(dataset: Dataset[_], firstParamPair: ParamPair[_], otherParamPairs: ParamPair[_]*): DataFrame

Definition Classes

Transformer

Annotations

@Since( "2.0.0" ) @varargs()
def transformFn: (I) ⇒ I

Function used to convert input to output

Function used to convert input to output

Definition Classes

FilterMap → OpTransformer1
lazy val transformKeyValue: (KeyValue) ⇒ Any

Creates a transform function to transform any key/value to a value

Creates a transform function to transform any key/value to a value

returns

a transform function to transform any key/value to a value

Definition Classes

OpTransformer1 → OpTransformer
def transformMap: (Map[String, Any]) ⇒ Any

Creates a transform function to transform Map to a value

Creates a transform function to transform Map to a value

returns

a transform function to transform Map to a value

Definition Classes

OpTransformer
def transformRow: (Row) ⇒ Any

Creates a transform function to transform Row to a value

Creates a transform function to transform Row to a value

returns

a transform function to transform Row to a value

Definition Classes

OpTransformer
final def transformSchema(schema: StructType): StructType

This function translates the input and output features into spark schema checks and changes that will occur on the underlying data frame

This function translates the input and output features into spark schema checks and changes that will occur on the underlying data frame

schema

schema of the input data frame

returns

a new schema with the output features added

Definition Classes

OpPipelineStageBase
def transformSchema(schema: StructType, logging: Boolean): StructType

Attributes

protected

Definition Classes

PipelineStage

Annotations

@DeveloperApi()
implicit val tti: scala.reflect.api.JavaUniverse.TypeTag[I]

type tag for input

type tag for input

Definition Classes

UnaryTransformer → OpTransformer1
implicit val tto: scala.reflect.api.JavaUniverse.TypeTag[I]

type tag for output

type tag for output

Definition Classes

UnaryTransformer → HasOut
implicit val ttov: scala.reflect.api.JavaUniverse.TypeTag[I.Value]

type tag for output value

type tag for output value

Definition Classes

UnaryTransformer → HasOut
val uid: String

uid for instance

uid for instance

Definition Classes

UnaryTransformer → Identifiable
final def wait(): Unit

Definition Classes

AnyRef

Annotations

@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes

AnyRef

Annotations

@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes

AnyRef

Annotations

@throws( ... )
final def write: MLWriter

Definition Classes

OpPipelineStageBase → MLWritable

Related Doc: package feature

class FilterMap[I <: OPMap[_]] extends UnaryTransformer[I, I] with MapPivotParams with TextParams with CleanTextMapFun

Instance Constructors

new FilterMap(uid: String = UID[FilterMap[_]])(implicit tti: scala.reflect.api.JavaUniverse.TypeTag[I])

Type Members

final type InputFeatures = FeatureLike[I]

type KeyValue = (String) ⇒ Any

final type OutputFeatures = FeatureLike[I]

Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def $[T](param: Param[T]): T

final def ==(arg0: Any): Boolean

final val allowListKeys: StringArrayParam

final def asInstanceOf[T0]: T0

final val blockListKeys: StringArrayParam

final def checkInputLength(features: Array[_]): Boolean

final def checkSerializable: Try[Unit]

final val cleanKeys: BooleanParam

def cleanMap[V](m: Map[String, V], shouldCleanKey: Boolean, shouldCleanValue: Boolean): Map[String, V]

final val cleanText: BooleanParam

def cleanTextFn(s: String, shouldClean: Boolean): String

final def clear(param: Param[_]): FilterMap.this.type

def clone(): AnyRef

final def copy(extra: ParamMap): FilterMap.this.type

def copyValues[T <: Params](to: T, extra: ParamMap): T

final def defaultCopy[T <: Params](extra: ParamMap): T

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def explainParam(param: Param[_]): String

def explainParams(): String

final def extractParamMap(): ParamMap

final def extractParamMap(extra: ParamMap): ParamMap

def filterKeys[V](m: Map[String, V], shouldCleanKey: Boolean, shouldCleanValue: Boolean): Map[String, V]

def finalize(): Unit

final def get[T](param: Param[T]): Option[T]

final def getClass(): Class[_]

final def getDefault[T](param: Param[T]): Option[T]

final def getInputFeature[T <: FeatureType](i: Int): Option[FeatureLike[T]]

final def getInputFeatures(): Array[OPFeature]

final def getInputSchema(): StructType

final def getMetadata(): Metadata

final def getOrDefault[T](param: Param[T]): T

def getOutput(): FeatureLike[I]

final def getOutputFeatureName: String

def getParam(paramName: String): Param[Any]

final def getTransientFeature(i: Int): Option[TransientFeature]

final def getTransientFeatures(): Array[TransientFeature]

final def hasDefault[T](param: Param[T]): Boolean

def hasParam(paramName: String): Boolean

def hashCode(): Int

final def in1: TransientFeature

def initializeLogIfNecessary(isInterpreter: Boolean, silent: Boolean): Boolean

def initializeLogIfNecessary(isInterpreter: Boolean): Unit

final def inputAsArray(in: InputFeatures): Array[OPFeature]

final def isDefined(param: Param[_]): Boolean

final def isInstanceOf[T0]: Boolean

final def isSet(param: Param[_]): Boolean

def isTraceEnabled(): Boolean

def log: Logger

def logDebug(msg: ⇒ String, throwable: Throwable): Unit

def logDebug(msg: ⇒ String): Unit

def logError(msg: ⇒ String, throwable: Throwable): Unit

def logError(msg: ⇒ String): Unit

def logInfo(msg: ⇒ String, throwable: Throwable): Unit

def logInfo(msg: ⇒ String): Unit

def logName: String

def logTrace(msg: ⇒ String, throwable: Throwable): Unit

def logTrace(msg: ⇒ String): Unit

def logWarning(msg: ⇒ String, throwable: Throwable): Unit

def logWarning(msg: ⇒ String): Unit

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

def onGetMetadata(): Unit

def onSetInput(): Unit

val operationName: String

final def outputAsArray(out: OutputFeatures): Array[OPFeature]

def outputFeatureUid: String