HumanNameDetector

Instance Constructors

new HumanNameDetector(uid: String = UID[HumanNameDetector[T]], operationName: String = "humanNameDetect")(implicit tti: scala.reflect.api.JavaUniverse.TypeTag[T], ttiv: scala.reflect.api.JavaUniverse.TypeTag[Option[String]])

uid

uid for instance

operationName

unique name of the operation this stage performs

tti

type tag for input

ttiv

type tag for input value

Type Members

final type InputFeatures = FeatureLike[T]

Input Features type

Input Features type

Definition Classes

OpPipelineStage1 → OpPipelineStage → InputParams
final type OutputFeatures = FeatureLike[NameStats]

Definition Classes

OpPipelineStage → OpPipelineStageBase

Value Members

final def !=(arg0: Any): Boolean

Definition Classes

AnyRef → Any
final def ##(): Int

Definition Classes

AnyRef → Any
final def $[T](param: Param[T]): T

Attributes

protected

Definition Classes

Params
final def ==(arg0: Any): Boolean

Definition Classes

AnyRef → Any
final def asInstanceOf[T0]: T0

Definition Classes

Any
final def checkInputLength(features: Array[_]): Boolean

Checks the input length

Checks the input length

features

input features

returns

true is input size as expected, false otherwise

Definition Classes

OpPipelineStage1 → InputParams
final def checkSerializable: Try[Unit]

Check if the stage is serializable

Check if the stage is serializable

returns

Failure if not serializable

Definition Classes

UnaryEstimator → OpPipelineStageBase
final def clear(param: Param[_]): HumanNameDetector.this.type

Definition Classes

Params
def clone(): AnyRef

Attributes

protected[java.lang]

Definition Classes

AnyRef

Annotations

@throws( ... )
final def copy(extra: ParamMap): HumanNameDetector.this.type

This method is used to make a copy of the instance with new parameters in several methods in spark internals Default will find the constructor and make a copy for any class (AS LONG AS ALL CONSTRUCTOR PARAMS ARE VALS, this is why type tags are written as implicit vals in base classes).

This method is used to make a copy of the instance with new parameters in several methods in spark internals Default will find the constructor and make a copy for any class (AS LONG AS ALL CONSTRUCTOR PARAMS ARE VALS, this is why type tags are written as implicit vals in base classes).

Note: that the convention in spark is to have the uid be a constructor argument, so that copies will share a uid with the original (developers should follow this convention).

extra

new parameters want to add to instance

returns

a new instance with the same uid

Definition Classes

OpPipelineStageBase → Params
def copyValues[T <: Params](to: T, extra: ParamMap): T

Attributes

protected

Definition Classes

Params
final def defaultCopy[T <: Params](extra: ParamMap): T

Attributes

protected

Definition Classes

Params
final def eq(arg0: AnyRef): Boolean

Definition Classes

AnyRef
def equals(arg0: Any): Boolean

Definition Classes

AnyRef → Any
def explainParam(param: Param[_]): String

Definition Classes

Params
def explainParams(): String

Definition Classes

Params
final def extractParamMap(): ParamMap

Definition Classes

Params
final def extractParamMap(extra: ParamMap): ParamMap

Definition Classes

Params
def finalize(): Unit

Attributes

protected[java.lang]

Definition Classes

AnyRef

Annotations

@throws( classOf[java.lang.Throwable] )
def fit(dataset: Dataset[_]): UnaryModel[T, NameStats]

Spark operation on dataset to produce Dataset for constructor fit function and then turn output function into a Model

Spark operation on dataset to produce Dataset for constructor fit function and then turn output function into a Model

dataset

input data for this stage

returns

a fitted model that will perform the transformation specified by the function defined in constructor fit

Definition Classes

UnaryEstimator → Estimator
def fit(dataset: Dataset[_], paramMaps: Array[ParamMap]): Seq[UnaryModel[T, NameStats]]

Definition Classes

Estimator

Annotations

@Since( "2.0.0" )
def fit(dataset: Dataset[_], paramMap: ParamMap): UnaryModel[T, NameStats]

Definition Classes

Estimator

Annotations

@Since( "2.0.0" )
def fit(dataset: Dataset[_], firstParamPair: ParamPair[_], otherParamPairs: ParamPair[_]*): UnaryModel[T, NameStats]

Definition Classes

Estimator

Annotations

@Since( "2.0.0" ) @varargs()
def fitFn(dataset: Dataset[Option[String]]): HumanNameDetectorModel[T]

Function that fits the unary model

Function that fits the unary model

Definition Classes

HumanNameDetector → UnaryEstimator
final def get[T](param: Param[T]): Option[T]

Definition Classes

Params
final def getClass(): Class[_]

Definition Classes

AnyRef → Any
final def getDefault[T](param: Param[T]): Option[T]

Definition Classes

Params
final def getInputFeature[T <: FeatureType](i: Int): Option[FeatureLike[T]]

Gets an input feature Note: this method IS NOT safe to use outside the driver, please use getTransientFeature method instead

Gets an input feature Note: this method IS NOT safe to use outside the driver, please use getTransientFeature method instead

returns

array of features

Definition Classes

InputParams

Exceptions thrown

NoSuchElementException if the features are not set

RuntimeException in case one of the features is null
final def getInputFeatures(): Array[OPFeature]

Gets the input features Note: this method IS NOT safe to use outside the driver, please use getTransientFeatures method instead

Gets the input features Note: this method IS NOT safe to use outside the driver, please use getTransientFeatures method instead

returns

array of features

Definition Classes

InputParams

Exceptions thrown

NoSuchElementException if the features are not set

RuntimeException in case one of the features is null
final def getInputSchema(): StructType

Definition Classes

OpPipelineStageParams
final def getMetadata(): Metadata

Definition Classes

OpPipelineStageParams
final def getOrDefault[T](param: Param[T]): T

Definition Classes

Params
def getOutput(): FeatureLike[NameStats]

Output features that will be created by this stage

Output features that will be created by this stage

returns

feature of type OutputFeatures

Definition Classes

HasOut → OpPipelineStageBase
final def getOutputFeatureName: String

Name of output feature (i.e.

Name of output feature (i.e. column created by this stage)

Definition Classes

OpPipelineStage
def getParam(paramName: String): Param[Any]

Definition Classes

Params
def getSensitiveFeatureMode: SensitiveFeatureMode

Definition Classes

NameDetectParams
final def getTransientFeature(i: Int): Option[TransientFeature]

Gets an input feature at index i

Gets an input feature at index i

i

input index

returns

maybe an input feature

Definition Classes

InputParams
final def getTransientFeatures(): Array[TransientFeature]

Gets the input Features

Gets the input Features

returns

input features

Definition Classes

InputParams
val guardMaxNumberOfTokens: IntParam

Definition Classes

NameDetectParams
val guardMinCountForStdDevCheck: IntParam

Definition Classes

NameDetectParams
val guardMinCountForUniqueCheck: IntParam

Definition Classes

NameDetectParams
val guardMinStdDev: DoubleParam

Definition Classes

NameDetectParams
val guardMinTextLength: IntParam

Definition Classes

NameDetectParams
val guardMinUniqueCheck: IntParam

Definition Classes

NameDetectParams
val guardPctMaxNumberOfTokens: DoubleParam

Definition Classes

NameDetectParams
val guardPctMinTextLength: DoubleParam

Definition Classes

NameDetectParams
final def hasDefault[T](param: Param[T]): Boolean

Definition Classes

Params
def hasParam(paramName: String): Boolean

Definition Classes

Params
def hashCode(): Int

Definition Classes

AnyRef → Any
val iConvert: FeatureTypeSparkConverter[T]

Definition Classes

UnaryEstimator
implicit val iEncoder: Encoder[T.Value]

Definition Classes

UnaryEstimator
val ignoreNulls: BooleanParam

Definition Classes

NameDetectParams
final def in1: TransientFeature

Attributes

protected

Definition Classes

HasIn1
def initializeLogIfNecessary(isInterpreter: Boolean, silent: Boolean): Boolean

Attributes

protected

Definition Classes

Logging
def initializeLogIfNecessary(isInterpreter: Boolean): Unit

Attributes

protected

Definition Classes

Logging
final def inputAsArray(in: InputFeatures): Array[OPFeature]

Function to convert InputFeatures to an Array of FeatureLike

Function to convert InputFeatures to an Array of FeatureLike

returns

an Array of FeatureLike

Definition Classes

OpPipelineStage1 → InputParams
final def isDefined(param: Param[_]): Boolean

Definition Classes

Params
final def isInstanceOf[T0]: Boolean

Definition Classes

Any
final def isSet(param: Param[_]): Boolean

Definition Classes

Params
def isTraceEnabled(): Boolean

Attributes

protected

Definition Classes

Logging
def log: Logger

Attributes

protected

Definition Classes

Logging
def logDebug(msg: ⇒ String, throwable: Throwable): Unit

Attributes

protected

Definition Classes

Logging
def logDebug(msg: ⇒ String): Unit

Attributes

protected

Definition Classes

Logging
def logError(msg: ⇒ String, throwable: Throwable): Unit

Attributes

protected

Definition Classes

Logging
def logError(msg: ⇒ String): Unit

Attributes

protected

Definition Classes

Logging
def logInfo(msg: ⇒ String, throwable: Throwable): Unit

Attributes

protected

Definition Classes

Logging
def logInfo(msg: ⇒ String): Unit

Attributes

protected

Definition Classes

Logging
def logName: String

Attributes

protected

Definition Classes

Logging
def logTrace(msg: ⇒ String, throwable: Throwable): Unit

Attributes

protected

Definition Classes

Logging
def logTrace(msg: ⇒ String): Unit

Attributes

protected

Definition Classes

Logging
def logWarning(msg: ⇒ String, throwable: Throwable): Unit

Attributes

protected

Definition Classes

Logging
def logWarning(msg: ⇒ String): Unit

Attributes

protected

Definition Classes

Logging
val nameThreshold: DoubleParam

Definition Classes

NameDetectParams
final def ne(arg0: AnyRef): Boolean

Definition Classes

AnyRef
final def notify(): Unit

Definition Classes

AnyRef
final def notifyAll(): Unit

Definition Classes

AnyRef
def onGetMetadata(): Unit

Function to be called on getMetadata

Function to be called on getMetadata

Attributes

protected

Definition Classes

OpPipelineStageParams
def onSetInput(): Unit

Function to be called on setInput

Function to be called on setInput

Attributes

protected

Definition Classes

InputParams
val operationName: String

unique name of the operation this stage performs

unique name of the operation this stage performs

Definition Classes

UnaryEstimator → OpPipelineStageBase
final def outputAsArray(out: OutputFeatures): Array[OPFeature]

Function to convert OutputFeatures to an Array of FeatureLike

Function to convert OutputFeatures to an Array of FeatureLike

returns

an Array of FeatureLike

Definition Classes

OpPipelineStage → OpPipelineStageBase
def outputFeatureUid: String

Attributes

protected[com.salesforce.op]

Definition Classes

OpPipelineStage1 → OpPipelineStage
def outputIsResponse: Boolean

Should output feature be a response? Yes, if any of the input features are.

Should output feature be a response? Yes, if any of the input features are.

returns

true if the the output feature should be a response

Definition Classes

OpPipelineStage
lazy val params: Array[Param[_]]

Definition Classes

Params
def save(path: String): Unit

Definition Classes

MLWritable

Annotations

@Since( "1.6.0" ) @throws( ... )
val sensitiveFeatureMode: Param[String]

Definition Classes

NameDetectParams
final def set(paramPair: ParamPair[_]): HumanNameDetector.this.type

Attributes

protected

Definition Classes

Params
final def set(param: String, value: Any): HumanNameDetector.this.type

Attributes

protected

Definition Classes

Params
final def set[T](param: Param[T], value: T): HumanNameDetector.this.type

Definition Classes

Params
final def setDefault(paramPairs: ParamPair[_]*): HumanNameDetector.this.type

Attributes

protected

Definition Classes

Params
final def setDefault[T](param: Param[T], value: T): HumanNameDetector.this.type

Attributes

protected

Definition Classes

Params
def setGuardCheckValues(maxNumberOfTokens: Int = $(guardMaxNumberOfTokens), pctMaxNumberOfTokens: Double = $(guardPctMaxNumberOfTokens), minTextLength: Int = $(guardMinTextLength), pctMinTextLength: Double = $(guardPctMinTextLength), minCountForStdDevCheck: Int = $(guardMinCountForStdDevCheck), minStdDev: Double = $(guardMinStdDev), minCountForUniqueCheck: Int = $(guardMinCountForUniqueCheck), minUniqueCheck: Int = $(guardMinUniqueCheck)): HumanNameDetector.this.type

Definition Classes

NameDetectParams
def setIgnoreNulls(value: Boolean): HumanNameDetector.this.type

Definition Classes

NameDetectParams
final def setInput(features: InputFeatures): HumanNameDetector.this.type

Input features that will be used by the stage

Input features that will be used by the stage

returns

feature of type InputFeatures

Definition Classes

OpPipelineStageBase
final def setInputFeatures[S <: OPFeature](features: Array[S]): HumanNameDetector.this.type

Sets input features

Sets input features

S

feature like type

features

array of input features

returns

this stage

Attributes

protected

Definition Classes

InputParams
final def setMetadata(m: Metadata): HumanNameDetector.this.type

Definition Classes

OpPipelineStageParams
def setOutputFeatureName(name: String): HumanNameDetector.this.type

Definition Classes

OpPipelineStage
def setSensitiveFeatureMode(v: SensitiveFeatureMode): HumanNameDetector.this.type

Definition Classes

NameDetectParams
def setThreshold(value: Double): HumanNameDetector.this.type

Definition Classes

NameDetectParams
def shouldRemoveSensitive: Boolean

Definition Classes

NameDetectParams
final def stageName: String

Stage unique name consisting of the stage operation name and uid

Stage unique name consisting of the stage operation name and uid

returns

stage name

Definition Classes

OpPipelineStageBase
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes

AnyRef
def toString(): String

Definition Classes

Identifiable → AnyRef → Any
final def transformSchema(schema: StructType): StructType

This function translates the input and output features into spark schema checks and changes that will occur on the underlying data frame

This function translates the input and output features into spark schema checks and changes that will occur on the underlying data frame

schema

schema of the input data frame

returns

a new schema with the output features added

Definition Classes

OpPipelineStageBase
def transformSchema(schema: StructType, logging: Boolean): StructType

Attributes

protected

Definition Classes

PipelineStage

Annotations

@DeveloperApi()
implicit val tti: scala.reflect.api.JavaUniverse.TypeTag[T]

type tag for input

type tag for input

Definition Classes

UnaryEstimator
implicit val ttiv: scala.reflect.api.JavaUniverse.TypeTag[Option[String]]

type tag for input value

type tag for input value

Definition Classes

HumanNameDetector → UnaryEstimator
implicit val tto: scala.reflect.api.JavaUniverse.TypeTag[NameStats]

type tag for output

type tag for output

Definition Classes

UnaryEstimator → HasOut
implicit val ttov: scala.reflect.api.JavaUniverse.TypeTag[features.types.NameStats.Value]

type tag for output value

type tag for output value

Definition Classes

UnaryEstimator → HasOut
val uid: String

uid for instance

uid for instance

Definition Classes

UnaryEstimator → Identifiable
final def wait(): Unit

Definition Classes

AnyRef

Annotations

@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes

AnyRef

Annotations

@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes

AnyRef

Annotations

@throws( ... )
final def write: MLWriter

Definition Classes

OpPipelineStageBase → MLWritable

Related Doc: package feature

class HumanNameDetector[T <: Text] extends UnaryEstimator[T, NameStats] with NameDetectFun[T]

Instance Constructors

new HumanNameDetector(uid: String = UID[HumanNameDetector[T]], operationName: String = "humanNameDetect")(implicit tti: scala.reflect.api.JavaUniverse.TypeTag[T], ttiv: scala.reflect.api.JavaUniverse.TypeTag[Option[String]])

Type Members

final type InputFeatures = FeatureLike[T]

final type OutputFeatures = FeatureLike[NameStats]

Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def $[T](param: Param[T]): T

final def ==(arg0: Any): Boolean

final def asInstanceOf[T0]: T0

final def checkInputLength(features: Array[_]): Boolean

final def checkSerializable: Try[Unit]

final def clear(param: Param[_]): HumanNameDetector.this.type

def clone(): AnyRef

final def copy(extra: ParamMap): HumanNameDetector.this.type

def copyValues[T <: Params](to: T, extra: ParamMap): T

final def defaultCopy[T <: Params](extra: ParamMap): T

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def explainParam(param: Param[_]): String

def explainParams(): String

final def extractParamMap(): ParamMap

final def extractParamMap(extra: ParamMap): ParamMap

def finalize(): Unit

def fit(dataset: Dataset[_]): UnaryModel[T, NameStats]

def fit(dataset: Dataset[_], paramMaps: Array[ParamMap]): Seq[UnaryModel[T, NameStats]]

def fit(dataset: Dataset[_], paramMap: ParamMap): UnaryModel[T, NameStats]

def fit(dataset: Dataset[_], firstParamPair: ParamPair[_], otherParamPairs: ParamPair[_]*): UnaryModel[T, NameStats]

def fitFn(dataset: Dataset[Option[String]]): HumanNameDetectorModel[T]

final def get[T](param: Param[T]): Option[T]

final def getClass(): Class[_]

final def getDefault[T](param: Param[T]): Option[T]

final def getInputFeature[T <: FeatureType](i: Int): Option[FeatureLike[T]]

final def getInputFeatures(): Array[OPFeature]

final def getInputSchema(): StructType

final def getMetadata(): Metadata

final def getOrDefault[T](param: Param[T]): T

def getOutput(): FeatureLike[NameStats]

final def getOutputFeatureName: String

def getParam(paramName: String): Param[Any]

def getSensitiveFeatureMode: SensitiveFeatureMode

final def getTransientFeature(i: Int): Option[TransientFeature]

final def getTransientFeatures(): Array[TransientFeature]

val guardMaxNumberOfTokens: IntParam

val guardMinCountForStdDevCheck: IntParam

val guardMinCountForUniqueCheck: IntParam

val guardMinStdDev: DoubleParam

val guardMinTextLength: IntParam

val guardMinUniqueCheck: IntParam

val guardPctMaxNumberOfTokens: DoubleParam

val guardPctMinTextLength: DoubleParam

final def hasDefault[T](param: Param[T]): Boolean

def hasParam(paramName: String): Boolean

def hashCode(): Int

val iConvert: FeatureTypeSparkConverter[T]

implicit val iEncoder: Encoder[T.Value]

val ignoreNulls: BooleanParam

final def in1: TransientFeature

def initializeLogIfNecessary(isInterpreter: Boolean, silent: Boolean): Boolean

def initializeLogIfNecessary(isInterpreter: Boolean): Unit

final def inputAsArray(in: InputFeatures): Array[OPFeature]

final def isDefined(param: Param[_]): Boolean

final def isInstanceOf[T0]: Boolean

final def isSet(param: Param[_]): Boolean

def isTraceEnabled(): Boolean

def log: Logger

def logDebug(msg: ⇒ String, throwable: Throwable): Unit

def logDebug(msg: ⇒ String): Unit

def logError(msg: ⇒ String, throwable: Throwable): Unit

def logError(msg: ⇒ String): Unit

def logInfo(msg: ⇒ String, throwable: Throwable): Unit

def logInfo(msg: ⇒ String): Unit

def logName: String

def logTrace(msg: ⇒ String, throwable: Throwable): Unit

def logTrace(msg: ⇒ String): Unit

def logWarning(msg: ⇒ String, throwable: Throwable): Unit