SanityChecker

final def !=(arg0: Any): Boolean

final def ##(): Int

final def $[T](param: Param[T]): T

final def ==(arg0: Any): Boolean

final def asInstanceOf[T0]: T0

final val categoricalLabel: BooleanParam

final def checkInputLength(features: Array[_]): Boolean

Checks the input length

final val checkSample: DoubleParam

final def checkSerializable: Try[Unit]

Check if the stage is serializable

final def clear(param: Param[_]): SanityChecker.this.type

def clone(): AnyRef

val convertI1: FeatureTypeSparkConverter[RealNN]

val convertI2: FeatureTypeSparkConverter[OPVector]

final def copy(extra: ParamMap): SanityChecker.this.type

This method is used to make a copy of the instance with new parameters in several methods in spark internals Default will find the constructor and make a copy for any class (AS LONG AS ALL CONSTRUCTOR PARAMS ARE VALS, this is why type tags are written as implicit vals in base classes).

def copyValues[T <: Params](to: T, extra: ParamMap): T

final val correlationExclusion: Param[String]

final val correlationType: Param[String]

final def defaultCopy[T <: Params](extra: ParamMap): T

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def explainParam(param: Param[_]): String

def explainParams(): String

final def extractParamMap(): ParamMap

final def extractParamMap(extra: ParamMap): ParamMap

final val featureFeatureCorrLevel: Param[String]

def finalize(): Unit

def fit(dataset: Dataset[_]): BinaryModel[RealNN, OPVector, OPVector]

Spark operation on dataset to produce RDD for constructor fit function and then turn output function into a Model

def fit(dataset: Dataset[_], paramMaps: Array[ParamMap]): Seq[BinaryModel[RealNN, OPVector, OPVector]]

def fit(dataset: Dataset[_], paramMap: ParamMap): BinaryModel[RealNN, OPVector, OPVector]

def fit(dataset: Dataset[_], firstParamPair: ParamPair[_], otherParamPairs: ParamPair[_]*): BinaryModel[RealNN, OPVector, OPVector]

def fitFn(data: Dataset[(Option[Double], Vector)]): BinaryModel[RealNN, OPVector, OPVector]

The SanityChecker's core 'estimator' function, computes statistics on the features and the list of features to be removed.

def fraction(totalSize: Long): Double

Estimate of a fraction of data that is being checked.

final def get[T](param: Param[T]): Option[T]

def getCategoricalLabel: Boolean

def getCheckSample: Double

final def getClass(): Class[_]

def getCorrelationExclusion: CorrelationExclusion

def getCorrelationType: CorrelationType

final def getDefault[T](param: Param[T]): Option[T]

def getFeatureFeatureCorrLevel: CorrelationLevel

final def getInputFeature[T <: FeatureType](i: Int): Option[FeatureLike[T]]

Gets an input feature Note: this method IS NOT safe to use outside the driver, please use getTransientFeature method instead

final def getInputFeatures(): Array[OPFeature]

Gets the input features Note: this method IS NOT safe to use outside the driver, please use getTransientFeatures method instead

final def getInputSchema(): StructType

def getMaxCorrelation: Double

def getMaxCramersV: Double

def getMaxFeatureCorrelation: Double

def getMaxRuleConfidence: Double

final def getMetadata(): Metadata

def getMinCorrelation: Double

def getMinRequiredRuleSupport: Double

def getMinVariance: Double

final def getOrDefault[T](param: Param[T]): T

def getOutput(): FeatureLike[OPVector]

Output features that will be created by this stage

final def getOutputFeatureName: String

Name of output feature (i.e.

def getParam(paramName: String): Param[Any]

def getProtectTextSharedHash: Boolean

def getRemoveBadFeatures: Boolean

def getRemoveFeatureGroup: Boolean

def getSampleLowerLimit: Int

def getSampleSeed: Long

def getSampleUpperLimit: Int

final def getTransientFeature(i: Int): Option[TransientFeature]

Gets an input feature at index i

final def getTransientFeatures(): Array[TransientFeature]

Gets the input Features

final def hasDefault[T](param: Param[T]): Boolean

def hasParam(paramName: String): Boolean

def hashCode(): Int

implicit val i1Encoder: Encoder[features.types.RealNN.Value]

implicit val i2Encoder: Encoder[Value]

final def in1: TransientFeature

final def in2: TransientFeature

def initializeLogIfNecessary(isInterpreter: Boolean, silent: Boolean): Boolean

def initializeLogIfNecessary(isInterpreter: Boolean): Unit

final def inputAsArray(in: InputFeatures): Array[OPFeature]

Function to convert InputFeatures to an Array of FeatureLike

final def isDefined(param: Param[_]): Boolean

final def isInstanceOf[T0]: Boolean

final def isSet(param: Param[_]): Boolean

def isTraceEnabled(): Boolean

def log: Logger

def logDebug(msg: ⇒ String, throwable: Throwable): Unit

def logDebug(msg: ⇒ String): Unit

def logError(msg: ⇒ String, throwable: Throwable): Unit

def logError(msg: ⇒ String): Unit

def logInfo(msg: ⇒ String, throwable: Throwable): Unit

def logInfo(msg: ⇒ String): Unit

final val logLevel: Param[String]

def logName: String

def logTrace(msg: ⇒ String, throwable: Throwable): Unit

def logTrace(msg: ⇒ String): Unit

def logWarning(msg: ⇒ String, throwable: Throwable): Unit

def logWarning(msg: ⇒ String): Unit

final val maxCorrelation: DoubleParam

final val maxCramersV: DoubleParam

final val maxFeatureCorrelation: DoubleParam

final val maxRuleConfidence: DoubleParam

final val minCorrelation: DoubleParam

final val minRequiredRuleSupport: DoubleParam

final val minVariance: DoubleParam

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

def onGetMetadata(): Unit

Function to be called on getMetadata

def onSetInput(): Unit

Function to be called on setInput

val operationName: String

unique name of the operation this stage performs

final def outputAsArray(out: OutputFeatures): Array[OPFeature]

Function to convert OutputFeatures to an Array of FeatureLike

def outputFeatureUid: String

def outputIsResponse: Boolean

Should output feature be a response? Yes, if any of the input features are.

lazy val params: Array[Param[_]]

final val protectTextSharedHash: BooleanParam

final val removeBadFeatures: BooleanParam

final val removeFeatureGroup: BooleanParam

final val sampleLowerLimit: IntParam

final val sampleSeed: LongParam

final val sampleUpperLimit: IntParam

def save(path: String): Unit

final def set(paramPair: ParamPair[_]): SanityChecker.this.type

final def set(param: String, value: Any): SanityChecker.this.type

final def set[T](param: Param[T], value: T): SanityChecker.this.type

def setCategoricalLabel(value: Boolean): SanityChecker.this.type

def setCheckSample(value: Double): SanityChecker.this.type

def setCorrelationExclusion(v: CorrelationExclusion): SanityChecker.this.type

def setCorrelationType(value: CorrelationType): SanityChecker.this.type

final def setDefault(paramPairs: ParamPair[_]*): SanityChecker.this.type

final def setDefault[T](param: Param[T], value: T): SanityChecker.this.type

def setFeatureFeatureCorrLevel(value: CorrelationLevel): SanityChecker.this.type

final def setInput(features: InputFeatures): SanityChecker.this.type

Input features that will be used by the stage

final def setInputFeatures[S <: OPFeature](features: Array[S]): SanityChecker.this.type

Sets input features

def setMaxCorrelation(value: Double): SanityChecker.this.type

def setMaxCramersV(value: Double): SanityChecker.this.type

def setMaxFeatureCorrelation(value: Double): SanityChecker.this.type

def setMaxRuleConfidence(value: Double): SanityChecker.this.type

final def setMetadata(m: Metadata): SanityChecker.this.type

def setMinCorrelation(value: Double): SanityChecker.this.type

def setMinRequiredRuleSupport(value: Double): SanityChecker.this.type

def setMinVariance(value: Double): SanityChecker.this.type

def setOutputFeatureName(name: String): SanityChecker.this.type

def setProtectTextSharedHash(value: Boolean): SanityChecker.this.type

def setRemoveBadFeatures(value: Boolean): SanityChecker.this.type

def setRemoveFeatureGroup(value: Boolean): SanityChecker.this.type

def setSampleLowerLimit(value: Int): SanityChecker.this.type

def setSampleSeed(value: Long): SanityChecker.this.type

def setSampleUpperLimit(value: Int): SanityChecker.this.type

final def stageName: String

Stage unique name consisting of the stage operation name and uid

final def synchronized[T0](arg0: ⇒ T0): T0

def toString(): String

final def transformSchema(schema: StructType): StructType

This function translates the input and output features into spark schema checks and changes that will occur on the underlying data frame

def transformSchema(schema: StructType, logging: Boolean): StructType

implicit val tti1: scala.reflect.api.JavaUniverse.TypeTag[RealNN]

type tag for first input

implicit val tti2: scala.reflect.api.JavaUniverse.TypeTag[OPVector]

type tag for second input

implicit val ttiv1: scala.reflect.api.JavaUniverse.TypeTag[features.types.RealNN.Value]

type tag for first input value

implicit val ttiv2: scala.reflect.api.JavaUniverse.TypeTag[Value]

type tag for second input value

implicit val tto: scala.reflect.api.JavaUniverse.TypeTag[OPVector]

type tag for output

implicit val ttov: scala.reflect.api.JavaUniverse.TypeTag[Value]

type tag for output value

implicit val tupleEncoder: Encoder[(features.types.RealNN.Value, Value)]

val uid: String

uid for instance

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

final def write: MLWriter

def getFeatureLabelCorrOnly: Boolean

def setFeatureLabelCorrOnly(value: Boolean): SanityChecker.this.type