FileStreamingAvroReader

Instance Constructors

new FileStreamingAvroReader(key: (T) ⇒ String, filter: (Path) ⇒ Boolean, newFilesOnly: Boolean)(implicit ctt: ClassTag[T], wtt: scala.reflect.api.JavaUniverse.WeakTypeTag[T])

key

function for extracting key from avro record

filter

Function to filter paths to process

newFilesOnly

Should process only new files and ignore existing files in the directory

Value Members

final def !=(arg0: Any): Boolean

Definition Classes

AnyRef → Any
final def ##(): Int

Definition Classes

AnyRef → Any
final def ==(arg0: Any): Boolean

Definition Classes

AnyRef → Any
final def asInstanceOf[T0]: T0

Definition Classes

Any
def clone(): AnyRef

Attributes

protected[java.lang]

Definition Classes

AnyRef

Annotations

@throws( ... )
implicit val ctt: ClassTag[T]

Reader class tag

Reader class tag

Definition Classes

FileStreamingAvroReader → StreamingReader
final def eq(arg0: AnyRef): Boolean

Definition Classes

AnyRef
def equals(arg0: Any): Boolean

Definition Classes

AnyRef → Any
val filter: (Path) ⇒ Boolean

Function to filter paths to process
def finalize(): Unit

Attributes

protected[java.lang]

Definition Classes

AnyRef

Annotations

@throws( classOf[java.lang.Throwable] )
final def fullTypeName: String

Full reader input type name

Full reader input type name

returns

full input type name

Definition Classes

ReaderType
final def getClass(): Class[_]

Definition Classes

AnyRef → Any
final def getReaderParams(opParams: OpParams): Option[ReaderParams]

Default method for extracting this reader's parameters from readerParams in OpParams

Default method for extracting this reader's parameters from readerParams in OpParams

opParams

contains map of reader type to ReaderParams instances

returns

ReaderParams instance if it exists

Definition Classes

ReaderType
def hashCode(): Int

Definition Classes

AnyRef → Any
final def isInstanceOf[T0]: Boolean

Definition Classes

Any
val key: (T) ⇒ String

function for extracting key from avro record

function for extracting key from avro record

Definition Classes

FileStreamingAvroReader → ReaderKey
final def ne(arg0: AnyRef): Boolean

Definition Classes

AnyRef
val newFilesOnly: Boolean

Should process only new files and ignore existing files in the directory
final def notify(): Unit

Definition Classes

AnyRef
final def notifyAll(): Unit

Definition Classes

AnyRef
def stream(params: OpParams)(implicit streaming: StreamingContext): DStream[T]

Function which creates an stream of T to read from

Function which creates an stream of T to read from

params

parameters used to carry out specialized logic in reader (passed in from workflow)

streaming

spark streaming context

returns

stream of T to read from

Definition Classes

FileStreamingAvroReader → StreamingReader
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes

AnyRef
def toString(): String

Definition Classes

AnyRef → Any
final def typeName: String

Short reader input type name

Short reader input type name

returns

short reader input type name

Definition Classes

ReaderType
final def wait(): Unit

Definition Classes

AnyRef

Annotations

@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes

AnyRef

Annotations

@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes

AnyRef

Annotations

@throws( ... )
implicit val wtt: scala.reflect.api.JavaUniverse.WeakTypeTag[T]

Reader type tag

Reader type tag

Definition Classes

FileStreamingAvroReader → ReaderType

Related Doc: package readers

class FileStreamingAvroReader[T <: GenericRecord] extends StreamingReader[T]

Instance Constructors

new FileStreamingAvroReader(key: (T) ⇒ String, filter: (Path) ⇒ Boolean, newFilesOnly: Boolean)(implicit ctt: ClassTag[T], wtt: scala.reflect.api.JavaUniverse.WeakTypeTag[T])

Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

final def asInstanceOf[T0]: T0

def clone(): AnyRef

implicit val ctt: ClassTag[T]

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

val filter: (Path) ⇒ Boolean

def finalize(): Unit

final def fullTypeName: String

final def getClass(): Class[_]

final def getReaderParams(opParams: OpParams): Option[ReaderParams]

def hashCode(): Int

final def isInstanceOf[T0]: Boolean

val key: (T) ⇒ String

final def ne(arg0: AnyRef): Boolean

val newFilesOnly: Boolean

final def notify(): Unit

final def notifyAll(): Unit

def stream(params: OpParams)(implicit streaming: StreamingContext): DStream[T]

final def synchronized[T0](arg0: ⇒ T0): T0

def toString(): String

final def typeName: String

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

implicit val wtt: scala.reflect.api.JavaUniverse.WeakTypeTag[T]

Inherited from StreamingReader[T]

Inherited from ReaderKey[T]

Inherited from ReaderType[T]

Inherited from Serializable

Inherited from Serializable

Inherited from AnyRef

Inherited from Any

Ungrouped