Package

io.smartdatalake.workflow

connection

Permalink

package connection

Visibility
  1. Public
  2. All

Type Members

  1. case class ConnectionMetadata(name: Option[String] = None, description: Option[String] = None, layer: Option[String] = None, subjectArea: Option[String] = None, tags: Seq[String] = Seq()) extends Product with Serializable

    Permalink

    Additional metadata for a Connection

    Additional metadata for a Connection

    name

    Readable name of the Connection

    description

    Description of the content of the Connection

    layer

    Name of the layer this Connection belongs to

    subjectArea

    Name of the subject area this Connection belongs to

    tags

    Optional custom tags for this object

  2. case class HadoopFileConnection(id: ConnectionId, pathPrefix: String, acl: Option[AclDef] = None, metadata: Option[ConnectionMetadata] = None) extends Connection with Product with Serializable

    Permalink

    Connection information for files on hadoop

    Connection information for files on hadoop

    id

    unique id of this connection

    pathPrefix

    schema, authority and base path for accessing files on hadoop

    acl

    permissions for files created with this connection

  3. case class HiveTableConnection(id: ConnectionId, db: String, pathPrefix: String, acl: Option[AclDef] = None, metadata: Option[ConnectionMetadata] = None) extends Connection with Product with Serializable

    Permalink

    Connection information for hive tables

    Connection information for hive tables

    id

    unique id of this connection

    db

    hive db

    pathPrefix

    schema, authority and base path for tables directory on hadoop

    acl

    permissions for files created with this connection

  4. case class JdbcTableConnection(id: ConnectionId, url: String, driver: String, authMode: Option[AuthMode] = None, db: Option[String] = None, metadata: Option[ConnectionMetadata] = None) extends Connection with SmartDataLakeLogger with Product with Serializable

    Permalink

    Connection information for jdbc tables.

    Connection information for jdbc tables. If authentication is needed, user and password must be provided.

    id

    unique id of this connection

    url

    jdbc connection url

    driver

    class name of jdbc driver

    authMode

    optional authentication information: for now BasicAuthMode is supported.

    db

    jdbc database

  5. case class KafkaConnection(id: ConnectionId, brokers: String, schemaRegistry: Option[String] = None, datasourceOptions: Map[String, String] = Map(), authMode: Option[AuthMode] = None, metadata: Option[ConnectionMetadata] = None) extends Connection with Product with Serializable

    Permalink

    Connection information for kafka

    Connection information for kafka

    id

    unique id of this connection

    brokers

    comma separated list of kafka bootstrap server incl. port, e.g. "host1:9092,host2:9092:

    schemaRegistry

    url of schema registry service, e.g. "https://host2"

    datasourceOptions

    Options for the Kafka stream reader (see https://spark.apache.org/docs/latest/structured-streaming-kafka-integration.html)

  6. case class SftpFileRefConnection(id: ConnectionId, host: String, port: Int = 22, authMode: AuthMode, ignoreHostKeyVerification: Boolean = false, maxParallelConnections: Int = 1, metadata: Option[ConnectionMetadata] = None) extends Connection with Product with Serializable

    Permalink

    SFTP Connection information

    SFTP Connection information

    id

    unique id of this connection

    host

    sftp host

    port

    port of sftp service, default is 22

    authMode

    authentication information: for now BasicAuthMode and PublicKeyAuthMode are supported.

    ignoreHostKeyVerification

    do not validate host key if true, default is false

    maxParallelConnections

    number of parallel sftp connections created by an instance of this connection

  7. case class SplunkConnection(id: ConnectionId, host: String, port: Int, authMode: AuthMode, metadata: Option[ConnectionMetadata] = None) extends Connection with SplunkConnectionService with Product with Serializable

    Permalink

    Connection information for splunk queries

    Connection information for splunk queries

    id

    unique id of this connection

  8. trait SplunkConnectionService extends AnyRef

    Permalink

Value Members

  1. object HadoopFileConnection extends FromConfigFactory[Connection] with Serializable

    Permalink
  2. object HiveTableConnection extends FromConfigFactory[Connection] with Serializable

    Permalink
  3. object JdbcTableConnection extends FromConfigFactory[Connection] with Serializable

    Permalink
  4. object KafkaConnection extends FromConfigFactory[Connection] with Serializable

    Permalink
  5. object SftpFileRefConnection extends FromConfigFactory[Connection] with Serializable

    Permalink
  6. object SplunkConnection extends FromConfigFactory[Connection] with Serializable

    Permalink

Ungrouped