global {
  spark-options {
    "spark.sql.shuffle.partitions" = 2
  }
}

dataObjects {
  ext-departures {
    type = WebserviceFileDataObject
    url = "https://opensky-network.org/api/flights/departure?airport=LSZB&begin=1696854853&end=1697027653"
    readTimeoutMs=200000
  }
  stg-departures {
    type = JsonFileDataObject
    path = "~{id}"
  }

  ext-airports {
    type = WebserviceFileDataObject
    # uses redirects to the URL below
    url = "https://ourairports.com/data/airports.csv"
    # url = "https://davidmegginson.github.io/ourairports-data/airports.csv"
    followRedirects = true
    readTimeoutMs=200000
  }
  stg-airports {
    type = CsvFileDataObject
    path = "~{id}"
  }
  int-airports {
    type = CsvFileDataObject
    path = "~{id}"
  }
}

actions {
  download-departures {
    type = FileTransferAction
    inputId = ext-departures
    outputId = stg-departures
    metadata {
      feed = download
    }
  }

  download-airports {
    type = FileTransferAction
    inputId = ext-airports
    outputId = stg-airports
    metadata {
      feed = download
    }
  }

  select-airport-cols {
    type = CopyAction
    inputId = stg-airports
    outputId = int-airports
    transformers = [{
      type = SQLDfTransformer
      code = "select ident, name, latitude_deg, longitude_deg from stg_airports"
    }]
    metadata {
      feed = compute
    }
  }
}