global {
  spark-options {
    "spark.sql.shuffle.partitions" = 2
  }
}

dataObjects {
  NOPEext-departures {
    type = WebserviceFileDataObject
    url = "https://opensky-network.org/api/flights/departure?airport=LSZB&begin=1696854853&end=1697027653"
    readTimeoutMs=200000
  }
  stg-departures {
    type = JsonFileDataObject
    path = "~{id}"
  }

  ext-airports {
    type = WebserviceFileDataObject
    # uses redirects to the URL below
    url = "https://ourairports.com/data/airports.csv"
    # url = "https://davidmegginson.github.io/ourairports-data/airports.csv"
    followRedirects = true
    readTimeoutMs=200000
  }
  stg-airports {
    type = UnicornFileDataObject
    path = "~{id}"
  }
}

actions {
  download-departures {
    type = FileTransferAction
    inputId = ext-departures
    outputId = stg-departures
    metadata {
      feed = download
    }
  }

  download-airports {
    type = FileTransferAction
    inputId = ext-airports
    outputId = stg-airports
    metadata {
      feed = download
    }
  }
}