Skip to content

Instantly share code, notes, and snippets.

@soxofaan

soxofaan/issue127.ipynb

Created February 24, 2022 20:16
Show Gist options
  • Save soxofaan/b69c54acc51a36f47b9284da2ca04e8e to your computer and use it in GitHub Desktop.
Save soxofaan/b69c54acc51a36f47b9284da2ca04e8e to your computer and use it in GitHub Desktop.
Download ZIP
Display the source blob
Display the rendered blob
Raw
issue127.ipynb
{
"cells": [
{
"cell_type": "markdown",
"id": "7a25c4dd-9888-4e90-9450-d53b6ff8ee2d",
"metadata": {},
"source": [
"- https://discuss.eodc.eu/t/caching-openeo-results/197/10\n",
"- https://github.com/Open-EO/openeo-geopyspark-driver/issues/127"
]
},
{
"cell_type": "code",
"execution_count": 1,
"id": "f47b2eed-cb68-4a5c-8ef6-512cd0d939a8",
"metadata": {},
"outputs": [],
"source": [
"import openeo\n",
"from pathlib import Path"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "3d5880a9-64ae-4bab-8872-e7e645becb14",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Authenticated using refresh token.\n"
]
},
{
"data": {
"text/plain": [
"<Connection to 'https://openeo-dev.vito.be/openeo/1.0/' with BearerAuth>"
]
},
"execution_count": 2,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"con = openeo.connect(\n",
" \"openeo-dev.vito.be\"\n",
")\n",
"con.authenticate_oidc()"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "4f32d7c3-c43c-4d5c-a42b-536b5c29ac67",
"metadata": {},
"outputs": [],
"source": [
"download_dir = Path(\"issue127-data\")\n",
"download_dir.mkdir(exist_ok=True)"
]
},
{
"cell_type": "markdown",
"id": "d4d198d1-808b-4bb9-9622-1c9ccf7e27f3",
"metadata": {},
"source": [
"## Initial batch job"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "a18abcdf-7a89-4481-8bc5-e64f1224a0b9",
"metadata": {},
"outputs": [],
"source": [
"cube1 = con.load_collection(\n",
" \"TERRASCOPE_S2_TOC_V2\",\n",
" temporal_extent=[\"2021-09-01\", \"2021-09-30\"],\n",
" spatial_extent={\"west\": 4, \"south\": 51, \"east\": 4.1, \"north\": 51.1}\n",
")\n",
"cube1 = cube1.save_result(\"GTiff\")"
]
},
{
"cell_type": "markdown",
"id": "eee91f8e-5979-4f8a-a334-2ce29783341a",
"metadata": {},
"source": [
"Reuse an existing job or create a new one"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "22b055b8-e124-4b85-97fd-61ce4e8cc4eb",
"metadata": {},
"outputs": [],
"source": [
"job1 = con.job(\"337fd792-bb5f-42d9-b055-26d669125fe3\")\n",
"\n",
"# job1 = cube1.send_job()\n",
"# job1.start_and_wait()"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "ec1c2159-6ab0-44f2-8eeb-819e96c097eb",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"issue127-data/1-openEO_2021-09-02Z.tif\n",
"issue127-data/1-openEO_2021-09-07Z.tif\n",
"issue127-data/1-openEO_2021-09-12Z.tif\n",
"issue127-data/1-openEO_2021-09-17Z.tif\n",
"issue127-data/1-openEO_2021-09-27Z.tif\n"
]
}
],
"source": [
"for asset in job1.get_results().get_assets():\n",
" print(asset.download(download_dir / f\"1-{asset.name}\"))"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "4aa61744-7241-45e4-a8f2-43f449c54966",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"'337fd792-bb5f-42d9-b055-26d669125fe3'"
]
},
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"first_job_id = job1.job_id\n",
"first_job_id"
]
},
{
"cell_type": "markdown",
"id": "04c27b8f-e729-4d7a-ae72-e019f37c5614",
"metadata": {},
"source": [
"## Load job results with `load_result` and download synchronously as-is"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "25861594-1374-4786-866a-9cfb429dd35d",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/home/lippenss/src/openeo/openeo-python-client/openeo/metadata.py:240: UserWarning: No cube:dimensions metadata\n",
" complain(\"No cube:dimensions metadata\")\n"
]
}
],
"source": [
"cube2 = con.datacube_from_process(\"load_result\", id=first_job_id)"
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "084011ce-e039-451f-ae58-cb96f9d34444",
"metadata": {},
"outputs": [],
"source": [
"cube2.download(download_dir / \"2-sync.tiff\")"
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "a4ad85f9-996d-4b4d-88b3-f03d83ede6fe",
"metadata": {},
"outputs": [],
"source": [
"import rasterio"
]
},
{
"cell_type": "code",
"execution_count": 11,
"id": "ff7a52a7-e429-4c44-8393-3b6ce47cdf8e",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"bands: 1, width: 1166, height: 1854\n",
"data pixels: 0\n",
"nodata pixels: 2161764\n"
]
}
],
"source": [
"with rasterio.open(download_dir / \"2-sync.tiff\") as ds:\n",
" print(f\"bands: {ds.count}, width: {ds.width}, height: {ds.height}\")\n",
" data = ds.read()\n",
" print(\"data pixels:\", (data != ds.nodata).sum())\n",
" print(\"nodata pixels:\", (data == ds.nodata).sum())"
]
},
{
"cell_type": "code",
"execution_count": 12,
"id": "ba5d4984-7fda-4def-8194-a4ce6c28ca29",
"metadata": {},
"outputs": [
{
"ename": "OpenEoApiError",
"evalue": "[500] unknown: / by zero",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mOpenEoApiError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m<ipython-input-12-e641f0a7c9a8>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mcube2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdownload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdownload_dir\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0;34m\"2-sync.nc\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
"\u001b[0;32m~/src/openeo/openeo-python-client/openeo/rest/datacube.py\u001b[0m in \u001b[0;36mdownload\u001b[0;34m(self, outputfile, format, options)\u001b[0m\n\u001b[1;32m 1523\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1524\u001b[0m \u001b[0mcube\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave_result\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptions\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1525\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_connection\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdownload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcube\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mflat_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutputfile\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1526\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1527\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mvalidate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mList\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mdict\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/src/openeo/openeo-python-client/openeo/rest/connection.py\u001b[0m in \u001b[0;36mdownload\u001b[0;34m(self, graph, outputfile, timeout)\u001b[0m\n\u001b[1;32m 1014\u001b[0m \"\"\"\n\u001b[1;32m 1015\u001b[0m \u001b[0mrequest\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_build_request_with_process_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprocess_graph\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mgraph\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1016\u001b[0;31m \u001b[0mresponse\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpost\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"/result\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mjson\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mrequest\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexpected_status\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m200\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstream\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1017\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1018\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0moutputfile\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/src/openeo/openeo-python-client/openeo/rest/connection.py\u001b[0m in \u001b[0;36mpost\u001b[0;34m(self, path, json, **kwargs)\u001b[0m\n\u001b[1;32m 170\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;32mreturn\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mResponse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 171\u001b[0m \"\"\"\n\u001b[0;32m--> 172\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrequest\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"post\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpath\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mjson\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mjson\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mallow_redirects\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 173\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 174\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mdelete\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mResponse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/src/openeo/openeo-python-client/openeo/rest/connection.py\u001b[0m in \u001b[0;36mrequest\u001b[0;34m(self, method, path, headers, auth, check_error, expected_status, **kwargs)\u001b[0m\n\u001b[1;32m 118\u001b[0m \u001b[0mexpected_status\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mensure_list\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mexpected_status\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mexpected_status\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 119\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mcheck_error\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mstatus\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0;36m400\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mstatus\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mexpected_status\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 120\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_raise_api_error\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresp\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 121\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mexpected_status\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mstatus\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mexpected_status\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 122\u001b[0m raise OpenEoRestError(\"Got status code {s!r} for `{m} {p}` (expected {e!r})\".format(\n",
"\u001b[0;32m~/src/openeo/openeo-python-client/openeo/rest/connection.py\u001b[0m in \u001b[0;36m_raise_api_error\u001b[0;34m(self, response)\u001b[0m\n\u001b[1;32m 149\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 150\u001b[0m \u001b[0mexception\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mOpenEoApiError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhttp_status_code\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mstatus_code\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmessage\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 151\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mexception\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 152\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 153\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstream\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mauth\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mAuthBase\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mResponse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mOpenEoApiError\u001b[0m: [500] unknown: / by zero"
]
}
],
"source": [
"cube2.download(download_dir / \"2-sync.nc\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "413b795a-c6a8-4ab7-b190-9d5a98d566be",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "5c490e80-d6d5-4272-8ee8-524a41b2dd60",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "c19b1920-8e77-46f8-a6ba-22eb71f24bd0",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "efb676e8-3da7-4672-8cbc-d1a14f2492af",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "openeo-py39",
"language": "python",
"name": "openeo-py39"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.7"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment