03_Upload_bucket_to_EE_asset.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "# 03. Upload polygon masks from Cloud bucket to GEE Assets\n",
    "\n",
    "This notebook uploads the polygon mask created for each Sentinel-2 and Landsat tile from GCP (Google Cloud Platform) cloud bucket.\n",
    "\n",
    "This notebook reads in the output of the notebook [01_Create_polygon_mask.ipynb](https://github.com/ShiruiH/time-series-OFS/blob/main/01_Create_polygon_mask.ipynb), which is located in `/outputs/Sentinel2_tiles_mask` and `/outputs/Landsat_tiles_mask`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "executionInfo": {
     "elapsed": 1343,
     "status": "ok",
     "timestamp": 1723730822198,
     "user": {
      "displayName": "Shirui Hao",
      "userId": "16127122271389846236"
     },
     "user_tz": -600
    },
    "id": "LFEnIdxeEO4v",
    "tags": []
   },
   "outputs": [],
   "source": [
    "# import libraries\n",
    "import os\n",
    "import glob\n",
    "from pathlib import Path\n",
    "import shutil\n",
    "import json\n",
    "from google.cloud import storage\n",
    "import ee"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "executionInfo": {
     "elapsed": 13867,
     "status": "ok",
     "timestamp": 1723730943967,
     "user": {
      "displayName": "Shirui Hao",
      "userId": "16127122271389846236"
     },
     "user_tz": -600
    },
    "id": "FkfczT_pN66h",
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Authenticate and Initialize ee\n",
    "# ee.Authenticate()\n",
    "ee.Initialize(project='nsw-dpe-gee-tst')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 17727,
     "status": "ok",
     "timestamp": 1723730963893,
     "user": {
      "displayName": "Shirui Hao",
      "userId": "16127122271389846236"
     },
     "user_tz": -600
    },
    "id": "QnlWkPLarwPB",
    "outputId": "0c47fe85-1787-4bb2-dea6-9c9e28adaed1",
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Your browser has been opened to visit:\n",
      "\n",
      "    https://accounts.google.com/o/oauth2/auth?response_type=code&client_id=32555940559.apps.googleusercontent.com&redirect_uri=http%3A%2F%2Flocalhost%3A8085%2F&scope=openid+https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fuserinfo.email+https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fcloud-platform+https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fappengine.admin+https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fsqlservice.login+https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fcompute+https%3A%2F%2Fwww.googleapis.com%2Fauth%2Faccounts.reauth&state=hoQ3jUHiRkI3ik0AmXf6UkmdT5GgoE&access_type=offline&code_challenge=0OhbUGM0kdM17QnIfj2gP-O01_BuffJ_tFK3MpoGEs0&code_challenge_method=S256\n",
      "\n",
      "\n",
      "You are now logged in as [kilian.vos@dpie.nsw.gov.au].\n",
      "Your current project is [nsw-dpe-gee-tst].  You can change this setting by running:\n",
      "  $ gcloud config set project PROJECT_ID\n",
      "Updated property [core/project].\n"
     ]
    }
   ],
   "source": [
    "! gcloud auth login\n",
    "! gcloud config set project nsw-dpe-gee-tst"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Auxiliary functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "executionInfo": {
     "elapsed": 348,
     "status": "ok",
     "timestamp": 1723730966838,
     "user": {
      "displayName": "Shirui Hao",
      "userId": "16127122271389846236"
     },
     "user_tz": -600
    },
    "id": "wJIJAbYTEjWO",
    "tags": []
   },
   "outputs": [],
   "source": [
    "# list files in the bucket\n",
    "def list_blobs(bucket_name, folder_path):\n",
    "    storage_client = storage.Client()\n",
    "    blobs = storage_client.list_blobs(bucket_name, prefix=folder_path)\n",
    "    blobs = [blob for blob in blobs if blob.name != folder_path and not blob.name.endswith('/')]\n",
    "    return blobs\n",
    "# write json file\n",
    "def write_json(new_data, filename):\n",
    "    with open(filename, 'r+') as file:\n",
    "        # Load existing data into a dict\n",
    "        file_data = json.load(file)\n",
    "        # Write new data\n",
    "        file_data['name'] = new_data\n",
    "        # Sets file's current position at offset\n",
    "        file.seek(0)\n",
    "        # convert back to json\n",
    "        json.dump(file_data, file, indent=4)\n",
    "        file.truncate()\n",
    "        file.close()\n",
    "# write manifest for multiple files\n",
    "def write_json_manifest_multi_tiles(sources, tile, filename):\n",
    "    with open(filename, 'r+') as file:\n",
    "        # Load existing data into a dict\n",
    "        file_data = json.load(file)\n",
    "        # Write new data\n",
    "        file_data['tilesets'][0]['sources'] = [sources]\n",
    "        file_data['properties'] = tile\n",
    "        # Sets file's current position at offset\n",
    "        file.seek(0)\n",
    "        # convert back to json\n",
    "        json.dump(file_data, file, indent=4)\n",
    "        file.truncate()\n",
    "        file.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Locate cloud bucket (make sure the path to the cloud bucket is correct)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 322,
     "status": "ok",
     "timestamp": 1723730969799,
     "user": {
      "displayName": "Shirui Hao",
      "userId": "16127122271389846236"
     },
     "user_tz": -600
    },
    "id": "yOtnTmCPGMJv",
    "outputId": "3290d1f7-52f6-4557-beeb-64996198be78",
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Polygon masks from satellite: Landsat\n"
     ]
    }
   ],
   "source": [
    "# Buckets args set up\n",
    "BUCKET_NAME = 'label-tiles'\n",
    "FOLDER_PATH = 'Landsat_tiles_mask/'\n",
    "\n",
    "sat_name = FOLDER_PATH.split('_')[0]\n",
    "print('Polygon masks from satellite:', sat_name)\n",
    "\n",
    "# GEE Assets args set up\n",
    "PROJ_NAME = 'nsw-dpe-gee-tst'\n",
    "ASSETS_SUB_FOLDER = 'OFS' # Can be nested folder, e.g., OFS/sub_folder\n",
    "IMAGE_COLLECTION_NAME = f'base_{sat_name}_tiles'\n",
    "\n",
    "image_collection_id = f'projects/{PROJ_NAME}/assets/{ASSETS_SUB_FOLDER}/{IMAGE_COLLECTION_NAME}'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No existing asset found at projects/nsw-dpe-gee-tst/assets/OFS/exp_baseOFS_Landsat_tiles, proceeding to create a new one\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'type': 'IMAGE_COLLECTION',\n",
       " 'name': 'projects/nsw-dpe-gee-tst/assets/OFS/exp_baseOFS_Landsat_tiles',\n",
       " 'id': 'projects/nsw-dpe-gee-tst/assets/OFS/exp_baseOFS_Landsat_tiles',\n",
       " 'updateTime': '2024-08-28T03:58:36.944791Z'}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create the ImageCollection\n",
    "# Check if the asset already exists\n",
    "# Make sure all images in the ImageCollection have been deleted using 04_Reset_EE_asset_collection.ipynb prior to re-create the ImageCollection\n",
    "try:\n",
    "    ee.data.getAsset(image_collection_id)\n",
    "    # If the asset exists, delete it\n",
    "    ee.data.deleteAsset(image_collection_id)\n",
    "    print(f'Deleted existing asset at {image_collection_id}, proceeding to create a new one')\n",
    "except:\n",
    "    print(f'No existing asset found at {image_collection_id}, proceeding to create a new one')\n",
    "\n",
    "ee.data.createAsset({'type': 'ImageCollection'}, image_collection_id)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load template json manifest (to be updated in the next cell)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "executionInfo": {
     "elapsed": 2147,
     "status": "ok",
     "timestamp": 1723730973689,
     "user": {
      "displayName": "Shirui Hao",
      "userId": "16127122271389846236"
     },
     "user_tz": -600
    },
    "id": "nwpN1ZhbGTdN",
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3 polygon masks found in cloud bucket\n"
     ]
    }
   ],
   "source": [
    "fp_outputs = os.path.join(os.getcwd(),'outputs')\n",
    "fp_json = os.path.join(fp_outputs,'json_manifest')\n",
    "if not os.path.exists(fp_json):\n",
    "    os.makedirs(fp_json)\n",
    "# JSON file template\n",
    "src_file = glob.glob(f'{fp_outputs}/Template_tileset.json')[0]\n",
    "blobs_lst = list_blobs(BUCKET_NAME, FOLDER_PATH)\n",
    "upload_lst = [blob for blob in blobs_lst if '.tif' in blob.name]\n",
    "print('%d polygon masks found in cloud bucket'%len(upload_lst))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Ingest file into GEE Assets using a json manifest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "executionInfo": {
     "elapsed": 9812,
     "status": "ok",
     "timestamp": 1723730985127,
     "user": {
      "displayName": "Shirui Hao",
      "userId": "16127122271389846236"
     },
     "user_tz": -600
    },
    "id": "JqUU0PuXGW8L",
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3 polygon masks uploaded to GEE Assets under projects/nsw-dpe-gee-tst/assets/OFS/exp_baseOFS_Landsat_tiles\n"
     ]
    }
   ],
   "source": [
    "# loop through blobs (elements in cloud bucket)\n",
    "for blob in upload_lst:\n",
    "    blob_name = blob.name\n",
    "    file_name = blob_name.split('/')[1].split('.')[0]\n",
    "    if sat_name == 'Sentinel2':\n",
    "        tile = file_name.split('_')[-1][1:] # 'T' needs to be removed\n",
    "        # add tile property to the image object\n",
    "        tile_arg = {\n",
    "            \"Tile\": f\"{tile}\"\n",
    "            }\n",
    "        dst_file = f'{fp_json}/{sat_name}_EE_upload_{tile}.json'\n",
    "    else:\n",
    "        path = int(file_name.split('_')[1][0:3])\n",
    "        row = int(file_name.split('_')[1][3:6])\n",
    "        # add tile property to the image object\n",
    "        tile_arg = {\n",
    "            \"Path\": path,\n",
    "            \"Row\": row\n",
    "            }\n",
    "        dst_file = f'{fp_json}/{sat_name}_EE_upload_{path}{row}.json'\n",
    "    shutil.copy(src_file, dst_file)\n",
    "    assets_name = f'{image_collection_id}/{file_name}'\n",
    "    source_arg = {\n",
    "        \"uris\": [\n",
    "            f\"gs://{BUCKET_NAME}/{blob_name}\"\n",
    "            ]\n",
    "        }\n",
    "    # Change the base name in the json file\n",
    "    write_json(assets_name, dst_file)\n",
    "    write_json_manifest_multi_tiles(source_arg, tile_arg, filename=dst_file)\n",
    "    # Submit EE Tasks to ingest tiles\n",
    "    reqID = ee.data.newTaskId()[0]\n",
    "    with open(f'{dst_file}') as f:\n",
    "        params = json.load(f)\n",
    "    ee.data.startIngestion(request_id=reqID, params=params)\n",
    "print('%d polygon masks uploaded to GEE Assets under %s'%(len(upload_lst),image_collection_id))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "authorship_tag": "ABX9TyOZZ9X8+D/guAKhQu6ppshl",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}