unzip_7z_files.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting py7zr\n",
      "  Downloading py7zr-0.11.1-py3-none-any.whl (63 kB)\n",
      "\u001b[K     |████████████████████████████████| 63 kB 2.3 MB/s  eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: texttable in /home/ec2-user/anaconda3/envs/amazonei_tensorflow_p36/lib/python3.6/site-packages (from py7zr) (1.6.3)\n",
      "Requirement already satisfied: importlib-metadata in /home/ec2-user/anaconda3/envs/amazonei_tensorflow_p36/lib/python3.6/site-packages (from py7zr) (3.1.0)\n",
      "Requirement already satisfied: zipp>=0.5 in /home/ec2-user/anaconda3/envs/amazonei_tensorflow_p36/lib/python3.6/site-packages (from importlib-metadata->py7zr) (3.4.0)\n",
      "Collecting pycryptodome\n",
      "  Downloading pycryptodome-3.9.9-cp36-cp36m-manylinux1_x86_64.whl (13.7 MB)\n",
      "\u001b[K     |████████████████████████████████| 13.7 MB 8.0 MB/s eta 0:00:01\n",
      "\u001b[?25hInstalling collected packages: pycryptodome, py7zr\n",
      "Successfully installed py7zr-0.11.1 pycryptodome-3.9.9\n",
      "\u001b[33mWARNING: You are using pip version 20.3; however, version 20.3.3 is available.\n",
      "You should consider upgrading via the '/home/ec2-user/anaconda3/envs/amazonei_tensorflow_p36/bin/python -m pip install --upgrade pip' command.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!pip install py7zr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tvpr-dataset\n"
     ]
    }
   ],
   "source": [
    "import boto3\n",
    "s3_bucket = \"tvpr-dataset\"\n",
    "\n",
    "s3 = boto3.resource('s3')\n",
    "bucket = s3.Bucket(s3_bucket)\n",
    "print(bucket.name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[s3.ObjectSummary(bucket_name='tvpr-dataset', key='oni-7z/g001.oni.7z'), s3.ObjectSummary(bucket_name='tvpr-dataset', key='oni-7z/g002.oni.7z'), s3.ObjectSummary(bucket_name='tvpr-dataset', key='oni-7z/g003.oni.7z'), s3.ObjectSummary(bucket_name='tvpr-dataset', key='oni-7z/g004.oni.7z'), s3.ObjectSummary(bucket_name='tvpr-dataset', key='oni-7z/g005.oni.7z'), s3.ObjectSummary(bucket_name='tvpr-dataset', key='oni-7z/g006.oni.7z'), s3.ObjectSummary(bucket_name='tvpr-dataset', key='oni-7z/g007.oni.7z'), s3.ObjectSummary(bucket_name='tvpr-dataset', key='oni-7z/g008.oni.7z'), s3.ObjectSummary(bucket_name='tvpr-dataset', key='oni-7z/g009.oni.7z'), s3.ObjectSummary(bucket_name='tvpr-dataset', key='oni-7z/g010.oni.7z'), s3.ObjectSummary(bucket_name='tvpr-dataset', key='oni-7z/g011.oni.7z'), s3.ObjectSummary(bucket_name='tvpr-dataset', key='oni-7z/g012.oni.7z'), s3.ObjectSummary(bucket_name='tvpr-dataset', key='oni-7z/g013.oni.7z'), s3.ObjectSummary(bucket_name='tvpr-dataset', key='oni-7z/g014.oni.7z'), s3.ObjectSummary(bucket_name='tvpr-dataset', key='oni-7z/g015.oni.7z'), s3.ObjectSummary(bucket_name='tvpr-dataset', key='oni-7z/g016.oni.7z'), s3.ObjectSummary(bucket_name='tvpr-dataset', key='oni-7z/g017.oni.7z'), s3.ObjectSummary(bucket_name='tvpr-dataset', key='oni-7z/g018.oni.7z'), s3.ObjectSummary(bucket_name='tvpr-dataset', key='oni-7z/g019.oni.7z'), s3.ObjectSummary(bucket_name='tvpr-dataset', key='oni-7z/g020.oni.7z'), s3.ObjectSummary(bucket_name='tvpr-dataset', key='oni-7z/g021.oni.7z'), s3.ObjectSummary(bucket_name='tvpr-dataset', key='oni-7z/g022.oni.7z'), s3.ObjectSummary(bucket_name='tvpr-dataset', key='oni-7z/g023.oni.7z')]\n",
      "['oni-7z/g001.oni.7z', 'oni-7z/g002.oni.7z', 'oni-7z/g003.oni.7z', 'oni-7z/g004.oni.7z', 'oni-7z/g005.oni.7z', 'oni-7z/g006.oni.7z', 'oni-7z/g007.oni.7z', 'oni-7z/g008.oni.7z', 'oni-7z/g009.oni.7z', 'oni-7z/g010.oni.7z', 'oni-7z/g011.oni.7z', 'oni-7z/g012.oni.7z', 'oni-7z/g013.oni.7z', 'oni-7z/g014.oni.7z', 'oni-7z/g015.oni.7z', 'oni-7z/g016.oni.7z', 'oni-7z/g017.oni.7z', 'oni-7z/g018.oni.7z', 'oni-7z/g019.oni.7z', 'oni-7z/g020.oni.7z', 'oni-7z/g021.oni.7z', 'oni-7z/g022.oni.7z', 'oni-7z/g023.oni.7z']\n"
     ]
    }
   ],
   "source": [
    "source_prefix = 'oni-7z/'\n",
    "\n",
    "objects_7z = bucket.objects.filter(Prefix=source_prefix)\n",
    "objects_7z = [obj for obj in objects_7z if obj.key.endswith(\".7z\")]\n",
    "print(objects_7z)\n",
    "print([obj.key for obj in objects_7z])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "oni-7z/g008.oni.7z\n",
      "./temp//oni-7z/g008.oni.7z\n",
      "./temp//oni-7z/g008.oni.7z_/\n",
      "./temp//oni-7z/g008.oni.7z_/g008.oni\n",
      "oni/g008.oni\n",
      "oni-7z/g009.oni.7z\n",
      "./temp//oni-7z/g009.oni.7z\n",
      "./temp//oni-7z/g009.oni.7z_/\n",
      "./temp//oni-7z/g009.oni.7z_/g009.oni\n",
      "oni/g009.oni\n",
      "oni-7z/g010.oni.7z\n",
      "./temp//oni-7z/g010.oni.7z\n",
      "./temp//oni-7z/g010.oni.7z_/\n",
      "./temp//oni-7z/g010.oni.7z_/g010.oni\n",
      "oni/g010.oni\n",
      "oni-7z/g011.oni.7z\n",
      "./temp//oni-7z/g011.oni.7z\n",
      "./temp//oni-7z/g011.oni.7z_/\n",
      "./temp//oni-7z/g011.oni.7z_/g011.oni\n",
      "oni/g011.oni\n",
      "oni-7z/g012.oni.7z\n",
      "./temp//oni-7z/g012.oni.7z\n",
      "./temp//oni-7z/g012.oni.7z_/\n",
      "./temp//oni-7z/g012.oni.7z_/g012.oni\n",
      "oni/g012.oni\n",
      "oni-7z/g013.oni.7z\n",
      "./temp//oni-7z/g013.oni.7z\n",
      "./temp//oni-7z/g013.oni.7z_/\n",
      "./temp//oni-7z/g013.oni.7z_/g013.oni\n",
      "oni/g013.oni\n",
      "oni-7z/g014.oni.7z\n",
      "./temp//oni-7z/g014.oni.7z\n",
      "./temp//oni-7z/g014.oni.7z_/\n",
      "./temp//oni-7z/g014.oni.7z_/g014.oni\n",
      "oni/g014.oni\n",
      "oni-7z/g015.oni.7z\n",
      "./temp//oni-7z/g015.oni.7z\n",
      "./temp//oni-7z/g015.oni.7z_/\n",
      "./temp//oni-7z/g015.oni.7z_/g015.oni\n",
      "oni/g015.oni\n",
      "oni-7z/g016.oni.7z\n",
      "./temp//oni-7z/g016.oni.7z\n",
      "./temp//oni-7z/g016.oni.7z_/\n",
      "./temp//oni-7z/g016.oni.7z_/g016.oni\n",
      "oni/g016.oni\n",
      "oni-7z/g017.oni.7z\n",
      "./temp//oni-7z/g017.oni.7z\n",
      "./temp//oni-7z/g017.oni.7z_/\n",
      "./temp//oni-7z/g017.oni.7z_/g017.oni\n",
      "oni/g017.oni\n",
      "oni-7z/g018.oni.7z\n",
      "./temp//oni-7z/g018.oni.7z\n",
      "./temp//oni-7z/g018.oni.7z_/\n",
      "./temp//oni-7z/g018.oni.7z_/g018.oni\n",
      "oni/g018.oni\n",
      "oni-7z/g019.oni.7z\n",
      "./temp//oni-7z/g019.oni.7z\n",
      "./temp//oni-7z/g019.oni.7z_/\n",
      "./temp//oni-7z/g019.oni.7z_/g019.oni\n",
      "oni/g019.oni\n",
      "oni-7z/g020.oni.7z\n",
      "./temp//oni-7z/g020.oni.7z\n",
      "./temp//oni-7z/g020.oni.7z_/\n",
      "./temp//oni-7z/g020.oni.7z_/g020.oni\n",
      "oni/g020.oni\n",
      "oni-7z/g021.oni.7z\n",
      "./temp//oni-7z/g021.oni.7z\n",
      "./temp//oni-7z/g021.oni.7z_/\n",
      "./temp//oni-7z/g021.oni.7z_/g021.oni\n",
      "oni/g021.oni\n",
      "oni-7z/g022.oni.7z\n",
      "./temp//oni-7z/g022.oni.7z\n",
      "./temp//oni-7z/g022.oni.7z_/\n",
      "./temp//oni-7z/g022.oni.7z_/g022.oni\n",
      "oni/g022.oni\n",
      "oni-7z/g023.oni.7z\n",
      "./temp//oni-7z/g023.oni.7z\n",
      "./temp//oni-7z/g023.oni.7z_/\n",
      "./temp//oni-7z/g023.oni.7z_/g023.oni\n",
      "oni/g023.oni\n"
     ]
    }
   ],
   "source": [
    "import py7zr\n",
    "import os\n",
    "import glob\n",
    "\n",
    "temp_dir = f\"./temp/\"\n",
    "os.makedirs(temp_dir, exist_ok=True)\n",
    "\n",
    "target_prefix_oni = \"oni/\" \n",
    "\n",
    "for object_7z in objects_7z[7:]:\n",
    "    filename = temp_dir + \"/\" + object_7z.key\n",
    "    print(object_7z.key)\n",
    "    print(filename)\n",
    "\n",
    "    os.makedirs(os.path.dirname(filename), exist_ok=True)\n",
    "    if not os.path.exists(filepath):\n",
    "        bucket.download_file(Filename=filename, Key=object_7z.key)\n",
    "    \n",
    "    extract_dir = filename + \"_/\"\n",
    "    print(extract_dir)\n",
    "    \n",
    "    archive = py7zr.SevenZipFile(filename, mode='r')\n",
    "    ret = archive.extractall(path=os.path.dirname(extract_dir))\n",
    "    \n",
    "    for extracted_file in glob.glob(extract_dir + \"/*\"):\n",
    "        print(extracted_file)\n",
    "        if not extracted_file.endswith(\"oni\"):\n",
    "            continue\n",
    "        key = target_prefix_oni+os.path.basename(extracted_file)\n",
    "        print(key)\n",
    "        bucket.upload_file(Filename=extracted_file, Key=key)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "conda_amazonei_tensorflow_p36",
   "language": "python",
   "name": "conda_amazonei_tensorflow_p36"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}