2_MaterialsAPI_Data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import os\n",
    "import pandas as pd\n",
    "import gzip\n",
    "import pickle as pkl\n",
    "from itertools import combinations\n",
    "\n",
    "from pymatgen import MPRester \n",
    "    # Look to gist.github or pymatgen docs for examples\n",
    "from pymatgen import Composition\n",
    "import json\n",
    "\n",
    "from matminer.featurizers.base import MultipleFeaturizer\n",
    "from matminer.featurizers import composition as cf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Obtain data from MAPI via RESTurl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Keep private, scrub when publishing\n",
    "APIkey = '3LVbHWfwhG7Tzy0i'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "mpr = MPRester(api_key=APIkey)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = mpr.get_data('ZrAlNi',prop='density')\n",
    "data1 = mpr.get_data('ZrAlNi')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'density': 9.909384947524844,\n",
       "  'material_id': 'mp-8566',\n",
       "  'pretty_formula': 'Ag'},\n",
       " {'density': 9.888476577052216,\n",
       "  'material_id': 'mp-10597',\n",
       "  'pretty_formula': 'Ag'},\n",
       " {'density': 9.948341484985864,\n",
       "  'material_id': 'mp-124',\n",
       "  'pretty_formula': 'Ag'},\n",
       " {'density': 9.922633071705896,\n",
       "  'material_id': 'mp-989737',\n",
       "  'pretty_formula': 'Ag'}]"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mpr.query(['Ag'][0], properties=['material_id', 'density', 'pretty_formula'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'density': 0.39860429617432785,\n",
       "  'energy': -10.46073226,\n",
       "  'material_id': 'mp-1095860',\n",
       "  'pretty_formula': 'Al2CuNi'},\n",
       " {'density': 0.6139380274508028,\n",
       "  'energy': -10.63099917,\n",
       "  'material_id': 'mp-1096066',\n",
       "  'pretty_formula': 'AlCuNi2'},\n",
       " {'density': 5.785421822945918,\n",
       "  'energy': -99.45244755000002,\n",
       "  'material_id': 'mvc-2226',\n",
       "  'pretty_formula': 'AlCu3(NiO3)4'},\n",
       " {'density': 5.783943989953349,\n",
       "  'energy': -50.47093414,\n",
       "  'material_id': 'mvc-7674',\n",
       "  'pretty_formula': 'BaAlCuNiO5'},\n",
       " {'density': 5.042235453759157,\n",
       "  'energy': -149.40831635,\n",
       "  'material_id': 'mvc-864',\n",
       "  'pretty_formula': 'Sr2AlCu2NiO7'}]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mpr.query({'elements':{'$size':3, '$all':['Cu', 'Al', 'Ni']}}, properties=['material_id', 'density', 'pretty_formula','energy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'band_gap': 0.0,\n",
       " 'cif': \"# generated using pymatgen\\ndata_ZrAlNi\\n_symmetry_space_group_name_H-M   'P 1'\\n_cell_length_a   3.50853034\\n_cell_length_b   6.90772970\\n_cell_length_c   6.90716467\\n_cell_angle_alpha   120.00270371\\n_cell_angle_beta   90.00000000\\n_cell_angle_gamma   90.00000000\\n_symmetry_Int_Tables_number   1\\n_chemical_formula_structural   ZrAlNi\\n_chemical_formula_sum   'Zr3 Al3 Ni3'\\n_cell_volume   144.97034775\\n_cell_formula_units_Z   3\\nloop_\\n _symmetry_equiv_pos_site_id\\n _symmetry_equiv_pos_as_xyz\\n  1  'x, y, z'\\nloop_\\n _atom_site_type_symbol\\n _atom_site_label\\n _atom_site_symmetry_multiplicity\\n _atom_site_fract_x\\n _atom_site_fract_y\\n _atom_site_fract_z\\n _atom_site_occupancy\\n  Zr  Zr1  1  0.500000  0.592952  1.000000  1\\n  Zr  Zr2  1  0.500000  0.407024  0.407037  1\\n  Zr  Zr3  1  0.500000  0.999988  0.592963  1\\n  Al  Al4  1  0.000000  0.249448  0.000000  1\\n  Al  Al5  1  -0.000000  0.750544  0.750578  1\\n  Al  Al6  1  -0.000000  0.999967  0.249422  1\\n  Ni  Ni7  1  0.500000  0.000001  0.000000  1\\n  Ni  Ni8  1  0.000000  0.333340  0.666703  1\\n  Ni  Ni9  1  -0.000000  0.666636  0.333297  1\\n\",\n",
       " 'density': 6.078779520689209,\n",
       " 'diel': None,\n",
       " 'e_above_hull': 0,\n",
       " 'elasticity': {'G_Reuss': 66.0,\n",
       "  'G_VRH': 71.0,\n",
       "  'G_Voigt': 76.0,\n",
       "  'G_Voigt_Reuss_Hill': 71.0,\n",
       "  'K_Reuss': 138.0,\n",
       "  'K_VRH': 138.0,\n",
       "  'K_Voigt': 138.0,\n",
       "  'K_Voigt_Reuss_Hill': 138.0,\n",
       "  'compliance_tensor': [[6.4, -0.1, -3.9, 0.0, 0.0, -0.0],\n",
       "   [-0.1, 6.4, -3.9, 0.0, 0.0, -0.0],\n",
       "   [-3.9, -3.9, 10.2, -0.0, -0.0, 0.0],\n",
       "   [0.0, 0.0, -0.0, 10.7, -0.0, 0.0],\n",
       "   [0.0, 0.0, -0.0, -0.0, 10.7, -0.0],\n",
       "   [-0.0, -0.0, 0.0, 0.0, -0.0, 12.9]],\n",
       "  'elastic_anisotropy': 0.69,\n",
       "  'elastic_tensor': [[228.0, 73.0, 115.0, 0.0, 0.0, 0.0],\n",
       "   [73.0, 228.0, 115.0, 0.0, 0.0, 0.0],\n",
       "   [115.0, 115.0, 186.0, 0.0, 0.0, 0.0],\n",
       "   [0.0, 0.0, 0.0, 94.0, 0.0, 0.0],\n",
       "   [0.0, 0.0, 0.0, 0.0, 94.0, 0.0],\n",
       "   [0.0, 0.0, 0.0, 0.0, 0.0, 77.0]],\n",
       "  'elastic_tensor_original': [[229.19355882797788,\n",
       "    72.54575399375729,\n",
       "    114.99211590498733,\n",
       "    0.0,\n",
       "    0.0,\n",
       "    0.0],\n",
       "   [71.21102734419985, 227.86626029245733, 113.32235977267648, 0.0, 0.0, 0.0],\n",
       "   [115.04968688270137, 115.04967610224668, 185.93580725738425, 0.0, 0.0, 0.0],\n",
       "   [8.451010238625879e-05,\n",
       "    0.00015700389598746184,\n",
       "    0.00015784712052740226,\n",
       "    93.61783139741557,\n",
       "    0.0,\n",
       "    0.0],\n",
       "   [-6.387632067526433e-05,\n",
       "    -0.00036985605137119414,\n",
       "    -9.792291110045933e-05,\n",
       "    0.0,\n",
       "    93.71639370293845,\n",
       "    0.0],\n",
       "   [-0.0003359997218415392,\n",
       "    -0.0005542061936396585,\n",
       "    -0.0006506160758258414,\n",
       "    0.0,\n",
       "    0.0,\n",
       "    76.62945513543546]],\n",
       "  'homogeneous_poisson': 0.28,\n",
       "  'nsites': 9,\n",
       "  'poisson_ratio': 0.28,\n",
       "  'universal_anisotropy': 0.69,\n",
       "  'warnings': []},\n",
       " 'elements': ['Al', 'Ni', 'Zr'],\n",
       " 'energy': -60.05130156,\n",
       " 'energy_per_atom': -6.67236684,\n",
       " 'formation_energy_per_atom': -0.6468004433333334,\n",
       " 'full_formula': 'Zr3Al3Ni3',\n",
       " 'hubbards': {},\n",
       " 'icsd_id': None,\n",
       " 'icsd_ids': [608978, 608967, 410399, 186404, 152131, 106999, 58080],\n",
       " 'is_compatible': True,\n",
       " 'is_hubbard': False,\n",
       " 'material_id': 'mp-5723',\n",
       " 'nelements': 3,\n",
       " 'nsites': 9,\n",
       " 'oxide_type': 'None',\n",
       " 'piezo': None,\n",
       " 'pretty_formula': 'ZrAlNi',\n",
       " 'spacegroup': {'crystal_system': 'hexagonal',\n",
       "  'hall': 'P -6 -2',\n",
       "  'number': 189,\n",
       "  'point_group': '-6m2',\n",
       "  'source': 'spglib',\n",
       "  'symbol': 'P-62m'},\n",
       " 'tags': ['Aluminum nickel zirconium (1/1/1)',\n",
       "  'Aluminium nickel zirconium (1/1/1)',\n",
       "  'Aluminium zirconium nickel (1/1/1)',\n",
       "  'Zirconium nickel aluminium (1/1/1)'],\n",
       " 'task_ids': ['mp-993092',\n",
       "  'mp-5723',\n",
       "  'mp-994408',\n",
       "  'mp-991116',\n",
       "  'mp-1078484',\n",
       "  'mp-1079400'],\n",
       " 'total_magnetization': 0.0001018,\n",
       " 'unit_cell_formula': {'Al': 3.0, 'Ni': 3.0, 'Zr': 3.0},\n",
       " 'volume': 144.97034775488515}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data1[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Try getting density from MAPI"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "with gzip.open('./datasets/density_data.pkl.gz', 'rb') as fd:\n",
    "    densityData = pkl.load(fd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# make set of all ternary systems, grab all MP alloys in same ternary\n",
    "comps = densityData['comp']\n",
    "compSet = set()\n",
    "\n",
    "def getAlloySet(data):\n",
    "    '''Returns set of alloy element sets, iterating over \n",
    "        all items in ['composition'] column\n",
    "        \n",
    "    Args: \n",
    "        data (list-like): compositions\n",
    "    Returns:\n",
    "        (set): each item of the returned set is a set with strings \n",
    "                denoting element symbols\n",
    "    '''\n",
    "    alloys=set()\n",
    "    \n",
    "    for comp in data.tolist():\n",
    "        curr = frozenset(comp.as_dict().keys())\n",
    "        alloys.add(curr)\n",
    "    \n",
    "    return alloys\n",
    "\n",
    "for x in comps: # Generate all ternaries to look through\n",
    "    tern = frozenset(x.as_dict().keys())\n",
    "    compSet.add(tern)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "matches = mpr.query({'elements':{'$all':list(frozenset({'Ag', 'Hg', 'Au'}))}}, properties=['material_id', 'density', 'pretty_formula'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{frozenset({'Rh', 'Al', 'Co'}), frozenset({'Au', 'Hg', 'Ag'}), frozenset({'Cr', 'Mo', 'V'})}\n",
      "{frozenset({'Au', 'Hg', 'Ag'})}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{frozenset({'Ag', 'Au', 'Hg'})}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "issueComps = {frozenset({'Ag', 'Hg', 'Au'}), frozenset({'Al', 'Co','Rh'}), frozenset({'V', 'Cr', 'Mo'})}\n",
    "matchedComps = {frozenset(Composition(x['pretty_formula']).as_dict().keys()) for x in matches}\n",
    "print(issueComps)\n",
    "print(matchedComps)\n",
    "issueComps & matchedComps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "frozenset({'Au'})\n",
      "frozenset({'Hg'})\n",
      "found\n",
      "[frozenset({'Ag'})]\n"
     ]
    }
   ],
   "source": [
    "a = frozenset({'Ag','Hg','Au'})\n",
    "b = frozenset({'Ag'})\n",
    "c = frozenset({'Hg', 'Au', 'Ag'})\n",
    "comb = combinations(a, 1)\n",
    "for i in comb:\n",
    "    if frozenset(i) in [b]:\n",
    "        print('found')\n",
    "        continue\n",
    "    print(frozenset(i))\n",
    "\n",
    "frozenset('Ag') in [b]\n",
    "print([b])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Out of 251 compositions, 1 ternaries had no examples, found 3174 other items\n"
     ]
    }
   ],
   "source": [
    "cols = ['pretty_formula', 'density', 'material_id']\n",
    "MAPIdf = pd.DataFrame(columns=cols)\n",
    "dataDict = dict()\n",
    "failLog = set()\n",
    "\n",
    "issueComps = {frozenset({'Ag', 'Hg', 'Au'}), frozenset({'Al', 'Co','Rh'}), frozenset({'V', 'Cr', 'Mo'})}\n",
    "\n",
    "seenComps = set()\n",
    "rep=0\n",
    "for comp in compSet:\n",
    "    # Look for all data within the defined ternaries\n",
    "    # iterate through all lengths of combinations\n",
    "    \n",
    "    #if rep > 20:\n",
    "      #  break\n",
    "    \n",
    "    for n in range(len(comp)):\n",
    "        for c in combinations(comp, n): # single elements\n",
    "            fc = frozenset(c)\n",
    "            if fc in seenComps:\n",
    "                #print('found repeat: {}'.format(c))\n",
    "                continue\n",
    "            # look for composition with exact length of combination and containing elements in comb\n",
    "            matches = mpr.query({'elements':{'$size':len(c), '$all':list(c)}}, \n",
    "                                properties=['material_id', 'density', 'pretty_formula'])\n",
    "            seenComps.add(frozenset(c))\n",
    "\n",
    "            # format and add to database\n",
    "            if len(matches) == 0:\n",
    "                #print('no data for: {}'.format(tern))\n",
    "                failLog.add(tern)\n",
    "            else:\n",
    "                MAPIdf = MAPIdf.append(pd.DataFrame(matches))\n",
    "    #rep += 1\n",
    "\n",
    "print('Out of {} compositions, {} ternaries had no examples, found {} other items'.format(\n",
    "    len(compSet), len(failLog), len(MAPIdf)))\n",
    "        \n",
    "# add Composition object\n",
    "MAPIdf['comp'] = [Composition(x) for x in MAPIdf['pretty_formula']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>density</th>\n",
       "      <th>material_id</th>\n",
       "      <th>pretty_formula</th>\n",
       "      <th>comp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>8.902616</td>\n",
       "      <td>mp-998890</td>\n",
       "      <td>Cu</td>\n",
       "      <td>(Cu)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>8.930286</td>\n",
       "      <td>mp-989695</td>\n",
       "      <td>Cu</td>\n",
       "      <td>(Cu)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>8.935841</td>\n",
       "      <td>mp-989782</td>\n",
       "      <td>Cu</td>\n",
       "      <td>(Cu)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>8.886169</td>\n",
       "      <td>mp-1010136</td>\n",
       "      <td>Cu</td>\n",
       "      <td>(Cu)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.174855</td>\n",
       "      <td>mp-1056079</td>\n",
       "      <td>Cu</td>\n",
       "      <td>(Cu)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    density material_id pretty_formula  comp\n",
       "0  8.902616   mp-998890             Cu  (Cu)\n",
       "1  8.930286   mp-989695             Cu  (Cu)\n",
       "2  8.935841   mp-989782             Cu  (Cu)\n",
       "3  8.886169  mp-1010136             Cu  (Cu)\n",
       "4  0.174855  mp-1056079             Cu  (Cu)"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "MAPIdf.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "mapiCompSet = getAlloySet(MAPIdf['comp'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5026"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(MAPIdf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "base_featurizer = MultipleFeaturizer([cf.Stoichiometry(), cf.ElementProperty.from_preset(\"magpie\"),\n",
    "                                 cf.ValenceOrbital(props=['avg']), cf.IonProperty(fast=True),\n",
    "                                cf.YangSolidSolution(), cf.AtomicPackingEfficiency()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 0 ns\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "46b879d5344a4284bac50e3930448c1f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='MultipleFeaturizer', max=3174), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Computed 152 features\n"
     ]
    }
   ],
   "source": [
    "if True:\n",
    "    %%time\n",
    "    X_mapi = base_featurizer.featurize_many(MAPIdf['comp'], ignore_errors=True)\n",
    "    X_mapi = np.array(X_mapi)\n",
    "    X_mapi.astype(float)\n",
    "    print('Computed {} features'.format(X_mapi.shape[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "152"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feat_labels = base_featurizer.feature_labels()\n",
    "len(feat_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "with gzip.open('./datasets/MAPI_density_data.pkl.gz', 'wb') as fp:\n",
    "    pkl.dump(MAPIdf, fp)\n",
    "with gzip.open('./datasets/MAPI_density_features.pkl.gz', 'wb') as fx:\n",
    "    pkl.dump(X_mapi, fx)\n",
    "with gzip.open('./datasets/MAPI_density_featLabels.pkl.gz', 'wb') as fl:\n",
    "    pkl.dump(feat_labels, fl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "41751"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "matches = mpr.query({'nelements':3}, properties=['material_id', 'density', 'pretty_formula'])\n",
    "len(matches)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}