xmlreader.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os \n",
    "os.chdir('K:/ecole/github/televerser/civ6/ref/resources')\n",
    "from bs4 import BeautifulSoup as bs\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#get critical information from xml files\n",
    "def get_info_from_xml(page,input_dict,tag_name,attr1,attr2):\n",
    "    \n",
    "    try:\n",
    "        #get info name\n",
    "        ftype=[row.get(attr1) for row in page.find(tag_name).find_all('row')]\n",
    "        \n",
    "        #get its attributes\n",
    "        ttype=[row.get(attr2) for row in page.find(tag_name).find_all('row')]\n",
    "    \n",
    "        #due to weird structure of suitable terrains,features\n",
    "        #the data structure has to be dictionary to host different values\n",
    "        for ind,fname in enumerate(ftype):\n",
    "            if fname not in input_dict:\n",
    "                input_dict[fname]=[ttype[ind]]\n",
    "            else:\n",
    "                input_dict[fname]+=[ttype[ind]]\n",
    "    except AttributeError:\n",
    "        pass\n",
    "    \n",
    "    return input_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.chdir('K:/ecole/github/televerser/civ6/ref/features')\n",
    "\n",
    "#initial data structures\n",
    "feature_list=[];feature_impassable=[];feature_tiles=[]\n",
    "feature_nocoast=[];feature_noriver=[]\n",
    "feature_validterrains={};feature_adjacentterrains={};feature_notadjacentterrains={}\n",
    "feature_adjacentfeatures={};feature_notnearfeatures={}\n",
    "\n",
    "#iterate through Australia_Features.xml; Expansion1_Features.xml;\n",
    "#Expansion1_Features_Major.xml; Expansion2_Features.xml;\n",
    "#Features.xml; GranColombia_Maya_Expansion2.xml;\n",
    "#GranColombia_Maya_Features.xml; Indonesia_Khmer_GameplayData.xml;\n",
    "#VikingsLandmarks_Features.xml\n",
    "for filename in os.listdir():\n",
    "    \n",
    "    #read xml\n",
    "    f=open(filename,'r')\n",
    "    text=f.read()\n",
    "    f.close()\n",
    "    page=bs(text,'lxml')\n",
    "    \n",
    "    #get simple attrs\n",
    "    feature_names=[i.get('type') for i in page.find('types').find_all('row')]\n",
    "    feature_list+=[i for i in feature_names if 'feature' in i.lower()]\n",
    "    feature_impassable+=[i.get('impassable') for i in page.find('features').find_all('row')]\n",
    "    feature_tiles+=[i.get('tiles') for i in page.find('features').find_all('row')]\n",
    "    feature_nocoast+=[i.get('nocoast') for i in page.find('features').find_all('row')]\n",
    "    feature_noriver+=[i.get('noriver') for i in page.find('features').find_all('row')]\n",
    "    \n",
    "    #get complex attrs\n",
    "    feature_validterrains=get_info_from_xml(page,feature_validterrains,\n",
    "                                            'feature_validterrains',\n",
    "                                            'featuretype','terraintype')   \n",
    "    feature_adjacentterrains=get_info_from_xml(page,feature_adjacentterrains,\n",
    "                                            'feature_adjacentterrains',\n",
    "                                            'featuretype','terraintype')\n",
    "    feature_notadjacentterrains=get_info_from_xml(page,feature_notadjacentterrains,\n",
    "                                            'feature_adjacentterrains',\n",
    "                                            'featuretype','terraintype')\n",
    "    feature_adjacentfeatures=get_info_from_xml(page,feature_adjacentfeatures,\n",
    "                                            'feature_adjacentfeatures',\n",
    "                                            'featuretype','featuretypeadjacent')\n",
    "    feature_notnearfeatures=get_info_from_xml(page,feature_notnearfeatures,\n",
    "                                            'feature_notnearfeatures',\n",
    "                                            'featuretype','featuretypeavoid')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#construct dataframe\n",
    "feature_df=pd.DataFrame()\n",
    "feature_df['name']=feature_list\n",
    "feature_df['impassable']=feature_impassable\n",
    "feature_df['tiles']=feature_tiles\n",
    "feature_df['nocoast']=feature_nocoast\n",
    "feature_df['noriver']=feature_noriver\n",
    "feature_df['validterrains']=feature_df['name'].apply(lambda x:'' if x not in feature_validterrains else ';'.join(feature_validterrains[x]))\n",
    "feature_df['adjacentterrains']=feature_df['name'].apply(lambda x:'' if x not in feature_adjacentterrains else ';'.join(feature_adjacentterrains[x]))\n",
    "feature_df['notadjacentterrains']=feature_df['name'].apply(lambda x:'' if x not in feature_notadjacentterrains else ';'.join(feature_notadjacentterrains[x]))\n",
    "feature_df['adjacentfeatures']=feature_df['name'].apply(lambda x:'' if x not in feature_adjacentfeatures else ';'.join(feature_adjacentfeatures[x]))\n",
    "feature_df['notnearfeatures']=feature_df['name'].apply(lambda x:'' if x not in feature_notnearfeatures else ';'.join(feature_notnearfeatures[x]))\n",
    "\n",
    "#fillna\n",
    "feature_df['noriver']=feature_df['noriver'].fillna('false')\n",
    "feature_df['nocoast']=feature_df['nocoast'].fillna('false')\n",
    "\n",
    "# #output\n",
    "# feature_df.drop_duplicates().to_csv('feature mapping.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.chdir('K:/ecole/github/televerser/civ6/ref/resources')\n",
    "\n",
    "#initial data structures\n",
    "resource_list=[];resource_frequency=[];\n",
    "resource_happiness=[];resource_seafrequency=[];\n",
    "resource_validterrains={};resource_validfeatures={}\n",
    "\n",
    "#iterate through Expansion1_Resources.xml; Expansion2_Resources.xml;\n",
    "#GranColombia_Maya_Expansion2.xml; GranColombia_Maya_Resources.xml;\n",
    "#Resources.xml\n",
    "for i in os.listdir():\n",
    "    \n",
    "    #read xml\n",
    "    f=open(i,'r')\n",
    "    text=f.read()\n",
    "    f.close()\n",
    "    page=bs(text,'lxml')\n",
    "    \n",
    "    try:\n",
    "        \n",
    "        #get simple attrs\n",
    "        resource_name=[i.get('type') for i in page.find('types').find_all('row')]\n",
    "        resource_list+=[i for i in resource_name if 'resource' in i.lower()]\n",
    "        resource_frequency+=[i.get('frequency') for i in page.find('resources').find_all('row')]\n",
    "        resource_happiness+=[i.get('happiness') for i in page.find('resources').find_all('row')]\n",
    "        resource_seafrequency+=[i.get('seafrequency') for i in page.find('resources').find_all('row')]\n",
    "        \n",
    "        #get complex attrs\n",
    "        resource_validterrains=get_info_from_xml(page,resource_validterrains,\n",
    "                                            'resource_validterrains',\n",
    "                                            'resourcetype','terraintype')\n",
    "\n",
    "        resource_validfeatures=get_info_from_xml(page,resource_validfeatures,\n",
    "                                                    'resource_validfeatures',\n",
    "                                                    'resourcetype','featuretype')\n",
    "    except AttributeError:\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#construct dataframe\n",
    "resource_df=pd.DataFrame()\n",
    "resource_df['name']=resource_list\n",
    "resource_df['frequency']=resource_frequency\n",
    "resource_df['happiness']=resource_happiness\n",
    "resource_df['seafrequency']=resource_seafrequency\n",
    "resource_df['validterrains']=resource_df['name'].apply(lambda x:'' if x not in resource_validterrains else ';'.join(resource_validterrains[x]))\n",
    "resource_df['validfeatures']=resource_df['name'].apply(lambda x:'' if x not in resource_validfeatures else ';'.join(resource_validfeatures[x]))\n",
    "\n",
    "#fillna\n",
    "resource_df['frequency'][resource_df['frequency'].isnull()]=resource_df['seafrequency'][resource_df['frequency'].isnull()]\n",
    "del resource_df['seafrequency']\n",
    "\n",
    "# #output\n",
    "# resource_df.drop_duplicates().to_csv('resource mapping.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}