xg-boost-steps.ipynb

{"cells":[{"metadata":{},"cell_type":"markdown","source":"# XG Boost Steps\n#### 1)Import and split\n#### 2)Set and fit the model\n#### 3)Predict\n#### 4)Model Tuning\n#### 5)Find best params, set and fit the model again, find final RMSE."},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd \nfrom sklearn.model_selection import train_test_split, GridSearchCV,cross_val_score\nfrom sklearn.metrics import mean_squared_error, r2_score\nimport matplotlib.pyplot as plt\nfrom sklearn.preprocessing import scale \nfrom sklearn import model_selection\nfrom sklearn.tree import DecisionTreeRegressor, DecisionTreeClassifier\nfrom sklearn.neighbors import KNeighborsRegressor\nfrom sklearn.ensemble import BaggingRegressor\n\nfrom warnings import filterwarnings\nfilterwarnings('ignore')","execution_count":1,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Import and split ","execution_count":2,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"hit = pd.read_csv(\"../input/hittlers/Hitters.csv\")\ndf = hit.copy()\ndf = df.dropna()\ndms = pd.get_dummies(df[['League', 'Division', 'NewLeague']])\ny = df[\"Salary\"]\nX_ = df.drop(['Salary', 'League', 'Division', 'NewLeague'], axis=1).astype('float64')\nX = pd.concat([X_, dms[['League_N', 'Division_W', 'NewLeague_N']]], axis=1)\nX_train, X_test, y_train, y_test = train_test_split(X, y, \n                                                    test_size=0.25, \n                                                    random_state=42)","execution_count":3,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Set and fit the model","execution_count":4,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install xgboost","execution_count":5,"outputs":[{"output_type":"stream","text":"Requirement already satisfied: xgboost in /opt/conda/lib/python3.7/site-packages (1.0.2)\nRequirement already satisfied: scipy in /opt/conda/lib/python3.7/site-packages (from xgboost) (1.4.1)\nRequirement already satisfied: numpy in /opt/conda/lib/python3.7/site-packages (from xgboost) (1.18.1)\n\u001b[33mWARNING: You are using pip version 20.1; however, version 20.1.1 is available.\nYou should consider upgrading via the '/opt/conda/bin/python3.7 -m pip install --upgrade pip' command.\u001b[0m\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"import xgboost as xgb","execution_count":6,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#We can use special data structure for XGBoost to have high performanced result.\n\nDM_train=xgb.DMatrix(data=X_train, label=y_train)   #enter dependent variable for \"label\"\nDM_test=xgb.DMatrix(data=X_test, label=y_test)  ","execution_count":7,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from xgboost import XGBRegressor","execution_count":8,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"xgb_model=XGBRegressor().fit(X_train,y_train) # i prefer to use classic data structure which i get accustomed to use","execution_count":9,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Prediction","execution_count":10,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_pred=xgb_model.predict(X_test)","execution_count":11,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.sqrt(mean_squared_error(y_pred,y_test))","execution_count":12,"outputs":[{"output_type":"execute_result","execution_count":12,"data":{"text/plain":"355.4651481224188"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Model Tuning","execution_count":13,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"xgb_model","execution_count":14,"outputs":[{"output_type":"execute_result","execution_count":14,"data":{"text/plain":"XGBRegressor(base_score=0.5, booster=None, colsample_bylevel=1,\n             colsample_bynode=1, colsample_bytree=1, gamma=0, gpu_id=-1,\n             importance_type='gain', interaction_constraints=None,\n             learning_rate=0.300000012, max_delta_step=0, max_depth=6,\n             min_child_weight=1, missing=nan, monotone_constraints=None,\n             n_estimators=100, n_jobs=0, num_parallel_tree=1,\n             objective='reg:squarederror', random_state=0, reg_alpha=0,\n             reg_lambda=1, scale_pos_weight=1, subsample=1, tree_method=None,\n             validate_parameters=False, verbosity=None)"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Important params\n#booster\n#colsample_bytree\n#learning_rate  :avoids overfitting. \n#max_depth      \n#n_estimators","execution_count":15,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"xgb_grid = {\n     'colsample_bytree': [0.4, 0.5,0.6,0.9,1], \n     'n_estimators':[100, 200, 500, 1000],\n     'max_depth': [2,3,4,5,6],\n     'learning_rate': [0.1, 0.01, 0.5]\n}","execution_count":16,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"xgb_cv=GridSearchCV(xgb_model, xgb_grid, cv=10,n_jobs=-1, verbose=2)\nxgb_cv.fit(X_train,y_train)","execution_count":17,"outputs":[{"output_type":"stream","text":"Fitting 10 folds for each of 300 candidates, totalling 3000 fits\n","name":"stdout"},{"output_type":"stream","text":"[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n[Parallel(n_jobs=-1)]: Done  33 tasks      | elapsed:    3.3s\n[Parallel(n_jobs=-1)]: Done 248 tasks      | elapsed:   12.1s\n[Parallel(n_jobs=-1)]: Done 654 tasks      | elapsed:   28.7s\n[Parallel(n_jobs=-1)]: Done 1220 tasks      | elapsed:   54.1s\n[Parallel(n_jobs=-1)]: Done 1950 tasks      | elapsed:  1.5min\n[Parallel(n_jobs=-1)]: Done 2606 tasks      | elapsed:  2.3min\n[Parallel(n_jobs=-1)]: Done 3000 out of 3000 | elapsed:  2.7min finished\n","name":"stderr"},{"output_type":"execute_result","execution_count":17,"data":{"text/plain":"GridSearchCV(cv=10, error_score=nan,\n             estimator=XGBRegressor(base_score=0.5, booster=None,\n                                    colsample_bylevel=1, colsample_bynode=1,\n                                    colsample_bytree=1, gamma=0, gpu_id=-1,\n                                    importance_type='gain',\n                                    interaction_constraints=None,\n                                    learning_rate=0.300000012, max_delta_step=0,\n                                    max_depth=6, min_child_weight=1,\n                                    missing=nan, monotone_constraints=None,\n                                    n_estimators=100, n_jobs=...\n                                    random_state=0, reg_alpha=0, reg_lambda=1,\n                                    scale_pos_weight=1, subsample=1,\n                                    tree_method=None, validate_parameters=False,\n                                    verbosity=None),\n             iid='deprecated', n_jobs=-1,\n             param_grid={'colsample_bytree': [0.4, 0.5, 0.6, 0.9, 1],\n                         'learning_rate': [0.1, 0.01, 0.5],\n                         'max_depth': [2, 3, 4, 5, 6],\n                         'n_estimators': [100, 200, 500, 1000]},\n             pre_dispatch='2*n_jobs', refit=True, return_train_score=False,\n             scoring=None, verbose=2)"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"xgb_cv.best_params_","execution_count":18,"outputs":[{"output_type":"execute_result","execution_count":18,"data":{"text/plain":"{'colsample_bytree': 0.6,\n 'learning_rate': 0.1,\n 'max_depth': 2,\n 'n_estimators': 1000}"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Final tuned model","execution_count":19,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"xgb_tuned= XGBRegressor(colsample_bytree=0.6,\n learning_rate=0.1,\n max_depth= 2,\n n_estimators=1000)","execution_count":21,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"xgb_tuned=xgb_tuned.fit(X_train,y_train)","execution_count":22,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_pred=xgb_tuned.predict(X_test)","execution_count":23,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.sqrt(mean_squared_error(y_test,y_pred))","execution_count":24,"outputs":[{"output_type":"execute_result","execution_count":24,"data":{"text/plain":"355.0512895885908"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"# We found 413 for KNN, \n#          367 for SVR,\n#          363 for Artifical Neural Network.\n#          376 for CART\n#          349 for Bagged Trees\n#          350 for Random Forest\n#          344 for GBM\n#And now,  355 for XG Boosting\n\n#In these models, the best one is GBM model for \"hitters\" data set, till now.","execution_count":25,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}