diff --git a/Task_assignment.ipynb b/Task_assignment.ipynb index e69de29..0f40423 100644 --- a/Task_assignment.ipynb +++ b/Task_assignment.ipynb @@ -0,0 +1,402 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Guide\n", + "1. Please write your solutions at the appropriate location **# YOUR CODE HERE**\n", + "2. Do not modify other cells" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "\n", + "# Numerical or binary attributes\n", + "num_or_bin= ['Income', 'Kidhome','Recency', 'MntWines', 'MntFruits', 'MntMeatProducts', 'MntFishProducts', 'MntSweetProducts','MntGoldProds', 'NumDealsPurchases', 'NumWebPurchases',\n", + " 'NumCatalogPurchases', 'NumStorePurchases', 'NumWebVisitsMonth','AcceptedCmp3', 'AcceptedCmp4', 'AcceptedCmp5', 'AcceptedCmp1','AcceptedCmp2']\n", + "\n", + "# Categorical attributes\n", + "cat = ['Education','Marital_Status']\n", + "\n", + "# Load the data source\n", + "pd.set_option('display.max_columns', None)\n", + "df = pd.read_csv(\"https://raw.githubusercontent.com/rpi-techfundamentals/website_fall_2021/master/site/public/marketing_campaign.csv\", delimiter='\\t')\n", + "df.head()" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Q1. Load and preprocess the data.\n", + "Build and transform your dataframe `df` using the following steps before modeling. Create a new dataframe `X` that has the following properties:\n", + "\n", + "1. It should include only the columns from lists `num_or_bin` and `cat` (in the code above).\n", + "2. Variables in cat should be transformed to `n-1` dummy variables (where `n` is the number of categories for each variable.)\n", + "3. Missing variables in the `Income` table should be filled with the median of the `income` table.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "raise NotImplementedError()\n", + "# YOUR CODE HERE" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Answer\n", + "'''\n", + "all_cols=num_or_bin+cat\n", + "X=pd.get_dummies(df[all_cols],columns=cat,drop_first=True)\n", + "X.loc[X['Income'].isnull(),'Income']=X['Income'].median()\n", + "'''" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#Please do not modify/delete this cell\n", + "# 1 Test Cell \n", + "#this tests the columns. \n", + "ecol = set(['Income', 'Kidhome', 'Recency', 'MntWines', 'MntFruits',\n", + " 'MntMeatProducts', 'MntFishProducts', 'MntSweetProducts',\n", + " 'MntGoldProds', 'NumDealsPurchases', 'NumWebPurchases',\n", + " 'NumCatalogPurchases', 'NumStorePurchases', 'NumWebVisitsMonth',\n", + " 'AcceptedCmp3', 'AcceptedCmp4', 'AcceptedCmp5', 'AcceptedCmp1',\n", + " 'AcceptedCmp2', 'Education_Basic', 'Education_Graduation',\n", + " 'Education_Master', 'Education_PhD', 'Marital_Status_Alone',\n", + " 'Marital_Status_Divorced', 'Marital_Status_Married',\n", + " 'Marital_Status_Single', 'Marital_Status_Together',\n", + " 'Marital_Status_Widow', 'Marital_Status_YOLO'])\n", + "assert len(ecol-set(X.columns))==0\n", + "assert len(set(X.columns))-len(ecol)==0" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#Please do not modify/delete this cell\n", + "# 2 Test Cell \n", + "#this tests the columns. \n", + "assert X.shape==(2240, 30)\n", + "q2=X.copy()\n", + "q2['Response']=df['Response']\n", + "q2['Year_Birth']=df['Year_Birth']\n", + "q2['Dt_Customer']=df['Dt_Customer']\n", + "q2.to_csv('q2.csv',index=False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#Please do not modify/delete this cell\n", + "# 3 Test Cell \n", + "assert len(X.loc[X['Income']==51381.5,'Income'])==24" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#Please do not modify/delete this cell\n", + "# 4 Test Cell\n", + "assert X.isnull().sum().sum()==0" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Reload data\n", + "import pandas as pd\n", + "df_cl=pd.read_csv('q2.csv')" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Q2. Feature Creation.\n", + "1. In the `df_cl` dataframe, create an integer feature `age` which is `2021` minus `Year_Birth`.\n", + "2. In the `df_cl` dataframe, create an integer feature `holiday` which is 1 if the month of `Dt_Customer` is equal to 12 and 0 otherwise. You can get the month by accessing `mm` from the date format `dd-mm-yyyy`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "raise NotImplementedError()\n", + "# YOUR CODE HERE" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Answer\n", + "'''\n", + "df_cl['age']=(2021-df_cl['Year_Birth']).astype(int)\n", + "df_cl['Dt_Customer_tmp']=df_cl['Dt_Customer'].apply(lambda x:x[3:5])\n", + "df_cl.loc[df_cl['Dt_Customer_tmp']=='12','holiday']=1\n", + "df_cl.loc[df_cl['Dt_Customer_tmp']!='12','holiday']=0\n", + "df_cl['holiday']=df_cl['holiday'].astype(int)\n", + "'''" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#Please do not modify/delete this cell\n", + "# 5 Test Cell\n", + "assert df_cl['age'].sum()==116915" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#Please do not modify/delete this cell\n", + "# 6 Test Cell\n", + "import numpy as np\n", + "assert isinstance(df_cl['age'][0], (np.int64,np.int32))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#Please do not modify/delete this cell\n", + "# 7 Test Cell\n", + "assert df_cl['holiday'].sum()==175" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#Please do not modify/delete this cell\n", + "# 8 Test Cell\n", + "import numpy as np\n", + "assert isinstance(df_cl['holiday'][0], (np.int64,np.int32))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "df_cl2=pd.read_csv('dfcl2.csv')\n", + "y_reload=df_cl2['Response']\n", + "X_reload=df_cl2.drop(['Response'], axis=1)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Q3. Train Test Split\n", + "Now split the data `X_reload`, `y_reload` intro training (50%) and testing data (50%) by using these variable names while making sure there are Make sure that there are equal number of positive cases (1's) in the train and the test set.\n", + "\n", + "`X_train`: Training feature columns\n", + "`X_test`: Testing feature columns\n", + "`y_train`: Training DV\n", + "`y_test`: Testing DV\n", + "\n", + "When doing so set the `random_state` parameter to 100 so you get the same split that is in the solution." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "raise NotImplementedError()\n", + "# YOUR CODE HERE" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Answer\n", + "'''\n", + "y_reload=df_cl2['Response']\n", + "X_reload=df_cl2.drop(['Response'], axis=1)\n", + "from sklearn.model_selection import train_test_split\n", + "X_train,X_test,y_train,y_test=train_test_split(X_reload,y_reload,\n", + " train_size=0.5,\n", + " test_size=0.5,\n", + " random_state=100,\n", + " stratify=y_reload)\n", + "'''" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#Please do not modify/delete this cell\n", + "# 9 Test Cell\n", + "assert X_train.shape==(1120, 31)\n", + "assert X_test.shape==(1120, 31)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#Please do not modify/delete this cell\n", + "# 10 Test Cell\n", + "assert y_train.sum()==y_test.sum()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "df_cl2=pd.read_csv('dfcl2.csv')\n", + "y_reload=df_cl2['Response']\n", + "X_reload=df_cl2.drop(['Response'], axis=1)\n", + "\n", + "X_train2, X_test2, y_train2, y_test2 = train_test_split(X_reload, y_reload, test_size=0.3, random_state = 100)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Q4. Classification\n", + "Use `X_train2`, `X_test2`, `y_train2`, `y_test2` from above. Use a Gradient Boosting classifier (as instantiated below).\n", + "\n", + "Please calculate:\n", + "1. `test_acc` The accuracy of the classifier on the test set.\n", + "2. `test_roc` The area under the ROC curve on on the test set." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn import metrics\n", + "from sklearn.ensemble import GradientBoostingClassifier\n", + "#use this classifier\n", + "classifier = GradientBoostingClassifier(n_estimators=100, learning_rate=1.0, max_depth=1, random_state=0)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "raise NotImplementedError()\n", + "# YOUR CODE HERE" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Answer\n", + "'''\n", + "classifier.fit(X_train2,y_train2)\n", + "test_y_pred=classifier.predict(X_test2)\n", + "test_y_pred_prob=classifier.predict_proba(X_test2)[:,1]\n", + "test_acc=metrics.accuracy_score(y_test2,test_y_pred)\n", + "test_roc=metrics.roc_auc_score(y_test2,test_y_pred_prob)\n", + "'''" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#Please do not modify/delete this cell\n", + "# 11 Test Cell\n", + "assert round(test_acc,3)==0.860" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#Please do not modify/delete this cell\n", + "# 12 Test Cell\n", + "assert round(test_roc,3)==0.846" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +}