python-bootcamp/Task_assignment.ipynb

Guide

1. Please write your solutions at the appropriate location # YOUR CODE HERE
2. Do not modify other cells

In [ ]:

import pandas as pd

# Numerical or binary attributes
num_or_bin= ['Income', 'Kidhome','Recency', 'MntWines', 'MntFruits', 'MntMeatProducts', 'MntFishProducts', 'MntSweetProducts','MntGoldProds', 'NumDealsPurchases', 'NumWebPurchases',
       'NumCatalogPurchases', 'NumStorePurchases', 'NumWebVisitsMonth','AcceptedCmp3', 'AcceptedCmp4', 'AcceptedCmp5', 'AcceptedCmp1','AcceptedCmp2']

# Categorical attributes
cat = ['Education','Marital_Status']

# Load the data source
pd.set_option('display.max_columns', None)
df  = pd.read_csv("https://raw.githubusercontent.com/rpi-techfundamentals/website_fall_2021/master/site/public/marketing_campaign.csv", delimiter='\t')
df.head()

Q1. Load and preprocess the data.

Build and transform your dataframe df using the following steps before modeling. Create a new dataframe X that has the following properties:

1. It should include only the columns from lists num_or_bin and cat (in the code above).
2. Variables in cat should be transformed to n-1 dummy variables (where n is the number of categories for each variable.)
3. Missing variables in the Income table should be filled with the median of the income table.

In [ ]:

raise NotImplementedError()
# YOUR CODE HERE

In [ ]:

# Answer
'''
all_cols=num_or_bin+cat
X=pd.get_dummies(df[all_cols],columns=cat,drop_first=True)
X.loc[X['Income'].isnull(),'Income']=X['Income'].median()
'''

In [ ]:

#Please do not modify/delete this cell
# 1 Test Cell 
#this tests the columns. 
ecol = set(['Income', 'Kidhome', 'Recency', 'MntWines', 'MntFruits',
       'MntMeatProducts', 'MntFishProducts', 'MntSweetProducts',
       'MntGoldProds', 'NumDealsPurchases', 'NumWebPurchases',
       'NumCatalogPurchases', 'NumStorePurchases', 'NumWebVisitsMonth',
       'AcceptedCmp3', 'AcceptedCmp4', 'AcceptedCmp5', 'AcceptedCmp1',
       'AcceptedCmp2', 'Education_Basic', 'Education_Graduation',
       'Education_Master', 'Education_PhD', 'Marital_Status_Alone',
       'Marital_Status_Divorced', 'Marital_Status_Married',
       'Marital_Status_Single', 'Marital_Status_Together',
       'Marital_Status_Widow', 'Marital_Status_YOLO'])
assert len(ecol-set(X.columns))==0
assert len(set(X.columns))-len(ecol)==0

In [ ]:

#Please do not modify/delete this cell
# 2 Test Cell 
#this tests the columns. 
assert X.shape==(2240, 30)
q2=X.copy()
q2['Response']=df['Response']
q2['Year_Birth']=df['Year_Birth']
q2['Dt_Customer']=df['Dt_Customer']
q2.to_csv('q2.csv',index=False)

In [ ]:

#Please do not modify/delete this cell
# 3 Test Cell 
assert len(X.loc[X['Income']==51381.5,'Income'])==24

In [ ]:

#Please do not modify/delete this cell
# 4 Test Cell
assert X.isnull().sum().sum()==0

In [ ]:

# Reload data
import pandas as pd
df_cl=pd.read_csv('q2.csv')

Q2. Feature Creation.

1. In the df_cl dataframe, create an integer feature age which is 2021 minus Year_Birth.
2. In the df_cl dataframe, create an integer feature holiday which is 1 if the month of Dt_Customer is equal to 12 and 0 otherwise. You can get the month by accessing mm from the date format dd-mm-yyyy.

In [ ]:

raise NotImplementedError()
# YOUR CODE HERE

In [ ]:

# Answer
'''
df_cl['age']=(2021-df_cl['Year_Birth']).astype(int)
df_cl['Dt_Customer_tmp']=df_cl['Dt_Customer'].apply(lambda x:x[3:5])
df_cl.loc[df_cl['Dt_Customer_tmp']=='12','holiday']=1
df_cl.loc[df_cl['Dt_Customer_tmp']!='12','holiday']=0
df_cl['holiday']=df_cl['holiday'].astype(int)
'''

In [ ]:

#Please do not modify/delete this cell
# 5 Test Cell
assert df_cl['age'].sum()==116915

In [ ]:

#Please do not modify/delete this cell
# 6 Test Cell
import numpy as np
assert isinstance(df_cl['age'][0], (np.int64,np.int32))

In [ ]:

#Please do not modify/delete this cell
# 7 Test Cell
assert df_cl['holiday'].sum()==175

In [ ]:

#Please do not modify/delete this cell
# 8 Test Cell
import numpy as np
assert isinstance(df_cl['holiday'][0], (np.int64,np.int32))

In [ ]:

import pandas as pd
df_cl2=pd.read_csv('dfcl2.csv')
y_reload=df_cl2['Response']
X_reload=df_cl2.drop(['Response'], axis=1)

Q3. Train Test Split

Now split the data X_reload, y_reload intro training (50%) and testing data (50%) by using these variable names while making sure there are Make sure that there are equal number of positive cases (1's) in the train and the test set.

X_train: Training feature columns X_test: Testing feature columns y_train: Training DV y_test: Testing DV

When doing so set the random_state parameter to 100 so you get the same split that is in the solution.

In [ ]:

raise NotImplementedError()
# YOUR CODE HERE

In [ ]:

# Answer
'''
y_reload=df_cl2['Response']
X_reload=df_cl2.drop(['Response'], axis=1)
from sklearn.model_selection import train_test_split
X_train,X_test,y_train,y_test=train_test_split(X_reload,y_reload,
                                               train_size=0.5,
                                               test_size=0.5,
                                               random_state=100,
                                               stratify=y_reload)
'''

In [ ]:

#Please do not modify/delete this cell
# 9 Test Cell
assert X_train.shape==(1120, 31)
assert X_test.shape==(1120, 31)

In [ ]:

#Please do not modify/delete this cell
# 10 Test Cell
assert  y_train.sum()==y_test.sum()

In [ ]:

import pandas as pd
df_cl2=pd.read_csv('dfcl2.csv')
y_reload=df_cl2['Response']
X_reload=df_cl2.drop(['Response'], axis=1)

X_train2, X_test2, y_train2, y_test2 = train_test_split(X_reload, y_reload, test_size=0.3, random_state = 100)

Q4. Classification

Use X_train2, X_test2, y_train2, y_test2 from above. Use a Gradient Boosting classifier (as instantiated below).

Please calculate:

1. test_acc The accuracy of the classifier on the test set.
2. test_roc The area under the ROC curve on on the test set.

In [ ]:

from sklearn import metrics
from sklearn.ensemble import GradientBoostingClassifier
#use this classifier
classifier = GradientBoostingClassifier(n_estimators=100, learning_rate=1.0, max_depth=1, random_state=0)

In [ ]:

raise NotImplementedError()
# YOUR CODE HERE

In [ ]:

# Answer
'''
classifier.fit(X_train2,y_train2)
test_y_pred=classifier.predict(X_test2)
test_y_pred_prob=classifier.predict_proba(X_test2)[:,1]
test_acc=metrics.accuracy_score(y_test2,test_y_pred)
test_roc=metrics.roc_auc_score(y_test2,test_y_pred_prob)
'''

In [ ]:

#Please do not modify/delete this cell
# 11 Test Cell
assert round(test_acc,3)==0.860

In [ ]:

#Please do not modify/delete this cell
# 12 Test Cell
assert round(test_roc,3)==0.846