import seaborn as sns
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import warnings
warnings.filterwarnings("ignore")

import plotly.express as px
import plotly.graph_objects as go

ok_cupid_df = pd.read_csv('data/okcupid_profiles.csv')
ok_cupid_df.columns

ok_cupid_df.shape

ok_cupid_df.info()

ok_cupid_df.head(2)

ok_cupid_df.isna().sum()

ok_cupid_df.isna().sum()/ok_cupid_df.shape[0]*100

ok_cupid_df['drugs'] = ok_cupid_df['drugs'].fillna('unknown_drugs')
ok_cupid_df['drugs'].value_counts()

ok_cupid_df['diet'] = ok_cupid_df['diet'].fillna('unknowndiet')
ok_cupid_df['diet'].value_counts()

#ok_cupid_df.loc[(ok_cupid_df['diet'] == 'unknown')&(ok_cupid_df['essay0'] == '57'), 'status'] = 'available'
ok_cupid_df.loc[ok_cupid_df['essay0'] == "im looking for someone to share some raging adhd. im a self motivated and light hearted superhero who enjoy's riding my bike everywhere and eating every goddamn thing i can.  im looking for someone to go adventuring with. i enjoy blind drunken adventures sometimes but you dont have to be a drinker. no vegans, i will eat anything... including people... especially hipsters. im not really a nerd (i don't play magic cards/excessive videogames) but i can like nerdy girls.  i just got this account, so gimmie some time to write down more shenanigans that are important  if u make chiptunes hit me the fuck up! i wanna make some!  i am awesome, eccentric, and energetic", 'diet'] = 'strictly anything'

ok_cupid_df.loc[ok_cupid_df['essay0'] == "rabid bibliophile, humorless feminist (that's a joke), eternal student. i like to write poetry on people, bake (vegan) cupcakes, make art and dress-up.  i identify as queer but my choices here are limited so i chose bisexual.  i am quiet, empathetic, and geeky", 'diet'] = 'vegan'

#ok_cupid_df[ok_cupid_df['diet'] == 'unknown_diet']['essay0'].tolist()

ok_cupid_df['status'].value_counts()

ok_cupid_df["status"].replace({'unknown': 'unknown_status'}, inplace=True)

ok_cupid_df['status'].value_counts()

ok_cupid_df['body_type'] = ok_cupid_df['body_type'].fillna('unknown_body_type')
ok_cupid_df['body_type'].value_counts()

ok_cupid_df['education'] = ok_cupid_df['education'].fillna('unknown_education')
ok_cupid_df['education'].value_counts()

ok_cupid_df['job'] = ok_cupid_df['job'].fillna('unknown_job')
#ok_cupid_df['job'].value_counts()

ok_cupid_df['ethnicity'] = ok_cupid_df['ethnicity'].fillna('unknown_ethnicity')
#ok_cupid_df['ethnicity'].value_counts()

ok_cupid_df['offspring'] = ok_cupid_df['offspring'].fillna('unknown_offspring')
#ok_cupid_df['offspring'].value_counts()

ok_cupid_df['pets'] = ok_cupid_df['pets'].fillna('unknown_pets')
#ok_cupid_df['pets'].value_counts()

ok_cupid_df['religion'] = ok_cupid_df['religion'].fillna('unknown_religion')
#ok_cupid_df['religion'].value_counts()

ok_cupid_df['sign'] = ok_cupid_df['sign'].fillna('unknown_sign')
#ok_cupid_df['sign'].value_counts()

ok_cupid_df['smokes'] = ok_cupid_df['smokes'].fillna('unknown_smokes')
ok_cupid_df['smokes'].value_counts()

ok_cupid_df['drinks'] = ok_cupid_df['drinks'].fillna('unknown_drinks')
ok_cupid_df['drinks'].value_counts()

ok_cupid_df['speaks'] = ok_cupid_df['speaks'].fillna('unknown_speaks')
ok_cupid_df['speaks'].value_counts()

ok_cupid_df[ok_cupid_df['height'].isna()]

ok_cupid_df.isna().sum()

ok_cupid_df[ok_cupid_df.duplicated()]

ok_cupid_df['essay0'] = ok_cupid_df['essay0'].fillna('')
ok_cupid_df['essay1'] = ok_cupid_df['essay1'].fillna('')
ok_cupid_df['essay2'] = ok_cupid_df['essay2'].fillna('')
ok_cupid_df['essay3'] = ok_cupid_df['essay3'].fillna('')
ok_cupid_df['essay4'] = ok_cupid_df['essay4'].fillna('')
ok_cupid_df['essay5'] = ok_cupid_df['essay5'].fillna('')
ok_cupid_df['essay6'] = ok_cupid_df['essay6'].fillna('')
ok_cupid_df['essay7'] = ok_cupid_df['essay7'].fillna('')
ok_cupid_df['essay8'] = ok_cupid_df['essay8'].fillna('')
ok_cupid_df['essay9'] = ok_cupid_df['essay9'].fillna('')

essay_columns = ['essay0', 'essay1', 'essay2', 'essay3', 'essay4', 'essay5', 'essay6', 'essay7', 'essay8', 'essay9']
ok_cupid_df['essay'] = ok_cupid_df[essay_columns].apply(lambda row: ' '.join(row.values.astype(object)), axis = 1)
ok_cupid_df['essay'] = ok_cupid_df['essay'].str.replace('[^\w\s]','')

avg_words = pd.DataFrame()
avg_words['avg_words'] = ok_cupid_df['essay'].str.split().str.len()

avg_words.describe().T

ok_cupid_df['essay_len'] = ok_cupid_df['essay'].str.len()

ok_cupid_df = ok_cupid_df.drop(columns = ['essay0', 'essay1', 'essay2', 'essay3', 'essay4', 'essay5', 'essay6', 'essay7', 'essay8', 'essay9'])

ok_cupid_df.head(2)

ok_cupid_df.isna().sum()

ok_cupid_df.dropna(inplace=True)

ok_cupid_df[ok_cupid_df['age']>100]

#ok_cupid_df = ok_cupid_df.drop(ok_cupid_df.index[[, 25324]])
ok_cupid_df = ok_cupid_df.drop(index = [2512, 25324])

ok_cupid_df[ok_cupid_df['age']>100]

ok_cupid_df.columns.duplicated()

ok_cupid_df[ok_cupid_df.duplicated()]

ok_cupid_df['income'].value_counts()

ok_cupid_df.shape

#for column in ok_cupid_df.columns:
    
#    plt.figure()
#    plt.hist(ok_cupid_df[column], bins=25)
#    plt.title(f'Histogram of {column}')
#    plt.show()

valid_income_df = ok_cupid_df[ok_cupid_df['income'] != -1]

male_df_income = valid_income_df[valid_income_df['sex'] == 'm']
female_df_income = valid_income_df[valid_income_df['sex'] == 'f']

ok_cupid_df2 = ok_cupid_df.copy()

ok_cupid_df2['body_type'] = ok_cupid_df2['body_type'].replace(['a little extra', 'full figured'],'curvy')
ok_cupid_df2['body_type'] = ok_cupid_df2['body_type'].replace(['thin'],'skinny')
ok_cupid_df2['body_type'] = ok_cupid_df2['body_type'].replace(['fit'],'athletic')
ok_cupid_df2['body_type'] = ok_cupid_df2['body_type'].replace(['unknown_body_type'],'rather not say')

# Add new binary column for gender
ok_cupid_df["male"] = np.where(ok_cupid_df["sex"]=="m", 1, 0)
ok_cupid_df.head(2)

# Drop the sex column
ok_cupid_df.drop(columns="sex", inplace=True)
ok_cupid_df.head(2)

ok_cupid_df.reset_index(drop=True, inplace=True)

ok_cupid_df2.columns

valid_income_df['education'] = valid_income_df['education'].replace({'working on space camp': 'space camp', 
                                                                     'working on college/university': 'college/university', 
                                                                     'graduated from college/university': 'college/university',
                                                                     'graduated from high school': 'high school',
                                                                     'dropped out of space camp': 'space camp',
                                                                     'graduated from space camp': 'space camp',
                                                                     'graduated from law school': 'law school',
                                                                     'graduated from masters program': 'masters program',
                                                                     'graduated from two-year college': 'two-year college',
                                                                     'working on med school': 'med school',
                                                                     'dropped out of high school': 'high school',
                                                                     'working on ph.d program': 'ph.d program',
                                                                     'graduated from ph.d program': 'ph.d program',
                                                                     'dropped out of college/university': 'college/university',
                                                                     'dropped out of two-year college': 'two-year college',
                                                                     'dropped out of med school': 'med school',
                                                                     'working on masters program': 'masters program',
                                                                     'working on two-year college': 'two-year college',
                                                                     'working on high school': 'high school',
                                                                     'graduated from med school': 'med school',
                                                                     'dropped out of masters program': 'masters program',
                                                                     'working on law school': 'law school',
                                                                     'dropped out of ph.d program': 'ph.d program',
                                                                     'dropped out of law school': 'law school'})

male_df_income = valid_income_df[valid_income_df['sex'] == 'm']
female_df_income = valid_income_df[valid_income_df['sex'] == 'f']

means_male = []
medians_male = []
for education in valid_income_df['education'].unique():
    means_male.append(male_df_income[male_df_income['education'] == education]['income'].mean())
    medians_male.append(male_df_income[male_df_income['education'] == education]['income'].median())
    
list_of_tuples_male = list(zip(male_df_income['education'].unique(), means_male, medians_male))

df_mean_incomes_male = pd.DataFrame(list_of_tuples_male, columns=['education', 'mean_income', 'median_income'])

means_female = []
medians_female = []
for education in valid_income_df['education'].unique():
    means_female.append(female_df_income[female_df_income['education'] == education]['income'].mean())
    medians_female.append(female_df_income[female_df_income['education'] == education]['income'].median())
    
list_of_tuples_female = list(zip(female_df_income['education'].unique(), means_female, medians_female))

df_mean_incomes_female = pd.DataFrame(list_of_tuples_female, columns=['education', 'mean_income', 'median_income'])

df_mean_incomes_male_mean = df_mean_incomes_male.sort_values(by = 'mean_income')
df_mean_incomes_male_median = df_mean_incomes_male.sort_values(by = 'median_income')

df_mean_incomes_female_mean = df_mean_incomes_female.sort_values(by = 'mean_income')
df_mean_incomes_female_median = df_mean_incomes_female.sort_values(by = 'median_income')
#df_mean_incomes_male_mean

educations = valid_income_df['education'].unique()

fig = go.Figure(data=[
    go.Bar(name='Male', 
           y=educations, 
           x=df_mean_incomes_male_median['median_income'], 
           text=df_mean_incomes_male_median['median_income'], orientation='h'),
    go.Bar(name='Female', 
           y=educations, 
           x=df_mean_incomes_female_median['median_income'],
           text=df_mean_incomes_female_median['median_income'], orientation='h')
])

fig.update_layout(barmode='group',
    title='Median Income for each education',
    #xaxis_tickfont_size=14,
    yaxis=dict(title='Educations',
               titlefont_size=16,
               tickfont_size=14),
    xaxis=dict(title='Median Income',
               titlefont_size=16,
               tickfont_size=14))
fig.update_traces(texttemplate='%{text:.2s}', textposition='outside')
fig.show()

fig = go.Figure()

fig.add_trace(go.Histogram(x=male_df_income['income'], name='Male'))
fig.add_trace(go.Histogram(x=female_df_income['income'], name='Female'))

fig.update_layout(barmode='overlay',
                  title='Distribution of Income for Males and Females',
                  #xaxis_tickfont_size=14,
                  yaxis=dict(title='Count',
                             titlefont_size=16,
                             tickfont_size=14),
                  xaxis=dict(title='Income',
                             titlefont_size=16,
                             tickfont_size=14))
# Reduce opacity to see both histograms
fig.update_traces(opacity=0.75)
fig.show()

labels = ['Male', 'Female']
values = ok_cupid_df2['sex'].value_counts().values
fig = go.Figure(data=[go.Pie(labels=labels, 
                             values=values, 
                             hole=.5)])
fig.update_layout(title='Male vs Female')
fig.show()

male_df = ok_cupid_df2[ok_cupid_df2['sex'] == 'm']
female_df = ok_cupid_df2[ok_cupid_df2['sex'] == 'f']

ok_cupid_df2['religion'] = ok_cupid_df2['religion'].replace({'agnosticism and very serious about it': 'agnosticism',
'agnosticism but not too serious about it' : 'agnosticism', 
'agnosticism and somewhat serious about it': 'agnosticism',
'agnosticism and laughing about it': 'agnosticism','agnosticism': 'agnostic'})

ok_cupid_df2['religion'] = ok_cupid_df2['religion'].replace({'atheism and laughing about it':'atheism',
'atheism and somewhat serious about it': 'atheism', 
'atheism but not too serious about it': 'atheism', 'atheism and very serious about it':'atheism', 'atheism': 'atheist'})

for religion in ok_cupid_df2['religion'].unique():
    if ((religion != 'atheist') and (religion != 'agnostic') and (religion != 'unknown_religion')):
        ok_cupid_df2['religion'] = ok_cupid_df2['religion'].replace({religion: 'religious'})

ok_cupid_df['religion_binary'] = ok_cupid_df['religion']

ok_cupid_df['religion_binary'] = ok_cupid_df['religion_binary'].replace({'agnosticism and very serious about it': 'agnostic',
'agnosticism but not too serious about it' : 'agnostic', 
'agnosticism and somewhat serious about it': 'agnostic',
'agnosticism and laughing about it': 'agnosticism','agnosticism': 'agnostic'})

ok_cupid_df['religion_binary'] = ok_cupid_df['religion_binary'].replace({'atheism and laughing about it':'atheism',
'atheism and somewhat serious about it': 'atheist', 
'atheism but not too serious about it': 'atheist', 'atheism and very serious about it':'atheist', 'atheism': 'atheist'})

for religion in ok_cupid_df['religion_binary'].unique():
    if ((religion != 'atheist') and (religion != 'agnostic') and (religion != 'unknown_religion')):
        ok_cupid_df['religion_binary'] = ok_cupid_df['religion_binary'].replace({religion: 'religious'})

ok_cupid_df2['religion'].value_counts()

ok_cupid_df['religion_binary'].value_counts()

ok_cupid_df.head(2)

ok_cupid_df['is_religious'] = np.where(ok_cupid_df['religion_binary'] == 'religious', 1, 0)
ok_cupid_df['is_agnostic'] = np.where(ok_cupid_df['religion_binary'] == 'agnostic', 1, 0)
ok_cupid_df['is_atheist'] = np.where(ok_cupid_df['religion_binary'] == 'atheist', 1, 0)

ok_cupid_df.head(2)

straight_df = ok_cupid_df[ok_cupid_df['orientation'] == 'straight']

non_straight_df = ok_cupid_df[ok_cupid_df['orientation'].isin(['bisexual', 'gay'])]
non_straight_df.head(2)

ok_cupid_df['is_straight'] = np.where(ok_cupid_df['orientation'] == 'straight', 1, 0)
ok_cupid_df['is_not_straight'] = np.where(ok_cupid_df['orientation'] != 'straight', 1, 0)

ok_cupid_df.head(2)

# relgns =ok_cupid_df['religion_binary'].unique()

# fig = go.Figure([go.Bar(x=relgns, y=ok_cupid_df["religion_binary"].value_counts().values, color=ok_cupid_df['is_straight'])])
# fig.show()

religious_views = ['religious', 'unknown_religion', 'agnostic', 'atheist']
fig = go.Figure(data=[
    go.Bar(name='Straight', 
           x=religious_views, 
           y=straight_df['religion'].value_counts().values, 
           text=straight_df['religion'].value_counts().values),
    go.Bar(name='Non-straight', 
           x=religious_views, 
           y=non_straight_df['religion'].value_counts().values,
           text=non_straight_df['religion'].value_counts().values)
])

fig.update_layout(barmode='group',
    title='Religions',
    #xaxis_tickfont_size=14,
    yaxis=dict(title='Counts',
               titlefont_size=16,
               tickfont_size=14),
    xaxis=dict(title='Religions',
               titlefont_size=16,
               tickfont_size=14))
fig.update_traces(texttemplate='%{text:.2s}', textposition='outside')
fig.show()

ok_cupid_df = ok_cupid_df.drop(columns=['is_not_straight', 'religion_binary'])

ok_cupid_df2.columns

ok_cupid_df2.columns

fig = go.Figure()

fig.add_trace(go.Histogram(x=male_df['age'], name='Male'))
fig.add_trace(go.Histogram(x=female_df['age'], name='Female'))

fig.update_layout(barmode='overlay',
                  title='Distribution of Age for Males and Females',
                  #xaxis_tickfont_size=14,
                  yaxis=dict(title='Count',
                             titlefont_size=16,
                             tickfont_size=14),
                  xaxis=dict(title='Age',
                             titlefont_size=16,
                             tickfont_size=14))
# Reduce opacity to see both histograms
fig.update_traces(opacity=0.75)
fig.show()

fig = go.Figure()

fig.add_trace(go.Histogram(x=male_df['height'], name='Male'))
fig.add_trace(go.Histogram(x=female_df['height'], name='Female'))

fig.update_layout(barmode='overlay',
                  title='Distribution of Heights for Males and Females',
                  #xaxis_tickfont_size=14,
                  yaxis=dict(title='Count',
                             titlefont_size=16,
                             tickfont_size=14),
                  xaxis=dict(title='Height',
                             titlefont_size=16,
                             tickfont_size=14))
# Reduce opacity to see both histograms
fig.update_traces(opacity=0.75)
fig.show()

orientations = ['straight', 'bisexual', 'gay']
fig = go.Figure(data=[
    go.Bar(name='Male', 
           x=orientations, 
           y=male_df['orientation'].value_counts().values, 
           text=male_df['orientation'].value_counts().values),
    go.Bar(name='Female', 
           x=orientations, 
           y=female_df['orientation'].value_counts().values,
           text=female_df['orientation'].value_counts().values)
])

fig.update_layout(barmode='group',
    title='Orientations',
    #xaxis_tickfont_size=14,
    yaxis=dict(title='Counts',
               titlefont_size=16,
               tickfont_size=14),
    xaxis=dict(title='Orientation',
               titlefont_size=16,
               tickfont_size=14))
fig.update_traces(texttemplate='%{text:.2s}', textposition='outside')
fig.show()

orientations = ['average', 'average', 'curvy', 'skinny', 'rather not say', 'overweight', 'jacked', 'used up']
fig = go.Figure(data=[
    go.Bar(name='Male', 
           y=orientations, 
           x=male_df['body_type'].value_counts().values, 
           text=male_df['body_type'].value_counts().values, orientation='h'),
    go.Bar(name='Female', 
           y=orientations, 
           x=female_df['body_type'].value_counts().values,
           text=female_df['body_type'].value_counts().values, orientation='h')
])

fig.update_layout(barmode='group',
    title='Body Types for Male and Female',
    #xaxis_tickfont_size=14,
    yaxis=dict(title='Body Types',
               titlefont_size=16,
               tickfont_size=14),
    xaxis=dict(title='Counts',
               titlefont_size=16,
               tickfont_size=14))
fig.update_traces(texttemplate='%{text:.2s}', textposition='outside')
fig.show()

orientations = ['single', 'available', 'seeing someone', 'married', 'unknown_status']
fig = go.Figure(data=[
    go.Bar(name='Male', 
           x=orientations, 
           y=male_df['status'].value_counts().values, 
           text=male_df['status'].value_counts().values),
    go.Bar(name='Female', 
           x=orientations, 
           y=female_df['status'].value_counts().values,
           text=female_df['status'].value_counts().values)
])

fig.update_layout(barmode='group',
    title='Status',
    #xaxis_tickfont_size=14,
    yaxis=dict(title='Counts',
               titlefont_size=16,
               tickfont_size=14),
    xaxis=dict(title='Status',
               titlefont_size=16,
               tickfont_size=14))
fig.update_traces(texttemplate='%{text:.2s}', textposition='outside')
fig.show()

from sklearn.preprocessing import OneHotEncoder

# Instantiate OneHotEncoder
orientation_ohe = OneHotEncoder()
status_ohe = OneHotEncoder()
job_ohe = OneHotEncoder()

# Fit the OneHotEncoder and transform
orientation = pd.DataFrame(ok_cupid_df['orientation'])
orientation_encoded = orientation_ohe.fit_transform(orientation)
display(orientation_encoded)

status = pd.DataFrame(ok_cupid_df['status'])
status_encoded = status_ohe.fit_transform(status)
display(status_encoded)

job = pd.DataFrame(ok_cupid_df['job'])
job_encoded = job_ohe.fit_transform(job)
display(job_encoded)

# Put into a dataframe to get column names
encoded_df_orientation = pd.DataFrame(orientation_encoded.toarray().astype(int), columns=orientation_ohe.categories_[0], dtype=int)
encoded_df_orientation = encoded_df_orientation.drop(encoded_df_orientation.columns[0], axis=1)
display(encoded_df_orientation.head(2))

# Status
encoded_df_status = pd.DataFrame(status_encoded.toarray().astype(int), columns=status_ohe.categories_[0], dtype=int)
encoded_df_status = encoded_df_status.drop(columns='unknown_status')
display(encoded_df_status.head(2))

# Jobs
encoded_df_job = pd.DataFrame(job_encoded.toarray().astype(int), columns=job_ohe.categories_[0], dtype=int)
encoded_df_job = encoded_df_job.drop(columns='unknown_job')
display(encoded_df_job.head(2))

encoded_df_job.rename(columns= {'other': 'otherjob'}, inplace = True)

# Join back to ok_cupid_df
ok_cupid_df = pd.concat([ok_cupid_df, encoded_df_orientation], axis=1)
ok_cupid_df.drop(columns='orientation', inplace=True)
ok_cupid_df.head(2)

ok_cupid_df = pd.concat([ok_cupid_df, encoded_df_status], axis=1)
ok_cupid_df.drop(columns='status', inplace=True)
ok_cupid_df.head(2)

ok_cupid_df = pd.concat([ok_cupid_df, encoded_df_job], axis=1)
ok_cupid_df.drop(columns='job', inplace=True)
ok_cupid_df.head(2)

from sklearn.feature_extraction.text import CountVectorizer

# import the nltk stopwords
!pip install nltk
import nltk
nltk.download('stopwords')
from nltk.corpus import stopwords

# Let's test it out
stemmer = nltk.stem.PorterStemmer()

ENGLISH_STOP_WORDS = stopwords.words('english')

def my_tokenizer(sentence):
    # remove punctuation and set to lower case
    # for punctuation_mark in string.punctuation:
    #     sentence = sentence.replace(punctuation_mark,'').lower()

    # split sentence into words
    listofwords = sentence.split(' ')
    listofstemmed_words = []
    
    # remove stopwords and any tokens that are just empty strings
    for word in listofwords:
        if (not word in ENGLISH_STOP_WORDS) and (word!=''):
            # Stem words
            stemmed_word = stemmer.stem(word)
            listofstemmed_words.append(stemmed_word)

    return listofstemmed_words

# 1. Instantiate 
# essay = CountVectorizer(stop_words="english", min_df=5, max_features=100, tokenizer = lambda x: [stemmer.stem(i) for i in x.split(' ')])
essay = CountVectorizer(min_df=5, max_features=100, tokenizer = my_tokenizer)

# 2. Fit 
essay.fit(ok_cupid_df["essay"])

# 3. Transform
essay_transformed = essay.transform(ok_cupid_df["essay"])
essay_transformed

essay_df = pd.DataFrame(columns=essay.get_feature_names(), data=essay_transformed.toarray())
#essay_df = essay0_df.drop(essay_df.columns[0], axis=1)
essay_df.head(2)

# Join back to ok_cupid_df
ok_cupid_df = pd.concat([ok_cupid_df, essay_df], axis=1)
ok_cupid_df.head(2)

# 1. Instantiate 
ethnicity = CountVectorizer()

# 2. Fit 
ethnicity.fit(ok_cupid_df["ethnicity"])

# 3. Transform
ethnicity_transformed = ethnicity.transform(ok_cupid_df["ethnicity"])
ethnicity_transformed

ethnicity_transformed.toarray()

ethnicity_df = pd.DataFrame(columns=ethnicity.get_feature_names(), data=ethnicity_transformed.toarray())

unknown_ethnicity = ethnicity_df[ethnicity_df['unknown_ethnicity'] == 1]['unknown_ethnicity']
unknown_ethnicity = unknown_ethnicity.sum()

# Drop one column to prevent redundant information
ethnicity_df = ethnicity_df.drop(columns='unknown_ethnicity')

ethnicity_df = ethnicity_df.rename(columns={'american': 'native_american', 'eastern': 'middle_eastern', 'islander': 'pacific_islander', 'hispanic': 'hispanic_latin'})
ethnicity_df = ethnicity_df.drop(columns=['native', 'middle', 'pacific', 'latin'])
ethnicity_df.head(2)

ethnicity_df.rename(columns = {'other':'other_ethnicity'}, inplace = True)

# Join back to ok_cupid_df
ok_cupid_df = pd.concat([ok_cupid_df, ethnicity_df], axis=1)
ok_cupid_df.head(2)

ethnicity_df['native_american'].value_counts()[1]

native_american = ethnicity_df[ethnicity_df['native_american'] == 1]['native_american']
native_american = native_american.sum()

asian = ethnicity_df[ethnicity_df['asian'] == 1]['asian']
asian = asian.sum()

black = ethnicity_df[ethnicity_df['black'] == 1]['black']
black = black.sum()

middle_eastern = ethnicity_df[ethnicity_df['middle_eastern'] == 1]['middle_eastern']
middle_eastern = middle_eastern.sum()

hispanic_latin = ethnicity_df[ethnicity_df['hispanic_latin'] == 1]['hispanic_latin']
hispanic_latin = hispanic_latin.sum()

indian = ethnicity_df[ethnicity_df['indian'] == 1]['indian']
indian = indian.sum()

pacific_islander = ethnicity_df[ethnicity_df['pacific_islander'] == 1]['pacific_islander']
pacific_islander = pacific_islander.sum()

other_ethnicity = ethnicity_df[ethnicity_df['other_ethnicity'] == 1]['other_ethnicity']
other_ethnicity = other_ethnicity.sum()

white = ethnicity_df[ethnicity_df['white'] == 1]['white']
white = white.sum()

# intialise data of lists.
sum_ethnicities = {'ethnicity':['native american', 'asian', 'black',
                                'middle eastern', 'hispanic/latin',
                                'indian', 'pacific islander',
                                'other_ethnicity', 'white', 'unknown_ethnicity'],
        'sum':[native_american, asian, black, middle_eastern,
               hispanic_latin, indian, pacific_islander, other_ethnicity, 
               white, unknown_ethnicity]}
  
# Create DataFrame
ethnicities_sum = pd.DataFrame(sum_ethnicities)

ethnicities_sum['sum'].unique()

#fig = px.bar(ethnicities_sum, x='ethnicity', y='sum')
#fig.show()

labels = ethnicities_sum['ethnicity'].unique()
values = ethnicities_sum['sum'].unique()
fig = go.Figure(data=[go.Pie(labels=labels, 
                             values=values, 
                             hole=.5)])
fig.update_layout(title='Ethinicities')
fig.show()

ok_cupid_df.drop(columns='ethnicity', inplace=True)

ok_cupid_df['body_type'].value_counts()

ok_cupid_df['body_type'] = ok_cupid_df['body_type'].replace(['a little extra', 'full figured'],'curvy')
ok_cupid_df['body_type'] = ok_cupid_df['body_type'].replace(['thin'],'skinny')
ok_cupid_df['body_type'] = ok_cupid_df['body_type'].replace(['fit'],'athletic')
ok_cupid_df['body_type'] = ok_cupid_df['body_type'].replace(['unknown_body_type'],'rather not say')

ok_cupid_df['body_type'].value_counts()

body_type_mapping = {'overweight':0, 'curvy':1, 'average':2, 'used up':3, 'rather not say': 4, 'skinny': 5, 'athletic': 6, 'jacked': 7}
body_type_mapped_data = ok_cupid_df['body_type'].map(body_type_mapping)

#ok_cupid_df = ok_cupid_df.drop(columns=['body_type'])
ok_cupid_df.drop(columns='body_type', inplace=True)

ok_cupid_df = pd.concat([ok_cupid_df, body_type_mapped_data], axis=1)
ok_cupid_df.head(2)

ok_cupid_df['drinks'].value_counts()

drinks_mapping = {'desperately':6, 'very often':5, 'often':4, 'unknown_drinks':3, 'socially': 2, 'rarely': 1, 'not at all': 0}
drinks_mapped_data = ok_cupid_df['drinks'].map(drinks_mapping)

ok_cupid_df = ok_cupid_df.drop(columns=['drinks'])
ok_cupid_df = pd.concat([ok_cupid_df, drinks_mapped_data], axis=1)
ok_cupid_df.head(2)

ok_cupid_df['diet'].value_counts()

#ok_cupid_df["diet"] = ok_cupid_df["diet"].replace({'anything': 'strictly anything', 'vegetarian': 'strictly vegetarian', 'other': 'strictly other',
 #                            'vegan': 'strictly vegan', 'kosher': 'strictly kosher', 'halal': 'strictly halal'})
ok_cupid_df['diet'] = ok_cupid_df['diet'].str.replace(' ', '')
ok_cupid_df['diet'].value_counts()

ok_cupid_df['diet'] = ok_cupid_df['diet'].str.replace('other', 'otherdiet')
#ok_cupid_df['diet'] = ok_cupid_df['diet'].str.replace(' ', '')
ok_cupid_df['diet'].value_counts()

diet = CountVectorizer()

diet_transformed = diet.fit_transform(ok_cupid_df["diet"])
diet_transformed

diet_df = pd.DataFrame(columns=diet.get_feature_names(), data=diet_transformed.toarray())

# Drop one column to prevent redundant information
diet_df = diet_df.drop(columns=['unknowndiet'])
diet_df.head(2)

diff_diets = diet_df.columns.tolist()

for d in diff_diets:
    if d.startswith('mostly'):
        diff_diets.remove(d)

for d in diff_diets:
    if d.startswith('strictly'):
        diff_diets.remove(d)

diets = diff_diets

diets_mostly = []
for diet in diets:
    diets_mostly.append('mostly'+diet)
    
diets_strictly = []
for diet in diets:
    diets_strictly.append('strictly'+diet)

for diet in diets:
    if diet in diet_df.columns:
        diet_df.loc[diet_df[diet] == 1, diet] = 2
        
for diet, diet_mostly in zip(diets, diets_mostly):
    if diet_mostly in diet_df.columns:
        diet_df.loc[diet_df[diet_mostly] == 1, diet] = 1
        diet_df = diet_df.drop(columns=diet_mostly)

for diet, diet_strictly in zip(diets, diets_strictly):
    if diet_strictly in diet_df.columns:
        diet_df.loc[diet_df[diet_strictly] == 1, diet] = 3
        diet_df = diet_df.drop(columns=diet_strictly)

diet_df.head(2)

diet_df.rename(columns = {'diet':'otherdiet'}, inplace = True)

# Join back to ok_cupid_df
ok_cupid_df = pd.concat([ok_cupid_df, diet_df], axis=1)
ok_cupid_df.drop(columns='diet', inplace=True)
ok_cupid_df.head(2)

ok_cupid_df['drugs'].value_counts()

drugs_mapping = {'often':3, 'sometimes':2, 'unknown_drugs':1, 'never':0}
drugs_mapped_data = ok_cupid_df['drugs'].map(drugs_mapping)
drugs_mapped_data = drugs_mapped_data.astype(int)

ok_cupid_df = ok_cupid_df.drop(columns=['drugs'])
ok_cupid_df = pd.concat([ok_cupid_df, drugs_mapped_data], axis=1)
ok_cupid_df.head(2)

ok_cupid_df['education'].value_counts()

education_mapping = {'dropped out of space camp':0, 'working on space camp':1, 'graduated from space camp':2,
                     'space camp': 2,
                     'dropped out of high school':3, 'working on high school': 4, 'graduated from high school': 5,
                     'high school': 5,
                     'dropped out of two-year college': 6, 'working on two-year college': 7,
                     'two-year college': 8,
                     'graduated from two-year college': 8, 'dropped out of college/university': 9,
                     'college/university': 11,
                     'working on college/university': 10, 'graduated from college/university': 11, 'unknown_education': 12,
                     'dropped out of masters program': 13, 'working on masters program': 14,
                     'masters program': 15,
                     'graduated from masters program': 15, 'dropped out of law school': 16,
                     'working on law school': 17, 'graduated from law school': 18,
                     'law school': 18,
                     'dropped out of ph.d program': 16, 'working on ph.d program': 17,
                     'ph.d program': 18,
                     'graduated from ph.d program': 18, 'dropped out of med school': 16, 
                     'working on med school': 17, 'graduated from med school': 18, 'med school': 18}
education_mapped_data = ok_cupid_df['education'].map(education_mapping)
education_mapped_data = education_mapped_data.astype(int)

ok_cupid_df = ok_cupid_df.drop(columns=['education'])
ok_cupid_df = pd.concat([ok_cupid_df, education_mapped_data], axis=1)
ok_cupid_df.head(2)

ok_cupid_df['smokes'].value_counts()

from sklearn.preprocessing import LabelEncoder
from sklearn.preprocessing import OrdinalEncoder # This is used for multiple columns

# Instantiate the label encoder
le = LabelEncoder()

# Fit and transform the order priority column
le.fit_transform(ok_cupid_df['smokes'])

le.classes_

smokes_mapping = {'yes':4, 'sometimes':3, 'when drinking':3, 'unknown_smokes':2, 'trying to quit':1, 'no': 0}
smokes_mapped_data = ok_cupid_df['smokes'].map(smokes_mapping)
smokes_mapped_data = smokes_mapped_data.astype(int)

ok_cupid_df = ok_cupid_df.drop(columns=['smokes'])
ok_cupid_df = pd.concat([ok_cupid_df, smokes_mapped_data], axis=1)
ok_cupid_df.head(2)

ok_cupid_df['location'].str.split(',', expand=True).head(2)

ok_cupid_df['location'].str.split(',', expand=True)[1].unique()

ok_cupid_df['city'] = ok_cupid_df['location'].str.split(',', expand=True)[0]

ok_cupid_df_copy_city = ok_cupid_df.copy()

city_data = {'city':ok_cupid_df_copy_city['city'].unique()} 
    
# Convert the dictionary into DataFrame 
cities_df = pd.DataFrame(city_data) 

from geopy.exc import GeocoderTimedOut
from geopy.geocoders import Nominatim

longitude = []
latitude = []
def findGeocode(city):
    try:
        geolocator = Nominatim(user_agent="your_app_name")
        return geolocator.geocode(city)
    except GeocoderTimedOut:
        return findGeocode(city)	

for i in (cities_df["city"]):
    if findGeocode(i) != None:
        loc = findGeocode(i)
        latitude.append(loc.latitude)
        longitude.append(loc.longitude)
    
    else:
        latitude.append(np.nan)
        longitude.append(np.nan)

cities_df["longitude"] = longitude
cities_df["latitude"] = latitude
  
cities_df.head(2)

#cities_df.rename(columns = {'City':'city'}, inplace = True)
df_cities = ok_cupid_df_copy_city['city'].value_counts().rename_axis('city').reset_index(name='counts')
df_cities.head(2)

df_cities = pd.merge(cities_df, df_cities, on='city', how='inner')

df_cities.head(2)

fig = px.scatter_mapbox(df_cities, lat="latitude", lon="longitude", 
                        size='counts',
                        color='city',
                        color_continuous_scale=px.colors.cyclical.IceFire, 
                        #size_max=15, zoom=10,
                        mapbox_style="carto-positron"
                       )
fig.show()

ok_cupid_df['state'] = ok_cupid_df['location'].str.split(',', expand=True)[1]
ok_cupid_df['state'] = ok_cupid_df['state'].str.lstrip(' ')
ok_cupid_df['state'].unique()

ok_cupid_df['country'] = 'united states'

ok_cupid_df.loc[ok_cupid_df['state'] == 'spain', 'state'] = 'madrid'
ok_cupid_df.loc[ok_cupid_df['city'] == 'madrid', 'country'] = 'spain'

ok_cupid_df.loc[ok_cupid_df['state'] == 'vietnam', 'state'] = 'khanh hoa'
ok_cupid_df.loc[ok_cupid_df['city'] == 'nha trang', 'country'] = 'vietnam'

ok_cupid_df.loc[ok_cupid_df['state'] == 'ireland', 'state'] = 'munster'
ok_cupid_df.loc[ok_cupid_df['city'] == 'cork', 'country'] = 'ireland'

ok_cupid_df.loc[ok_cupid_df['city'] == 'edinburgh', 'state'] = 'scotland'
ok_cupid_df.loc[ok_cupid_df['city'] == 'edinburgh', 'country'] = 'united kingdom'

ok_cupid_df.loc[ok_cupid_df['city'] == 'london', 'state'] = 'england'
ok_cupid_df.loc[ok_cupid_df['city'] == 'london', 'country'] = 'united kingdom'

ok_cupid_df.loc[ok_cupid_df['state'] == 'germany', 'state'] = 'hessen'
ok_cupid_df.loc[ok_cupid_df['city'] == 'kassel', 'country'] = 'germany'

ok_cupid_df.loc[ok_cupid_df['state'] == 'netherlands', 'state'] = 'north holland'
ok_cupid_df.loc[ok_cupid_df['city'] == 'amsterdam', 'country'] = 'netherlands'

ok_cupid_df.loc[ok_cupid_df['state'] == 'switzerland', 'state'] = 'graubunden'
ok_cupid_df.loc[ok_cupid_df['city'] == 'bonaduz', 'country'] = 'switzerland'

ok_cupid_df.loc[ok_cupid_df['city'] == 'vancouver', 'country'] = 'canada'

ok_cupid_df = ok_cupid_df.drop(columns=['location'])