R, Python 분석과 프로그래밍의 친구 (by R Friend)

# importing libraries

import numpy as np

import pandas as pd

import matplotlib.pyplot as plt

import seaborn as sns

plt.rcParams['figure.figsize'] = [10, 8] # setting figure size

# loading 'iris' dataset from seaborn

iris = sns.load_dataset('iris')

iris.shape

(150, 5)

iris.head()

iris.groupby('species').size()

species setosa 50 versicolor 50 virginica 50 dtype: int64

  (1) matplotlib 으로 그룹별 산점도 그리기 (scatter plot by groups via matplotlib)

# Scatter plot with a different color by groups

groups = iris.groupby('species')

fig, ax = plt.subplots()

for name, group in groups:

    ax.plot(group.petal_length, 

            group.petal_width, 

            marker='o', 

            linestyle='',

            label=name)

ax.legend(fontsize=12, loc='upper left') # legend position

plt.title('Scatter Plot of iris by matplotlib', fontsize=20)

plt.xlabel('Petal Length', fontsize=14)

plt.ylabel('Petal Width', fontsize=14)

plt.show()

  (2) seaborn 으로 그룹별 산점도 그리기 (scatter plot by groups via seaborn)

# Scatter plot by Groups

sns.scatterplot(x='petal_length', 

                y='petal_width', 

                hue='species', # different colors by group

                style='species', # different shapes by group

                s=100, # marker size

                data=iris)

plt.show()

# error: AttributeError: module 'seaborn' has no attribute 'scatterplot' 

# solution: upgrade seaborn

pip install --upgrade seaborn

  (3) pandas로 그룹별 산점도 그리기 (scatter plot by groups via pandas)

# adding 'color' column

iris['color'] = np.where(iris.species == 'setosa', 'red', 

                         np.where(iris.species =='versicolor', 

                         'green', 

                         'blue'))

# scatter plot

iris.plot(kind='scatter',

          x='petal_length', 

          y='petal_width', 

          s=50, # marker size

          c=iris['color']) # marker color by group

plt.title('Scatter Plot of iris by pandas', fontsize=20)

plt.xlabel('Petal Length', fontsize=14)

plt.ylabel('Petal Width', fontsize=14)

plt.show()

# importing libraries

import numpy as np

import pandas as pd

import matplotlib.pyplot as plt

import seaborn as sns

plt.rcParams['figure.figsize'] = [12, 8] # setting figure size

# loading 'iris' dataset from seaborn

iris = sns.load_dataset('iris')

iris.shape

(150, 5) 

iris.head()

  (1) matplotlib을 사용한 산점도 (scatter plot by matplotlib)

# Basic Scatter Plot

plt.plot('petal_length',  # x

         'petal_width',  # y

         data=iris, 

         linestyle='none', 

         marker='o', 

         markersize=10,

         color='blue', 

         alpha=0.5)

plt.title('Scatter Plot of iris by matplotlib', fontsize=20)

plt.xlabel('Petal Length', fontsize=14)

plt.ylabel('Petal Width', fontsize=14)

plt.show()

# adding a rectangle, a circle

import matplotlib.patches as patches

import matplotlib.pyplot as plt

fig1 = plt.figure()

ax1 = fig1.add_subplot(111)

# (0) scatter plot

ax1.plot('petal_length', 'petal_width', data=iris, linestyle='none', marker='o')

# (1) adding a rectangle

ax1.add_patch(

    patches.Rectangle(

        (3, 1), # (x, y)

        2, # width

        1, # height

        alpha=0.2, 

        facecolor="blue", 

        edgecolor="black", 

        linewidth=2, 

        linestyle="solid", 

        angle=-10))

# (2) adding a circle

ax1.add_patch(

    patches.Circle(

        (1.5, 0.25), # (x, y)

        0.5, # radius

        alpha=0.2, 

        facecolor="red", 

        edgecolor="black", 

        linewidth=2, 

        linestyle='solid'))

# (3) adding a line

plt.plot([4, 6], [2.2, 1.1], color="green", lw=4, linestyle='solid')

plt.title("Adding a Rectangle, a Circle and a Line", fontsize=20)

plt.xlabel('Petal Length', fontsize=14)

plt.ylabel('Petal Width', fontsize=14)

plt.show()

  (2) seaborn을 사용한 산점도 (scatter plot by seaborn)

# Basic Scatter Plot by seaborn

sns.regplot(x=iris['petal_length'], 

           y=iris['petal_width'], 

           fit_reg=False) # no regression line

plt.title('Scatter Plot of iris by regplot()', fontsize=20)

plt.xlabel('Petal Length', fontsize=14)

plt.ylabel('Petal Width', fontsize=14)

plt.show()

# Scatter Plot with regression line by seaborn regplot()

sns.regplot(x=iris['petal_length'], 

           y=iris['petal_width'], 

           fit_reg=True) # default

plt.title('Scatter Plot with Regression Line by regplot()', fontsize=20)

plt.show()

# Control color of each marker based on X and Y values

iris_df = iris.copy()

# Adding a 'color' column based on x and y values

cutoff = (iris_df['petal_length']>2.5) & (iris_df['petal_width'] > 0.8)

iris_df['color'] = np.where(cutoff==True, "red", "blue")

# Scatter Plot with different colors based on X and Y values

sns.regplot(x=iris['petal_length'], 

           y=iris['petal_width'], 

           fit_reg=False, 

           scatter_kws={'facecolors': iris_df['color']}) # marker color

plt.title('Scatter Plot with different colors by X & Y values', fontsize=20)

plt.show()

# scatter plot by seaborn scatterplot()

ax = sns.scatterplot(x='petal_length', 

                     y='petal_width', 

                     alpha=0.5,

                     data=iris)

plt.title('Scatter Plot by seaborn', fontsize=20)

plt.show()

  (3) pandas를 사용한 산점도 (scatter plot by pandas)

iris.plot.scatter(x='petal_length', 

                  y='petal_width', 

                  s=50, # marker size

                  c='blue', 

                  alpha=0.5)

plt.title('Scatter Plot of iris by pandas', fontsize=20)

plt.xlabel('Petal Length', fontsize=14)

plt.ylabel('Petal Width', fontsize=14)

plt.show()

  (1) 원 그래프 (Pie Chart)

# importing library and set figure size

import matplotlib.pyplot as plt

plt.rcParams['figure.figsize'] = [12, 8]

# information of groups

group_names = ['Group_A', 'Group_B', 'Group_C']

group_sizes = [95, 54, 25]

group_colors = ['yellowgreen', 'lightskyblue', 'lightcoral']

group_explodes = (0.1, 0, 0) # explode 1st slice

# Pie chart

plt.pie(group_sizes, 

        explode=group_explodes, 

        labels=group_names, 

        colors=group_colors, 

        autopct='%1.2f%%', # second decimal place

        shadow=True, 

        startangle=90,

        textprops={'fontsize': 14}) # text font size

plt.axis('equal') #  equal length of X and Y axis

plt.title('Pie Chart of Market Share', fontsize=20)

plt.show()

  (2) 하위 그룹을 포함한 도넛 그래프 (Donut Chart with Subgraphs)

# importing library and set figure size

import matplotlib.pyplot as plt

plt.rcParams['figure.figsize'] = [12, 8]

 # info. of groups

group_names = ['Group_A', 'Group_B', 'Group_C']

group_sizes = [95, 54, 25]

# info. of subgroups

subgroup_names = ['A_1', 'A_2', 'A_3', 'A_4', 

                  'B_1', 'B_2', 'B_3', 

                  'C_1', 'C_2']

subgroup_sizes = [50, 30, 10, 5, 30, 20, 4, 20, 5]

# colors

a, b, c = [plt.cm.Reds, plt.cm.Greens, plt.cm.Blues]

# width

width_num = 0.4

# Outside Ring

fig, ax = plt.subplots()

ax.axis('equal')

pie_outside, _ = ax.pie(group_sizes, 

                        radius=1.3, 

                        labels=group_names, 

                        labeldistance=0.8,

                        colors=[a(0.6), b(0.6), c(0.6)])

plt.setp(pie_outside, 

         width=width_num, 

         edgecolor='white')

# Inside Ring

pie_inside, plt_labels, junk = \

    ax.pie(subgroup_sizes, 

           radius=(1.3 - width_num), 

           labels=subgroup_names, 

           labeldistance=0.75, 

           autopct='%1.1f%%', 

           colors=[a(0.5), a(0.4), a(0.3), a(0.2), 

                   b(0.5), b(0.4), b(0.3), 

                   c(0.5), c(0.4)])

plt.setp(pie_inside, 

         width=width_num, 

         edgecolor='white')

plt.title('Donut Plot with Subgroups', fontsize=20)

plt.show()

# importing packages

import numpy as np

import pandas as pd

import matplotlib.pyplot as plt

import seaborn as sns

plt.rcParams['figure.figsize'] = [10, 6]

# Loading 'tips' dataset from seaborn

tips = sns.load_dataset('tips')

tips.shape

(244, 7)

tips.head()

  (1) matplotlib으로 막대 그리프 그리기 (bar chart by matplotlib)

# Summary Statistics

tips_sum_by_day = tips.groupby('day').tip.sum()

tips_sum_by_day

day
Thur    171.83
Fri      51.96
Sat     260.40
Sun     247.39 

Name: tip, dtype: float64

label = ['Thur', 'Fri', 'Sat', 'Sun']

index = np.arange(len(label))

# Basic Bar Chart

plt.bar(index, tips_sum_by_day)

plt.title('Sum of Tips by Day', fontsize=20)

plt.xlabel('Day', fontsize=18)

plt.ylabel('Sum of Tips', fontsize=18)

plt.xticks(index, label, fontsize=15)

plt.show()

# bar color, transparency

plt.bar(label, tips_sum_by_day, 

        color='red', # color

        alpha=0.5) # transparency

plt.show()

# bar width, align

plt.bar(label, tips_sum_by_day, 

        width=0.5, # default: 0.8

        align='edge') # default: 'center'

plt.show()

# X tick labels rotation

plt.bar(index, tips_sum_by_day)

plt.xlabel('Day', fontsize=18)

plt.xticks(index, label, fontsize=15, 

           rotation=90) # when X tick labels are long

plt.show()

# Horizontal Bar Chart

plt.barh(index, tips_sum_by_day)

plt.title('Sum of Tips by Day', fontsize=18)

plt.ylabel('Day', fontsize=15)

plt.xlabel('Sum of Tips', fontsize=15)

plt.yticks(index, label, fontsize=13, rotation=0)

plt.show()

# summary by group

tips_sum_by_day_male = tips[tips['sex'] == 'Male'].groupby('day').tip.sum()

tips_sum_by_day_female = tips[tips['sex'] == 'Female'].groupby('day').tip.sum()

print('--Male--')

print(tips_sum_by_day_male);

print(' ')

print('--Female--')

print(tips_sum_by_day_female);

--Male--

day
Thur     89.41
Fri      26.93
Sat     181.95
Sun     186.78
Name: tip, dtype: float64
 
--Female--
day
Thur    82.42
Fri     25.03
Sat     78.45
Sun     60.61
Name: tip, dtype: float64

# Bar Chart by 2 categorical variables

# Stacked Bar Chart

label = ['Thur', 'Fri', 'Sat', 'Sun']

N = len(tips['day'].unique())

index = np.arange(N)

alpha = 0.5

p1 = plt.bar(index, tips_sum_by_day_male, color='b', alpha=alpha)

p2 = plt.bar(index, tips_sum_by_day_female, color='r', alpha=alpha,

             bottom=tips_sum_by_day_male) # stacked bar chart

plt.title('Stacked Bar Chart of Sum of Tips by Day & Sex', fontsize=20)

plt.ylabel('Sum of Tips', fontsize=18)

plt.xlabel('Day', fontsize=18)

plt.xticks(index, label, fontsize=15)

plt.legend((p1[0], p2[0]), ('Male', 'Female'), fontsize=15)

plt.show()

# Dodged Bar Chart (with same X coordinates side by side)

bar_width = 0.35

alpha = 0.5

p1 = plt.bar(index, tips_sum_by_day_male, 

             bar_width, 

             color='b', 

             alpha=alpha,

             label='Male')

p2 = plt.bar(index + bar_width, tips_sum_by_day_female, 

             bar_width, 

             color='r', 

             alpha=alpha,

             label='Female')

plt.title('Dodged Bar Chart of Sum of Tips by Day & Sex', fontsize=20)

plt.ylabel('Sum of Tips', fontsize=18)

plt.xlabel('Day', fontsize=18)

plt.xticks(index, label, fontsize=15)

plt.legend((p1[0], p2[0]), ('Male', 'Female'), fontsize=15)

plt.show()

  (2) seaborn 으로 막대 그래프 그리기 (bar chart by seaborn)

tips_sum_by_day_sex = pd.DataFrame(tips.groupby(['day', 'sex']).tip.sum())

tips_sum_by_day_sex = tips_sum_by_day_sex.reset_index()

tips_sum_by_day_sex

# sort dataframe by 'day' and 'sex' in descending order first

tips_sum_by_day_sex = tips_sum_by_day_sex.sort_values(by=['day', 'sex'], ascending=False)

tips_sum_by_day_sex

# then, calculate cumulative summation by 'day' group

tips_sum_by_day_sex['tip_cumsum'] = tips_sum_by_day_sex.groupby(['day'])['tip'].cumsum(axis=0)

tips_sum_by_day_sex

# atfer that, sort agian by 'day' and 'sex' in ascending order

tips_sum_by_day_sex = tips_sum_by_day_sex.sort_values(by=['day', 'sex'], ascending=True)

tips_sum_by_day_sex

# Stacked Bar Chart

sns.barplot(x='day', y='tip_cumsum', hue='sex', data=tips_sum_by_day_sex, 

           dodge=False) # stacked bar chart

plt.title('Stacked Bar Chart by Seaborn', fontsize='20')

plt.show()

# Dodged Bar Chart

sns.barplot(x='day', y='tip', hue='sex', data=tips_sum_by_day_sex) # default : dodge=True

plt.title('Dodged Bar Chart by Seaborn', fontsize=20)

plt.legend(fontsize=12)

plt.show()

  (3) pandas 로 막대 그래프 그리기 (bar chart by pandas)

# make a DataFrame

tips_sum_by_day = pd.DataFrame(tips.groupby('day').tip.sum())

tips_sum_by_day = tips_sum_by_day.reset_index()

# basic bar chart with a single column

tips_sum_by_day.plot.bar(x='day', y='tip', rot=0)

plt.show()

# making a DataFrame with 'day' and 'sex' groups

tips_sum_by_day_sex = pd.DataFrame(tips.groupby(['day', 'sex']).tip.sum())

tips_sum_by_day_sex = tips_sum_by_day_sex.reset_index()

# pivot

tips_sum_by_day_sex_pivot = \

    tips_sum_by_day_sex.pivot(index='day', 

                              columns='sex', 

                              values='tip')

tips_sum_by_day_sex_pivot

# Stacked Bar Chart by pandas

tips_sum_by_day_sex_pivot.plot.bar(stacked=True, rot=0)

plt.title('Stacked Bar Chart by Pandas', fontsize=20)

plt.show()

# Dodged Bar Chart by pandas

tips_sum_by_day_sex_pivot.plot(kind='bar', rot=0)

plt.title('Dodged Bar Chart by Pandas', fontsize=20)

plt.legend(fontsize='12')

plt.show()

[상자 그림 그리는 순서 및 방법]

1. 주어진 데이터에서 각 사분위수를 계산한다.
2. 그래프에서 제1 사분위와 제3 사분위를 밑변으로 하는 직사각형을 그리고, 제 2사분위에 해당하는 위치에 선분을 긋는다.
3. 사분위수 범위(IQR, Interquartile range, )를 계산한다.
4. 과 차이가 1.5*IQR 이내인 값 중에서 최댓값을 과 직선으로 연결하고, 마찬가지로 과 차이가 1.5*IQR 이내인 값 중에서 최솟값을 과 연결한다.
5. 보다 1.5*IQR 이상 초과하는 값과 보다 1.5*IQR 이상 미달하는 값은 점이나, 원, 별표등으로 따로 표시한다(이상치 점).

위의 상자 그림 그리는 방법에 대한 내용은 아래의 표준정규분포 확률밀도함수 분포 곡선에 대한 상자 그림 매핑을 보면 좀더 이해가 쉬울 것입니다. 

[Boxplot and a probability density function (pdf) of a Normal N(0,1σ²) Population]

* source: https://en.wikipedia.org/wiki/Box_plot

import numpy as np

import pandas as pd

import matplotlib.pyplot as plt

import seaborn as sns

plt.rcParams['figure.figsize'] = [10, 6]

%matplotlib inline

# loading 'tips' dataset

tips = sns.load_dataset('tips')

tips.shape

(244, 7)

tips.head()

tips.groupby(['sex', 'day']).size()

sex     day 
Male    Thur    30
        Fri     10
        Sat     59
        Sun     58
Female  Thur    32
        Fri      9
        Sat     28
        Sun     18
dtype: int64

  (1) matplotlib 으로 상자 그림 그리기

# Basic box plot

plt.boxplot(tips['tip'])

plt.show()

# setting outlier symbol, title, xlabel

plt.boxplot(tips['tip'], sym="bo")

plt.title('Box plot of tip')

plt.xticks([1], ['tip'])

plt.show()

# Horizontal Box plot with notched box & red color outliers

plt.boxplot(tips['tip'], 

            notch=1, # if 'True' then notched box plot

            sym='rs', # symbol: red square

            vert=0 # vertical : if 'False' then horizontal box plot

           )

plt.show()

# Multiple box plots on one Axes

fig, ax = plt.subplots()

ax.boxplot([tips['total_bill'], tips['tip']], sym="b*")

plt.title('Multiple box plots of tips on one Axes')

plt.xticks([1, 2], 

           ['total_bill', 'tip'])

plt.show()

  (2) seaborn 으로 여러개 그룹에 대한 상자 그림 그리기 

      (Grouped box plots by seaborn)

# Grouped boxplots by seaborn

import seaborn as sns

sns.boxplot(x="day", 

            y="tip", 

            data=tips)

plt.show()

#  2 by 4 multi-Grouped boxplots by seaborn

sns.boxplot(x="day", 

            y="tip", 

            hue="sex",

            data=tips)

plt.show()

  (3) pandas 로 상자 그림 그리기

# Grouped boxplots by pandas

tips.boxplot(column=["tip"], by=["day", "sex"])

plt.show()

import numpy as np

import pandas as pd

import matplotlib.pyplot as plt

import seaborn as sns

# loading 'iris' dataset

iris = sns.load_dataset('iris')

iris.shape

(150, 5)

iris.head()

iris.groupby('species').size()

species
setosa        50
versicolor    50
virginica     50
dtype: int64

  (1) 여러개 그룹의 히스토그램, 커널밀도곡선 그리기

# 1-1. Multiple histograms on the same axis

sns.distplot(iris[iris.species == "setosa"]["petal_length"], 

             color="blue", label="setosa")

sns.distplot(iris[iris.species == "versicolor"]["petal_length"], 

             color="red", label="versicolor")

sns.distplot(iris[iris.species == "virginica"]["petal_length"], 

             color="green", label="virginica")

plt.legend(title="Species")

plt.show()

# 1-2. Via for loop

grp_col_dict = {'setosa': 'blue', 

                    'versicolor': 'red', 

                    'virginica': 'green'}

# for loop of species group

for group in grp_col_dict:

    # subset of group

    subset = iris[iris['species'] == group]

    # histogram and kernel density curve

    sns.distplot(subset['petal_length'], 

                    hist = True, # histogram

                    kde = True,  # density curve

                    kde_kws = {'linewidth': 2}, 

                    color = grp_col_dict[group],

                    label = group)

# setting plot format

plt.title('Histogram & Density Plot by Groups')

plt.xlabel('Petal Length(unit:cm)')

plt.ylabel('Density')

plt.legend(prop={'size': 12}, title = 'Group')

plt.show()

  (2) 여러개 변수의 히스토그램, 커널밀도곡선 그리기

# 2-1. Multiple histograms on the same axis

var_color_dict = {'sepal_length': 'blue', 

                      'sepal_width': 'red', 

                      'petal_length': 'yellow', 

                      'petal_width': 'green'}

# for loop

for var in var_color_dict:

    sns.distplot(iris[var],                  

                    color = var_color_dict[var], 

                    hist_kws = {'edgecolor': 'gray'}, 

                    label = var)

plt.legend(title = 'Variables')

plt.show()

# 2-2. Multiple histograms at separate windows

f, axes = plt.subplots(2, 2, figsize=(8, 6), sharex=True)

sns.distplot(iris["sepal_length"], color="blue", ax=axes[0, 0])

sns.distplot(iris["sepal_width"], color="red", ax=axes[0, 1])

sns.distplot(iris["petal_length"], color="yellow", ax=axes[1, 0])

sns.distplot(iris["petal_width"], color="green", ax=axes[1, 1])

plt.show()

var_color_dict = {'sepal_length': 'blue', 

                      'sepal_width': 'red', 

                      'petal_length': 'yellow', 

                      'petal_width': 'green'}

i = [0, 0, 1, 1]

j = [0, 1, 0, 1]

# for loop

f, axes = plt.subplots(2, 2, figsize=(8, 6), sharex=True)

for var, i, j in zip(var_color_dict, i, j):

    sns.distplot(iris[var],                  

                    color = var_color_dict[var],

                    ax = axes[i, j])

plt.show()

import numpy as np

import pandas as pd

import matplotlib.pyplot as plt

import seaborn as sns

# Data Loading

iris = sns.load_dataset('iris')

iris.shape

(150, 5)

iris.head()

iris.groupby('species').size()

species
setosa        50
versicolor    50
virginica     50
dtype: int64

  (1) matplotlib.pyplot 으로 히스토그램 그리기

plt.hist(iris['sepal_width'], bins=10)

n, bins, patches = plt.hist(iris['sepal_width'], bins=10)

n, bins, patches

(array([ 4.,  7., 22., 24., 37., 31., 10., 11.,  2.,  2.]),
 array([2.  , 2.24, 2.48, 2.72, 2.96, 3.2 , 3.44, 3.68, 3.92, 4.16, 4.4 ]),
 <a list of 10 Patch objects>)

# Y axis as density

n, bins, patches = plt.hist(iris['sepal_width'], bins=10, density=True)

# facecolor, alpha

n, bins, patches = plt.hist(iris['sepal_width'], bins=10, 

                            density=True, 

                            facecolor='blue', 

                            alpha=0.5)

# Setting X, Y label and Title, axis

n, bins, patches = plt.hist(iris['sepal_width'], 

                            bins=10, 

                            density=True, 

                            facecolor='blue', 

                            alpha=0.5)

plt.xlabel('X bins')

plt.ylabel('Density')

plt.title('Histogram of Sepal Width')

plt.axis([1.5, 4.5, 0, 1.1])

plt.show()

  (2) seaborn 패키지로 히스토그램 그리기

sns.distplot(iris['sepal_width'])

plt.show()

# Kernel Density Curve with histogram

ax = sns.distplot(iris['sepal_width'], 

                      hist=True, 

                      kde=True, 

                      bins=10, 

                      color='blue', 

                      hist_kws={'edgecolor': 'gray'}, 

                      kde_kws={'linewidth': 2})

ax.set_title('Histogram of Sepal Width')

ax.set_xlabel('Sepal Width(cm)')

ax.set_ylabel('Density')

plt.show()

# Kernel Density Estimate Plot

sns.kdeplot(iris['sepal_width'],  

               shade=True, 

               bw=2, 

               label="Sepal Width")

plt.legend()

plt.show()

  (3) pandas.DataFrame.hist 를 이용한 히스토그램 그리기

# Histogram by pandas.DataFrame.hist

iris['sepal_width'].hist(bins=10, grid=False)

plt.show()

import numpy as np

import pandas as pd

# setting seed number for reproducibility

np.random.seed(123)

# Make a DataFrame

df = pd.DataFrame({'grp': ['grp_1']*10 + ['grp_2']*10, 

                          'col_1': np.random.randint(20, size=20), 

                          'col_2': np.random.randint(20, size=20)})

df

# UDF of random sampling

def sampling_func(data, sample_pct):

    np.random.seed(123)

    N = len(data)

    sample_n = int(len(data)*sample_pct) # integer

    sample = data.take(np.random.permutation(N)[:sample_n])

    return sample

# Apply random sampling UDF to groups

sample_set = df.groupby('grp').apply(sampling_func, sample_pct=0.8)

sample_set.sort_index()

# disable group_keys

sample_set = df.groupby('grp', group_keys=False).apply(sampling_func, sample_pct=0.8)

sample_set.sort_index()

test_set = df.drop(df.index[sample_set.index])

test_set

import numpy as np

import pandas as pd

from sklearn import datasets, linear_model

diabetes = datasets.load_diabetes()

diabetes.DESCR 

'Diabetes dataset

================

Notes

-----

Ten baseline variables, age, sex, body mass index, average blood

npressure, and six blood serum measurements were obtained for each of n =442 diabetes patients, as well as the response of interest, a\nquantitative measure of disease progression one year after baseline.\n\nData Set Characteristics:\n\n  :Number of Instances: 442\n\n  :Number of Attributes: First 10 columns are numeric predictive values\n\n  :Target: Column 11 is a quantitative measure of disease progression one year after baseline\n\n  :Attributes:\n    :Age:\n    :Sex:\n    :Body mass index:\n    :Average blood pressure:\n    :S1:\n    :S2:\n    :S3:\n    :S4:\n    :S5:\n    :S6:\n\nNote: Each of these 10 feature variables have been mean centered and scaled by the standard deviation times `n_samples` (i.e. the sum of squares of each column totals 1).\n\nSource URL:\nhttp://www4.stat.ncsu.edu/~boos/var.select/diabetes.html\n\nFor more information see:\nBradley Efron, Trevor Hastie, Iain Johnstone and Robert Tibshirani (2004) "Least Angle Regression," Annals of Statistics (with discussion), 407-499.\n(http://web.stanford.edu/~hastie/Papers/LARS/LeastAngle_2002.pdf)\n'

diabetes.feature_names

['age', 'sex', 'bmi', 'bp', 's1', 's2', 's3', 's4', 's5', 's6']

diabetes.target[:5]

array([151.,  75., 141., 206., 135.])

# Make a DataFrame of Y ('target')

diabetes_Y = pd.DataFrame(diabetes.target, columns = ['target'])

diabetes_Y[:5]

# Make a DataFrame of X with age, sex, bmi

diabetes_X = pd.DataFrame(diabetes.data[:, 0:3], 

                          columns = ['age', 'sex', 'bmi']) # age, sex, bmi

diabetes_X[:5]

diabetes_df = pd.concat([diabetes_Y, diabetes_X], axis=1)

diabetes_df[:5]

diabetes_df['grp'] = np.where(diabetes_df['sex'] > 0, 'M', 'F')

diabetes_df.drop(columns=['sex'], inplace=True) 

diabetes_df[:3]

# UDF of linear regression model

def lin_regress(data, yvar, xvars):

    # output, input variables

    Y = data[yvar]

    X = data[xvars]

    # Create linear regression object

    linreg = linear_model.LinearRegression()

    # Fit the linear regression model

    model = linreg.fit(X, Y)

    # Get the intercept and coefficients

    intercept = model.intercept_

    coef = model.coef_

    result = [intercept, coef]

    return result

# GroupBy

grouped = diabetes_df.groupby('grp')

# Apply the UDF of linear regression model by Group

lin_reg_coef = grouped.apply(lin_regress, 'target', ['age', 'bmi'])

lin_reg_coef

grp
F    [152.40684676047456, [23.199210147823813, 814....
M    [148.21507864445124, [291.7563226838977, 1092.... 

dtype: object 

lin_reg_coef['M']

[148.21507864445124, array([ 291.75632268, 1092.80118705])]

lin_reg_coef['F']

[152.40684676047456, array([ 23.19921015, 814.50932703])]

import numpy as np

import pandas as pd

from pandas import DataFrame

# making DataFrame with 4 random variables

np.random.seed(123) # for reproducibility

df= DataFrame(np.random.randn(10, 4), 

                    columns=['a', 'b', 'c', 'd'])

# setting index with 2 group, 'grp1' and 'grp2'

df['group'] = ['grp1', 'grp1', 'grp1', 'grp1', 'grp1', 

                  'grp2', 'grp2', 'grp2', 'grp2', 'grp2']

df = df.set_index('group')

df

  (1) 'd' 변수와 나머지 모든 변수 간 그룹 별 상관계수 구하기 : x.corrwith(x['d'])

# correlation with columns: corrwith()

corr_with_d = lambda x: x.corrwith(x['d'])

grouped = df.groupby('group')

grouped.apply(corr_with_d) 

  (2) 'a'변수와 'd'변수 간 그룹 별 상관계수 구하기 : g['a'].corr[g['d'])

# inter-column correlation: corr()

corr_a_d = lambda g: g['a'].corr(g['d'])

grouped = df.groupby('group')

DataFrame(grouped.apply(corr_a_d))