R, Python 분석과 프로그래밍의 친구 (by R Friend)

  (1) 압축 ZIP 파일 쓰기 (write)

import zipfile
import os

## change working directory
base_dir = '/Users/ihongdon/Downloads'
os.chdir(base_dir)

## check the current working directory
os.getcwd()

[Out] '/Users/ihongdon/Downloads'

## show the lists of files in the current working directory
os.listdir()

['sample_2.txt', 'sample_3.txt', 'sample_1.txt']

## (1) mode='w': to truncate and write a new file
myzip_w = zipfile.ZipFile('sample.zip', 'w')
myzip_w.write('sample_1.txt')

## You must call close() before exiting your program,
## or essential records will not be written.
myzip_w.close()

## ZipFile.is_zipfile(): Return True if a valid ZIP file exists.
zipfile.is_zipfile('sample.zip')

[Out] True

with zipfile.ZipFile('sample.zip', 'w') as myzip:
    myzip.write('sample_1.txt')
    myzip.close()

## (2) mode='x': to exclusively create and write a new file.
## if file refers to an existing file, a 'FileExistsError' will be raised.
myzip_x = zipfile.ZipFile('sample.zip', 'x')

[Out]
---------------------------------------------------------------------------
FileExistsError                           Traceback (most recent call last)
<ipython-input-7-bd84b411165c> in <module>
      1 ## (2) mode='x': to exclusively create and write a new file.
      2 ## if file refers to an existing file, a 'FileExistsError' will be raised.
----> 3 myzip_x = zipfile.ZipFile('sample.zip', 'x')

~/opt/anaconda3/lib/python3.8/zipfile.py in __init__(self, file, mode, compression, allowZip64, compresslevel, strict_timestamps)
   1249             while True:
   1250                 try:
-> 1251                     self.fp = io.open(file, filemode)
   1252                 except OSError:
   1253                     if filemode in modeDict:

FileExistsError: [Errno 17] File exists: 'sample.zip'

myzip_x = zipfile.ZipFile('sample2.zip', 'x')
myzip_x.write('sample_2.txt')
myzip_x.close()

myzip_x.namelist()

[Out] ['sample_2.txt']

## (3) mode='a': to append to an existing file.
myzip_a = zipfile.ZipFile('sample.zip', 'a')
myzip_a.write('sample_2.txt')
myzip_a.write('sample_3.txt')
myzip_a.close()

myzip_a.namelist()

[Out] ['sample_1.txt', 'sample_2.txt', 'sample_3.txt']

## (4) writing files to a zip file: with statement & for loop
file_list = ['sample_1.txt', 'sample_2.txt', 'sample_3.txt']

with zipfile.ZipFile('sample_all.zip', 'w') as myzip_all:
    for f in file_list:
        myzip_all.write(f)
        print(f, 'is written to myzip_all.zip')
    myzip_all.close()

[Out] 
sample_1.txt is written to myzip_all.zip
sample_2.txt is written to myzip_all.zip
sample_3.txt is written to myzip_all.zip

myzip_all.namelist()

[Out] ['sample_1.txt', 'sample_2.txt', 'sample_3.txt']

매개변수

설명

 compression

 compression은 자료를 압축 파일에 쓰기 위한 ZIP 압축 메소드이며, 기본 설정값은 ZIP_STORED 입니다.

Python 버전 3.1 부터 아래의 파일과 같은 객체를 지원합니다.

• zipfile.ZIP_STORED  (* default)
  
• zipfile.ZIP_DEFLATED
• zipfile.ZIP_BZIP2

Python 버전 3.3 부터는 ZIP_LZMA 객체를 지원합니다.

• zipfile.ZIP_LZMA
  

 allowZip64

 allowZip64=True (기본 설정) 이면 ZIP 파일 크기가 4GB를 넘을 경우 ZIP64 extensions 를 사용해서 ZIP 파일을 생성합니다.

 만약 allowZip64=False 설정인데 ZIP 파일 크기가 4GB를 넘을 경우에는 exception error 가 발생합니다.

 compresslevel

 compresslevel 매개변수는 자료를 압축할 수준을 지정할 때 사용합니다.

(compression 이 ZIP_STORED, ZIP_LZMA 일 경우에는 효과가 없으며, ZIP_DEPLATED, ZIP_BZIP2 에만 설정 가능합니다.)

• compression=ZIP_DEFLATED 일 경우 compresslevel=0~9 까지 설정 가능
  
• compression=ZIP_BZIP2 일 경우 compresslevel=1~9 까지 설정 가능
  

  (2) 압축 ZIP 파일 읽기 (read)

import pandas as pd

sample_1_df = pd.read_csv('sample_1.txt')
print(sample_1_df)

[Out]
   x1  x2  x3
0   1   2   3
1   4   5   6
2   7   8   9

  (3) 압축 ZIP 파일 이름(filename), 자료 리스트(namelist()), 파일 정보(getinfo) 보기

file_list = ['sample_1.txt', 'sample_2.txt', 'sample_3.txt']

with zipfile.ZipFile('sample_all.zip', 'w') as myzip_all:
    for f in file_list:
        myzip_all.write(f)
    myzip_all.close()

## ZipFile.is_zipfile(): Return True if a valid ZIP file exists.
zipfile.is_zipfile('sample_all.zip')

[Out] True

## ZipFile.filename: Name of the ZIP file
myzip_all.filename

[Out] 'sample_all.zip'

## file name lists of sample_all.zip
myzip_all.namelist()

[Out] ['sample_1.txt', 'sample_2.txt', 'sample_3.txt']

## ZipFile.getinfo(): Zip information about the archive member name.
myzip_all.getinfo('sample_1.txt')

[Out] <ZipInfo filename='sample_1.txt' filemode='-rw-r--r--' file_size=27>

  (4) 압축 ZIP 파일 해제하기 (extract)

## (4-1) ZipFile.extract()
## : extracting a member from the archive to the current working directory.
extract_path = '/Users/ihongdon/Downloads/sample_3'
zipfile.ZipFile('sample_all.zip').extract('sample_3.txt', path=extract_path)

[Out] '/Users/ihongdon/Downloads/sample_3/sample_3.txt'

os.listdir(extract_path)

[Out] ['sample_3.txt']

## (4-2) ZipFile.extractall()
## : extracting all members from the archive to the current working directory.
extractall_path = '/Users/ihongdon/Downloads/sample_all'
zipfile.ZipFile('sample_all.zip').extractall(extractall_path)

os.listdir(extractall_path)

[Out] ['sample_2.txt', 'sample_3.txt', 'sample_1.txt']

  (5) 웹에서 ZIP 파일 다운로드하여 압축 해제하기 (download and extract ZIP file)

## Download the movielens data from website url
import tensorflow.keras as keras
from zipfile import ZipFile
from pathlib import Path

import os

movielens_data_file_url = (
    "http://files.grouplens.org/datasets/movielens/ml-latest-small.zip"
)

movielens_zipped_file = keras.utils.get_file(
    "ml-latest-small.zip", movielens_data_file_url, extract=False
)

keras_datasets_path = Path(movielens_zipped_file).parents[0]
movielens_dir = keras_datasets_path / "ml-latest-small"

## Only extract the data the first time the script is run.
if not movielens_dir.exists():
    with ZipFile(movielens_zipped_file, "r") as zip:
        zip.extractall(path=keras_datasets_path) # extract all members in a ZIP file

print('datasets path:', keras_datasets_path)

[Out] datasets path: /Users/ihongdon/.keras/datasets

print(os.listdir(keras_datasets_path))

[Out] ['cowper.txt', 'reuters_word_index.json', 'imdb_word_index.json', 'flower_photos.tar.gz', 'cifar-10-batches-py', 'mnist.npz', 'ml-latest-small.zip', 'ml-latest-small', 'fashion-mnist', 'butler.txt', 'imdb.npz', 'cifar-10-batches-py.tar.gz', 'boston_housing.npz', 'creditcard.csv', 'creditcard.zip', 'derby.txt', 'train.csv', 'flower_photos', 'reuters.npz', 'fsns.tfrec']

os.listdir(movielens_dir)

[Out] ['links.csv', 'tags.csv', 'ratings.csv', 'README.txt', 'movies.csv']

  (1) 이전 원소 대비 변동률 

       (Percentage change between the current and a prior element)

In [1]: import pandas as pd

In [2]:

s = pd.Series([10, 20, 50, 55, 70])

s.pct_change()

Out[2]:

0 NaN

1 1.000000

2 1.500000

3 0.100000

4 0.272727

dtype: float64

  (2) 이전 2개 원소 대비 변동률 

       (Percentage change between the current and 2 periods prior element)

In [3]:

s = pd.Series([10, 20, 50, 55, 70])

s.pct_change(periods=2)

Out[3]:

0 NaN

1 NaN

2 4.00

3 1.75

4 0.40

dtype: float64

  (3) 결측값을 이전 원소 값으로 대체 후 이전 원소 대비 변동률 

       (Percentage change between the current and a prior element
        after filling the missing values using the 'forward fill' method)

In [4]:

s2 = pd.Series([10, 20, 50, None, 70])

s2.pct_change(fill_method='ffill')

Out[4]:

0 NaN

1 1.0

2 1.5

3 0.0

4 0.4

dtype: float64

  (4) 그룹별 이전 분기 대비 변동률 

      (Percentage change between the current and a prior quarter by Group)

In [5]:

# input data sale = pd.DataFrame( {'product': ['a', 'a', 'a', 'a', 'a', 'a', 'a', 'a', 'a', 'a', 'a', 'a', 'b', 'b', 'b', 'b', 'b', 'b', 'b', 'b', 'b', 'b', 'b', 'b'], 'year': [2018, 2018, 2018, 2018, 2019, 2019, 2019, 2019, 2020, 2020, 2020, 2020, 2018, 2018, 2018, 2018, 2019, 2019, 2019, 2019, 2020, 2020, 2020, 2020], 'quarter': [1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4], 'sales': [5, 6, 6, 8, 10, 20, 30, 40, 12, 25, 38, 50, 60, 65, 80, 95, 100, 125, 130, 140, 110, 130, 132, 144]})

sale

Out[5]:

product year quarter sales

0 a 2018 1 5

1 a 2018 2 6

2 a 2018 3 6

3 a 2018 4 8

4 a 2019 1 10

5 a 2019 2 20

6 a 2019 3 30

7 a 2019 4 40

8 a 2020 1 12

9 a 2020 2 25

10 a 2020 3 38

11 a 2020 4 50

12 b 2018 1 60

13 b 2018 2 65

14 b 2018 3 80

15 b 2018 4 95

16 b 2019 1 100

17 b 2019 2 125

18 b 2019 3 130

19 b 2019 4 140

20 b 2020 1 110

21 b 2020 2 130

22 b 2020 3 132

23 b 2020 4 144

In [6]:

sale['sales_pct_change_by_1q'] = sale.\ sort_values(['year', 'quarter']).\ groupby(['product']).\ sales.pct_change()

sale

Out[6]:

product year quarter sales pct_change_by_1q

0 a 2018 1 NaN

1 a 2018 2 6 0.200000

2 a 2018 3 6 0.000000

3 a 2018 4 8 0.333333

4 a 2019 1 10 0.250000

5 a 2019 2 20 1.000000

6 a 2019 3 30 0.500000

7 a 2019 4 40 0.333333

8 a 2020 1 12 -0.700000

9 a 2020 2 25 1.083333

10 a 2020 3 38 0.520000

11 a 2020 4 50 0.315789

12 b 2018 1 60 NaN

13 b 2018 2 65 0.083333

14 b 2018 3 80 0.230769

15 b 2018 4 95 0.187500

16 b 2019 1 100 0.052632

17 b 2019 2 125 0.250000

18 b 2019 3 130 0.040000

19 b 2019 4 140 0.076923

20 b 2020 1 110 -0.214286

21 b 2020 2 130 0.181818

22 b 2020 3 132 0.015385

23 b 2020 4 144 0.090909

  (5) 그룹별 전년 동분기 대비 변동률  

       (Percentage change between the current and a year before by Group)

In [7]:

 sale['pct_change_by_1y'] = sale.sort_values(['year', 'quarter']).\

  groupby(['product']).\

  sales.pct_change(periods=4)

In [8]: sale.sort_values(by=['product', 'quarter', 'year'])

Out[8]:

product year quarter sales pct_change_by_1q pct_change_by_1y

0 a 2018 1 5 NaN NaN

4 a 2019 1 10 0.250000 1.000000

8 a 2020 1 12 -0.700000 0.200000

1 a 2018 2 6 0.200000 NaN

5 a 2019 2 20 1.000000 2.333333

9 a 2020 2 25 1.083333 0.250000

2 a 2018 3 6 0.000000 NaN

6 a 2019 3 30 0.500000 4.000000

10 a 2020 3 38 0.520000 0.266667

3 a 2018 4 8 0.333333 NaN

7 a 2019 4 40 0.333333 4.000000

11 a 2020 4 50 0.315789 0.250000

12 b 2018 1 60 NaN NaN

16 b 2019 1 100 0.052632 0.666667

20 b 2020 1 110 -0.214286 0.100000

13 b 2018 2 65 0.083333 NaN

17 b 2019 2 125 0.250000 0.923077

21 b 2020 2 130 0.181818 0.040000

14 b 2018 3 80 0.230769 NaN

18 b 2019 3 130 0.040000 0.625000

22 b 2020 3 132 0.015385 0.015385

15 b 2018 4 95 0.187500 NaN

19 b 2019 4 140 0.076923 0.473684

23 b 2020 4 144 0.090909 0.028571

# or equvalently

sale['pct_change_by_1y'] = sale.sort_values(by=['quarter', 'year']).\

    groupby(['product', 'quarter']).\

        sales.pct_change()

sale.sort_values(by=['product', 'quarter', 'year'])

  (1) 파이썬 사전형의 키와 값을 일정 간격을 두고 인쇄하기

      (Print Key, Value of Python Dictionary with fixed space)

my_dict = {'first name': 'KilDong.', 

           'last name': 'Hong.',

           'age': '30.', 

           'address': 'Seoul, Korea.'}

print(my_dict)

[Out]
{'first name': 'KilDong.', 'last name': 'Hong.', 'age': '30.', 'address': 'Seoul, Korea.'}

for k, v in my_dict.items():

    print("{} : {}".format(k, v))

[Out]

first name : KilDong.
last name : Hong.
age : 30.
address : Seoul, Korea.

for k, v in my_dict.items():

    print("{!r:15s} : {}".format(k, v))

[Out]
'first name'    : KilDong.
'last name'     : Hong.
'age'           : 30.
'address'       : Seoul, Korea.

  (2) 파이썬 인쇄 escape 옵션 (Python Print escape options)

for k, v in my_dict.items():

    print("\n{} : {}".format(k, v))

[Out]

first name : KilDong.

last name : Hong.

age : 30.

address : Seoul, Korea.

for k, v in my_dict.items():

    print("\t{} : {}".format(k, v))

[Out]
	first name : KilDong.
	last name : Hong.
	age : 30.
	address : Seoul, Korea.

for k, v in my_dict.items():

    print("{} : {}\b".format(k, v))

[Out]

first name : KilDong
last name : Hong
age : 30
address : Seoul, Korea

for k, v in my_dict.items():

    print("{} : \"{}\"".format(k, v))

[Out]

first name : "KilDong."
last name : "Hong."
age : "30."
address : "Seoul, Korea."

for k, v in my_dict.items():

    print("{} : \\{}\\".format(k, v))

[Out]

first name : \KilDong.\
last name : \Hong.\
age : \30.\
address : \Seoul, Korea.\

  (3) 파이썬 인쇄 sep, end 옵션 (Python Print sep, end options)

for k, v in my_dict.items():

    print(k, v, sep=" : ")

[Out]
first name : KilDong.
last name : Hong.
age : 30.
address : Seoul, Korea.

for k, v in my_dict.items():

    print(k + " : ", end=v+"\n")

[Out]

first name : KilDong.
last name : Hong.
age : 30. 

address : Seoul, Korea.

 CSR 자료구조의 장점

(Advantages of the CSR format)

CSR 자료구조의 단점

(Disadvantages of the CSR format)

•  효율적인 산술 연산
   (예: CSR + CSR, CSR * CSR 등)
• 효율적인 행 슬라이싱
  (efficient row slicing)
• 빠른 행렬 벡터 곱
  (fast matrix vector products)

• 느린 열 슬라이싱
  (slow column slicing) 
  --> Compressed Sparse Column format 고려
• 희소성 구조의 변화 시 연산비용 큼
  --> LIL, DOK 고려

  (1) NumPy array 행렬을 SciPy 압축 희소 행(CSR) 행렬로 변환하기

      (Transforming a NumPy matrix to a SciPy compressed sparse row matrix)

import numpy as np

from scipy.sparse import csr_matrix

arr = np.array([[0, 1, 0, 2], 

                [0, 3, 4, 5], 

                [0, 0, 0, 0], 

                [6, 0, 0, 7], 

                [0, 8, 0, 0]])

arr

[Out]

array([[0, 1, 0, 2],
       [0, 3, 4, 5],
       [0, 0, 0, 0],
       [6, 0, 0, 7],
       [0, 8, 0, 0]])

# converting NumPy array into SciPy Compressed Sparse Row matrix

csr_mat = csr_matrix(arr)

csr_mat

[Out] <5x4 sparse matrix of type '<class 'numpy.longlong'>'
	with 8 stored elements in Compressed Sparse Row format>

print('-- Compressed Sparse Row matrix --')

print('indptr:', csr_mat.indptr)

print('indices:', csr_mat.indices)

print('data:', csr_mat.data)

-- Compressed Sparse Row matrix --
indptr: [0 2 5 5 7 8]
indices: [1 3 1 2 3 0 3 1]
data: [1 2 3 4 5 6 7 8]

# converting NumPy array into SciPy Compressed Sparse Row matrix

indptr = np.array([0, 2, 5, 5, 7, 8]) # the location of the first element of the row.

indices = np.array([1, 3, 1, 2, 3, 0, 3, 1]) # column indices

data = np.array([1, 2, 3, 4, 5, 6, 7, 8])    # corresponding value

csr_mat2 = csr_matrix((data, indices, indptr), shape=(5, 4))

csr_mat2

[Out] <5x4 sparse matrix of type '<class 'numpy.int64'>'

	with 8 stored elements in Compressed Sparse Row format>

print('-- Compressed Sparse Row matrix 2 --')

print('indptr:', csr_mat2.indptr)

print('indices:', csr_mat2.indices)

print('data:', csr_mat2.data)

-- Compressed Sparse Row matrix 2 --
indptr: [0 2 5 5 7 8]
indices: [1 3 1 2 3 0 3 1]
data: [1 2 3 4 5 6 7 8]

# converting NumPy array into SciPy Compressed Sparse Row matrix

row = np.array([0, 0, 1, 1, 1, 3, 3, 4])

col = np.array([1, 3, 1, 2, 3, 0, 3, 1])

data = np.array([1, 2, 3, 4, 5, 6, 7, 8])

csr_mat3 = csr_matrix((data, (row, col)), shape=(5, 4))

csr_mat3

[Out] <5x4 sparse matrix of type '<class 'numpy.longlong'>'

	with 8 stored elements in Compressed Sparse Row format>

print('-- Compressed Sparse Row matrix 3 --')

print('indptr:', csr_mat3.indptr)

print('indices:', csr_mat3.indices)

print('data:', csr_mat3.data)

-- Compressed Sparse Row matrix 2 --
indptr: [0 2 4 4 6 7]
indices: [1 3 1 2 0 3 1]
data: [1 2 3 4 5 6 7]

-- Compressed Sparse Row matrix 3 --
indptr: [0 2 5 5 7 8]
indices: [1 3 1 2 3 0 3 1]
data: [1 2 3 4 5 6 7 8]

  (2) SciPy 압축 희소 행(CSR) 행렬을 NumPy 행렬로 변환하기

     (Transforming a SciPy compressed sparse row matrix into a NumPy matrix) 

# converting sparse matrix to NumPy array

csr_mat.toarray()

[Out]

array([[0, 1, 0, 2],
       [0, 3, 4, 5],
       [0, 0, 0, 0],
       [6, 0, 0, 7],
       [0, 8, 0, 0]], dtype=int64)

csr_mat2.toarray()

[Out]

array([[0, 1, 0, 2],
       [0, 3, 4, 5],
       [0, 0, 0, 0],
       [6, 0, 0, 7],
       [0, 8, 0, 0]], dtype=int64)

csr_mat3.toarray()

[Out]

array([[0, 1, 0, 2],
       [0, 3, 4, 5],
       [0, 0, 0, 0],
       [6, 0, 0, 7],
       [0, 8, 0, 0]], dtype=int64)

csr_mat.todense()

[Out]

array([[0, 1, 0, 2],
       [0, 3, 4, 5],
       [0, 0, 0, 0],
       [6, 0, 0, 7],
       [0, 8, 0, 0]], dtype=int64)

  (3) 동일 위치에 중복된 원소값은 합산 (Duplicate entries are summed together.)

  5 + 3

 0

 0

 0

 2 + 4

 0

 0

 0

 0

# Duplicate entries are summed together. 

row = np.array([0, 1, 1, 0])

col = np.array([0, 1, 1, 0])

data = np.array([5, 2, 4, 3])

csr_matrix((data, (row, col)), shape=(3, 3)).toarray()

[Out]

array([[8, 0, 0],
       [0, 6, 0],
       [0, 0, 0]], dtype=int64)

import numpy as np

# weights

a = np.array([0.5, 0.3, 0.1, 0.08, 0.02])

print('a shape:', a.shape)

a shape: (5,)

print(a)

[0.5  0.3  0.1  0.08 0.02]

# values

b = np.arange(15).reshape(5, 3)

print('b shape:', b.shape)

b shape: (5, 3)

print(b)

[[ 0  1  2]

 [ 3  4  5]

 [ 6  7  8]

 [ 9 10 11]

 [12 13 14]]

  (1) 내적을 이용한 가중합 계산: np.dot(a, b) 또는 np.matmul(a, b)

np.dot(a, b)

Out[2]: array([2.46, 3.46, 4.46])

np.matmul(a, b)

Out[3]: array([2.46, 3.46, 4.46])

  (2) Broadcasting을 이용하여 가중치와 값을 원소끼리 곱한 후, axis=0으로 합하기

# shape of a_rs and b are different

a_rs = a.reshape(5, 1)

print(a_rs.shape)

print(a_rs)

(5, 1)

print(b.shape)

(5, 3)

# multiply using boradcasting of a_rs

a_rs_b_mult = a_rs * b

print(a_rs_b_mult.shape)

(5, 3)

print(a_rs_b_mult)

[[0.   0.5  1.  ]

 [0.9  1.2  1.5 ]

 [0.6  0.7  0.8 ]

 [0.72 0.8  0.88]

 [0.24 0.26 0.28]]

# weighted sum

np.sum(a_rs_b_mult, axis=0)

Out[9]: array([2.46, 3.46, 4.46])

  (3) repeat()로 가중치를 반복 생성한 후, 가중치와 값을 원소끼리 곱한 후 합하기

# match the shape of a and b by repeatition 

a_rs_rp = a.reshape(5, 1).repeat(3, axis=1)

print(a_rs_rp.shape)

(5, 3)

print(a_rs_rp)

[[0.5  0.5  0.5 ]

 [0.3  0.3  0.3 ]

 [0.1  0.1  0.1 ]

 [0.08 0.08 0.08]

 [0.02 0.02 0.02]]

# multiplication of a_rs_rp and b per each elements

a_rs_rp_b_mult = a_rs_rp * b

print(a_rs_rp_b_mult.shape)

(5, 3)

print(a_rs_rp_b_mult)

[[0.   0.5  1.  ]

 [0.9  1.2  1.5 ]

 [0.6  0.7  0.8 ]

 [0.72 0.8  0.88]

 [0.24 0.26 0.28]]

# weighted sum

np.sum(a_rs_rp_b_mult, axis=0)

Out[17]: array([2.46, 3.46, 4.46])

  (1) iris dataset 의 species 를 분류할 수 있는 Random Forest 분류 모델 훈련

#%% import libraries

import pandas as pd

import numpy as np

import matplotlib.pyplot as plt

import seaborn as sns

#%% iris dataset

iris = sns.load_dataset('iris')

iris.info()

<class 'pandas.core.frame.DataFrame'>

RangeIndex: 150 entries, 0 to 149

Data columns (total 5 columns):

 #   Column        Non-Null Count  Dtype  

---  ------        --------------  -----  

 0   sepal_length  150 non-null    float64

 1   sepal_width   150 non-null    float64

 2   petal_length  150 non-null    float64

 3   petal_width   150 non-null    float64

 4   species       150 non-null    object 

dtypes: float64(4), object(1)

memory usage: 6.0+ KB

iris.groupby('species').size()

Out[1]: 

species

setosa        50

versicolor    50

virginica     50

dtype: int64

#%% scatter plot by 'species'

sns.scatterplot(x='petal_length', 

                y='petal_width', 

                hue='species', 

                style='species', 

                s=100, 

                data=iris)

plt.show()

#%% Classification using Random Forest

from sklearn.ensemble import RandomForestClassifier

X = np.array(iris[['petal_length', 'petal_width']])

y = iris['species']

y = np.where(y=='setosa', 0, np.where(y=='versicolor', 1, 2))

rfc = RandomForestClassifier(max_depth=2, n_estimators=200, random_state=1004)

rfc.fit(X, y)

  (2) numpy의 meshgrid() 로 설명변수 공간의 좌표들을 생성하여 

      Random Forest 분류 모델로 예측

#%% predict

pred = rfc.predict(X)

print(pred == y)

[ True  True  True  True  True  True  True  True  True  True  True  True

  True  True  True  True  True  True  True  True  True  True  True  True

  True  True  True  True  True  True  True  True  True  True  True  True

  True  True  True  True  True  True  True  True  True  True  True  True

  True  True  True  True  True  True  True  True  True  True  True  True

  True  True  True  True  True  True  True  True  True  True False  True

  True  True  True  True  True False  True  True  True  True  True False

  True  True  True  True  True  True  True  True  True  True  True  True

  True  True  True  True  True  True  True  True  True  True False  True

  True  True  True  True  True  True  True  True  True  True  True False

  True  True  True  True  True  True  True  True  True  True  True  True

  True False False  True  True  True  True  True  True  True  True  True

  True  True  True  True  True  True]

# create coordinate matrices from x_min~x_max, y_min~y_max coordinates

h = 0.01

x_min, x_max = X[:, 0].min() - .1, X[:, 0].max() + .1

y_min, y_max = X[:, 1].min() - .1, X[:, 1].max() + .1

xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h))

XY = np.c_[xx.ravel(), yy.ravel()]

XY.shape

Out[4]: (158600, 2)

# predict

pred_cls = rfc.predict(XY)

# align the shape of Z with xx

Z = pred_cls.reshape(xx.shape)

  (3) matplotlib.contourf() 로 Random Forest Classifier 예측 결과 시각화

      -->  (4) iris dataset 의 species 의 산점도를 겹쳐서 그리기

# Random Forest Classifier: Decision Boundary

plt.contourf(xx, yy, Z)

plt.axis('off')

plt.contourf(xx, yy, Z)

plt.axis('off')

sns.scatterplot(x='petal_length', 

                y='petal_width', 

                hue='species', 

                style='species', 

                s=100, 

                data=iris)

plt.title('Decision Boundary by Random Forest', fontsize=14)

plt.show()

plt.contourf(xx, yy, Z)

plt.axis('off')

# or plot data points using matplotlib and for loop

N = 50

CLS_NUM = 3

markers = ['o', 'x', 's']

for i in range(CLS_NUM):

    plt.scatter(X[i*N: (i+1)*N, 0], X[i*N:(i+1)*N, 1], s=40, marker=markers[i])

  (방법 1) for loop 으로 A배열의 idx 위치에 B배열의 원소 순서대로 더하기

import numpy as np

A = np.arange(24).reshape(8, 3)

print(A)

[[ 0  1  2]
 [ 3  4  5]
 [ 6  7  8]
 [ 9 10 11]
 [12 13 14]
 [15 16 17]
 [18 19 20]
 [21 22 23]]

B = np.arange(12).reshape(4, 3)

print(B)

[[ 0  1  2]
 [ 3  4  5]
 [ 6  7  8]
 [ 9 10 11]]

idx = np.array([0, 0, 3, 5])

for i, id in enumerate(idx):

    A[id] += B[i]

print(A)

[[ 3  6  9]
 [ 3  4  5]
 [ 6  7  8]
 [15 17 19]
 [12 13 14]
 [24 26 28]
 [18 19 20]
 [21 22 23]]

  (방법 2) np.add.at(A, idx, B) 함수로 A배열의 idx 위치에 B배열의 원소 순서대로 더하기

import numpy as np

A = np.arange(24).reshape(8, 3)

B = np.arange(12).reshape(4, 3)

idx = np.array([0, 0, 3, 5])

np.add.at(A, idx, B)

print(A)

[[ 3  6  9]
 [ 3  4  5]
 [ 6  7  8]
 [15 17 19]
 [12 13 14]
 [24 26 28]
 [18 19 20]
 [21 22 23]]

import numpy as np

import pandas as pd

# Make a DataFrame with missing values

df = pd.DataFrame({'a': [1, 2, 3, np.nan, 5], 

                   'b': [30, 20, np.nan, 35, 32], 

                   'c': [0.2, np.nan, 0.5, 0.3, 0.4], 

                   'd': ['c1', np.nan, 'c3', 'c4', 'c5'], 

                   'e': [10, 11, np.nan, 13, 15]})

df

missing_fill_val = {'a': 0, 

                    'b': df.b.mean(), 

                    'c': df.c.median(), 

                    'd': 'Unknown', 

                    'e': df.e.interpolate()}

print(missing_fill_val)

{'a': 0, 'c': 0.35, 'b': 29.25, 'e': 0    10.0
1    11.0
2    12.0
3    13.0
4    15.0
Name: e, dtype: float64, 'd': 'Unknown'}

df2 = df.fillna(missing_fill_val)

df2

df.fillna(missing_fill_val, inplace=True)