UofT Scientific Coders - Intro to pandas

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

df = pd.read_table('./gapminderDataFiveYear.txt')

df

type(df)

pandas.core.frame.DataFrame

df.shape

(1704, 6)

df.columns

Index(['country', 'year', 'pop', 'continent', 'lifeExp', 'gdpPercap'], dtype='object')

df.head()

df.head(6)

df.tail()

df.info()

<class 'pandas.core.frame.DataFrame'>
Int64Index: 1704 entries, 0 to 1703
Data columns (total 6 columns):
country      1704 non-null object
year         1704 non-null int64
pop          1704 non-null float64
continent    1704 non-null object
lifeExp      1704 non-null float64
gdpPercap    1704 non-null float64
dtypes: float64(3), int64(1), object(2)
memory usage: 93.2+ KB

df.dtypes

country       object
year           int64
pop          float64
continent     object
lifeExp      float64
gdpPercap    float64
dtype: object

df.describe()

#select multiple columns with a list of column names
col_list = ['year','lifeExp', 'country']
df[col_list]

#alternative selection with dot notation won't work if column names have spaces, uncommon characters or leading numbers
df.lifeExp

0       28.801
1       30.332
2       31.997
3       34.020
4       36.088
5       38.438
6       39.854
7       40.822
8       41.674
9       41.763
10      42.129
11      43.828
12      55.230
13      59.280
14      64.820
15      66.220
16      67.690
17      68.930
18      70.420
19      72.000
20      71.581
21      72.950
22      75.651
23      76.423
24      43.077
25      45.685
26      48.303
27      51.407
28      54.518
29      58.014
         ...
1674    49.113
1675    52.922
1676    55.599
1677    58.020
1678    60.308
1679    62.698
1680    42.038
1681    44.077
1682    46.023
1683    47.768
1684    50.107
1685    51.386
1686    51.821
1687    50.821
1688    46.100
1689    40.238
1690    39.193
1691    42.384
1692    48.451
1693    50.469
1694    52.358
1695    53.995
1696    55.635
1697    57.674
1698    60.363
1699    62.351
1700    60.377
1701    46.809
1702    39.989
1703    43.487
Name: lifeExp, dtype: float64

#using describe() on a categorical column
df[['country', 'continent']].describe()

#by index location
df.iloc[[1000]]

#you can provide a list of index values to select
df.iloc[[0,5,10]]

#or select with the slice notation
df[0:5]

#select by index label
#would require named index
country_index = df.set_index(['continent','country'])

country_index

country_index.loc['Americas','Canada']

/home/derek/anaconda2/envs/py3/lib/python3.5/site-packages/ipykernel/__main__.py:1: PerformanceWarning: indexing past lexsort depth may impact performance.
  if __name__ == '__main__':

#boolean indexing
large_pop = df[df['pop'] > 300000000]

large_pop

large_pop['country'].unique()

array(['China', 'India', 'United States'], dtype=object)

multi_criteria = df[(df['country']=='Canada') | (df['year'] > 1990)]

multi_criteria

df.country.unique()

array(['Afghanistan', 'Albania', 'Algeria', 'Angola', 'Argentina',
       'Australia', 'Austria', 'Bahrain', 'Bangladesh', 'Belgium', 'Benin',
       'Bolivia', 'Bosnia and Herzegovina', 'Botswana', 'Brazil',
       'Bulgaria', 'Burkina Faso', 'Burundi', 'Cambodia', 'Cameroon',
       'Canada', 'Central African Republic', 'Chad', 'Chile', 'China',
       'Colombia', 'Comoros', 'Congo, Dem. Rep.', 'Congo, Rep.',
       'Costa Rica', "Cote d'Ivoire", 'Croatia', 'Cuba', 'Czech Republic',
       'Denmark', 'Djibouti', 'Dominican Republic', 'Ecuador', 'Egypt',
       'El Salvador', 'Equatorial Guinea', 'Eritrea', 'Ethiopia',
       'Finland', 'France', 'Gabon', 'Gambia', 'Germany', 'Ghana',
       'Greece', 'Guatemala', 'Guinea', 'Guinea-Bissau', 'Haiti',
       'Honduras', 'Hong Kong, China', 'Hungary', 'Iceland', 'India',
       'Indonesia', 'Iran', 'Iraq', 'Ireland', 'Israel', 'Italy',
       'Jamaica', 'Japan', 'Jordan', 'Kenya', 'Korea, Dem. Rep.',
       'Korea, Rep.', 'Kuwait', 'Lebanon', 'Lesotho', 'Liberia', 'Libya',
       'Madagascar', 'Malawi', 'Malaysia', 'Mali', 'Mauritania',
       'Mauritius', 'Mexico', 'Mongolia', 'Montenegro', 'Morocco',
       'Mozambique', 'Myanmar', 'Namibia', 'Nepal', 'Netherlands',
       'New Zealand', 'Nicaragua', 'Niger', 'Nigeria', 'Norway', 'Oman',
       'Pakistan', 'Panama', 'Paraguay', 'Peru', 'Philippines', 'Poland',
       'Portugal', 'Puerto Rico', 'Reunion', 'Romania', 'Rwanda',
       'Sao Tome and Principe', 'Saudi Arabia', 'Senegal', 'Serbia',
       'Sierra Leone', 'Singapore', 'Slovak Republic', 'Slovenia',
       'Somalia', 'South Africa', 'Spain', 'Sri Lanka', 'Sudan',
       'Swaziland', 'Sweden', 'Switzerland', 'Syria', 'Taiwan', 'Tanzania',
       'Thailand', 'Togo', 'Trinidad and Tobago', 'Tunisia', 'Turkey',
       'Uganda', 'United Kingdom', 'United States', 'Uruguay', 'Venezuela',
       'Vietnam', 'West Bank and Gaza', 'Yemen, Rep.', 'Zambia', 'Zimbabwe'], dtype=object)

def print_stats(df,country,year):
    """ Prints the life expectancy, gdp per capita and population
    of country in year. """
    spec = ['lifeExp','pop','gdpPercap']

    print('Statistics for', country, 'in', year)
    print(df[(df['country']==country) & (df['year']==year)][spec])

print_stats(df, 'Canada', 2007)

Statistics for Canada in 2007
     lifeExp       pop    gdpPercap
251   80.653  33390141  36319.23501

continents = df.groupby('continent')
continents

<pandas.core.groupby.DataFrameGroupBy object at 0x7fbac870b128>

len(continents)

5

#helpful way to visualize the groupby object: gives first row of each group
continents.first()

continents.size()

continent
Africa      624
Americas    300
Asia        396
Europe      360
Oceania      24
dtype: int64

len(continents.get_group('Americas')['country'].unique())

25

continents['lifeExp'].mean()

continent
Africa      48.865330
Americas    64.658737
Asia        60.064903
Europe      71.903686
Oceania     74.326208
Name: lifeExp, dtype: float64

df.groupby(['continent', 'year']).agg(np.mean)

continents.get_group('Africa').describe()

continents.get_group('Asia').lifeExp.max()

82.602999999999994

continents.get_group('Asia').lifeExp.idxmax()

803

#idxmax convenience function will return the index with max value
df[df['continent']=='Asia']['lifeExp'].idxmax()

803

df.loc[803]

country            Japan
year                2007
pop          1.27468e+08
continent           Asia
lifeExp           82.603
gdpPercap        31656.1
Name: 803, dtype: object

sorted_by_lifeExp = df.sort_values('lifeExp', ascending=False)

sorted_by_lifeExp['lifeExp_rank'] = np.arange(len(sorted_by_lifeExp)) + 1

#lists all rows in order of lifeExp
sorted_by_lifeExp.head()

def ranker(df):
    """Assigns a rank to each country based on lifeExp, with 1 having the highest lifeExp.
    Assumes the data is DESC sorted by lifeExp."""
    df['lifeExp_rank'] = np.arange(len(df)) + 1
    return df

#apply the ranking function on a per year basis:
sorted_by_lifeExp = sorted_by_lifeExp.groupby('year').apply(ranker)

sorted_by_lifeExp[sorted_by_lifeExp.year == 2007].head()

sorted_by_lifeExp[(sorted_by_lifeExp['country']=='Canada')]

%matplotlib inline

df.plot(x='year',y='lifeExp',kind='scatter')

<matplotlib.axes._subplots.AxesSubplot at 0x7fbac8710b00>

df.plot(x='gdpPercap',y='lifeExp',kind='scatter', alpha = 0.2, s=50, marker='o')

<matplotlib.axes._subplots.AxesSubplot at 0x7fbac86e87f0>

df[df['gdpPercap'] > 55000]

df.hist(column='lifeExp')

array([[<matplotlib.axes._subplots.AxesSubplot object at 0x7fbac5fb3b00>]], dtype=object)

df.lifeExp.plot.hist(bins=200)

<matplotlib.axes._subplots.AxesSubplot at 0x7fbac5fa1a90>

df['lifeExp'].plot(kind='kde')

<matplotlib.axes._subplots.AxesSubplot at 0x7fbac5e5a588>

def compare_lifeExp(country1, country2):
    """Plot life expectancy vs year for country1 and country2"""
    ax = plt.subplot()
    for c in [country1,country2]:
        df[df['country']==c].plot(x='year',y='lifeExp', ax=ax)
    plt.legend((country1,country2))

compare_lifeExp('Canada', 'Mexico')

spec=['country','lifeExp']
df[df['year']==1982][spec].min()

country    Afghanistan
lifeExp         38.445
dtype: object

df[(df['year']==1982) & (df['country']=='Afghanistan')]

df.loc[df[df['year']==1982]['lifeExp'].idxmin()]['country']

'Sierra Leone'

continents = df.groupby(['continent'])
for continent in continents.groups:
    group = continents.get_group(continent)
    group[group['year']==2007].plot(kind='scatter', x='gdpPercap',
                                    y='lifeExp', title=continent)
    plt.axis([-10000,60000,30,90])

#Example
fig,ax = plt.subplots(1,1)
colours = ['m','b','r','g','y']
for continent, colour in zip(continents.groups, colours):
    group = continents.get_group(continent)
    group[group['year']==2007].plot(kind='scatter',x='gdpPercap',y='lifeExp',label=continent,ax=ax,color=colour,alpha=0.5)
ax.set_title(2007)
plt.legend(loc='lower right')

<matplotlib.legend.Legend at 0x7f5791e0db00>

def compare_gdp_lifeExp(df,country):
    """ plot GDP per capita against life expectancy for a given country.
    print year of min/max gdp per capita and life expectancy
    """

    sub = df[df['country']==country]
    sub.plot(x='gdpPercap',y='lifeExp',kind='scatter',title=country)

    print('Year of Min/Max GDP per capita')
    print(df.iloc[[sub['gdpPercap'].idxmin(),sub['gdpPercap'].idxmax()]]['year'])
    print('Year of Min/Max life expectancy')
    print(df.iloc[[sub['lifeExp'].idxmin(),sub['lifeExp'].idxmax()]]['year'])

compare_gdp_lifeExp(df,'Zimbabwe')

Year of Min/Max GDP per capita
1692    1952
1696    1972
Name: year, dtype: int64
Year of Min/Max life expectancy
1702    2002
1699    1987
Name: year, dtype: int64

compare_gdp_lifeExp(df,'Canada')

Year of Min/Max GDP per capita
240    1952
251    2007
Name: year, dtype: int64
Year of Min/Max life expectancy
240    1952
251    2007
Name: year, dtype: int64

import seaborn as sns

/home/derek/anaconda2/envs/py3/lib/python3.5/site-packages/IPython/html.py:14: ShimWarning: The `IPython.html` package has been deprecated. You should import from `notebook` instead. `IPython.html.widgets` has moved to `ipywidgets`.
  "`IPython.html.widgets` has moved to `ipywidgets`.", ShimWarning)

df.head()

sns.set_context("talk")
sns.factorplot(data=df, x='year', y='lifeExp', hue='continent', size=8)

<seaborn.axisgrid.FacetGrid at 0x7f57901cfa20>

sns.regplot(data=df, x='year', y='gdpPercap', fit_reg=False)

<matplotlib.axes._subplots.AxesSubplot at 0x7fbabf5472b0>

sns.lmplot(data=df, x='year', y='gdpPercap', hue='continent')

<seaborn.axisgrid.FacetGrid at 0x7f579197f710>

sns.lmplot(data=df, x='year', y='gdpPercap', row='continent')

<seaborn.axisgrid.FacetGrid at 0x7f578da9eb38>

sns.factorplot(data=df, x='continent', y='gdpPercap', kind='bar')

<seaborn.axisgrid.FacetGrid at 0x7f5791ac21d0>

g = sns.FacetGrid(df, col='continent', row='year')
g.map(plt.hist, 'lifeExp')

<seaborn.axisgrid.FacetGrid at 0x7f578d928eb8>