1
2

import pandas as pd
pd.read_csv('./data.csv')

 1
 2
 3
 4
 5
 6
 7
 8
 9
10

import pydata_google_auth
import pydata_google_auth.cache
from google.cloud.bigquery import Client
credentials = pydata_google_auth.get_user_credentials(
    scopes = ['https://www.googleapis.com/auth/bigquery'],
)
client = Client(project="myprojectname", credentials=credentials)

job = client.query("SELECT * FROM mytable;")
df = job.to_dataframe()

1
2

df.head() # 先頭
df.tail() # 末端

1

df.info()

1
2
3
4
5
6
7
8
9

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 891 entries, 0 to 890
Data columns (total 12 columns):
Name           891 non-null object
Sex            891 non-null object
Age            714 non-null float64
...
dtypes: float64(2), int64(5), object(5)
memory usage: 83.6+ KB

1

df.describe()

1

df.describe(include=['O'])

1

df.unique()

1

df.T

1

df[["hoge", "fuga"]].sort_values(by='hoge', ascending=False)

1

df.drop(columns=['hoge'])

1

df[df['Name'].isin(['Alice','Bob'])]

1

df.query('name == "Alice" or name == "Bob"', engine='python')

1

df.rename(columns={'A': 'Col_1'})

1

df[["hoge", "fuga"]].groupby(['hoge'], as_index=False).mean()

1

SELECT hoge, AVG(fuga) AS fuga FROM df GROUP BY hoge;

1

pd.merge(df_a, df_b, on='column1', how='inner')

1

SELECT * FROM df_a INNER JOIN df_b ON df_a.column1 = df_b.column1;

1

df['Flag'] = df['Flag'].map({'True': 1, 'False': 0}).astype(int)

1

df['Diff'] = df[['X','Y'].apply(lambda a: a['X'] - a['Y'], axis=1)

1

df=df.fillna("None")

1

pd.crosstab(df['Job'],df['Sex'])

1
2
3

from sklearn.preprocessing import StandardScaler
scaler = StandardScaler()
scaler.fit(df)

1
2
3

from sklearn.preprocessing import RobustScaler
scaler = RobustScaler(quantile_range(25.0,75.0))
scaler.fit(df)

1
2
3

from sklearn.preprocessing import MinMaxScaler
scaler = MinMaxScaler(feature_range(0,1))
scaler.fit(df)

1
2
3

from sklearn.preprocessing import LabelEncoder
le = LabelEncoder()
df = le.fit_transform(df)

1

df=pd.get_dummies(df, columns=['sex'])

1

df=pd.get_dummies(df, drop_first=True)

1
2
3

import seaborn as sns
import japanize_matplotlib # 文字化け対応
sns.pairplot(df)

1

sns.FacetGrid(df, col="time", row="sex").map(sns.scatterplot, "total_bill", "tip")

1

df.plot.scatter(x='length',y='width')

1

sns.histplot(data=df, x='Length', stat='percent')

1

sns.distplot(data=df, x='Length', kind="ecdf")

1

sns.distplot(data=df, x='Width', y='Length')

1

sns.heatmap(df)

1
2
3
4
5
6

from scipy import stats
x=500 #事象の発生回数
n=1000 #試行回数
a=0.5 #発生確率の帰無仮説
p = stats.binom_test(x, n, a)
print(p)

1
2
3

from scipy import stats
result = stats.mannwhitneyu(df['A'],df['B'],alternative='two-sided')
print(result.pvalue)

1
2
3
4

from scipy import stats
df=pd.crosstab(df['Job'],df['Sex'])
x2, p, dof, expected = stats.chi2_contingency(df)
print(p)

1
2
3
4
5
6
7

from lifelines import CoxPHFitter

cph = CoxPHFitter()
cph.fit(rossi, duration_col='lifetime', event_col='is_dead')

cph.print_summary()
cph.plot()

1
2
3
4
5

from sklearn.model_selection import train_test_split
X=df.drop(index='Target')
y=df['Target']

train_X, test_X, train_y, test_y = train_test_split(X,y)

1
2
3
4
5
6
7
8

from sklearn.model_selection import cross_validate

clf = RandomForestClassifier()
X=df.drop(index='Target')
y=df['Target']

scores = cross_validate(clf, X, y, scoring=['accuracy','precision','recall','f1'], cv=5)
print(scores)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

from sklearn.linear_model import LogisticRegression

logreg = LogisticRegression()
logreg.fit(X_train, y_train)

y_pred = logreg.predict(X_test)
print(logreg.score(X_train, y_train))

coeff_df = pd.DataFrame(train_df.columns.delete(0))
coeff_df.columns = ['Feature']
coeff_df["Correlation"] = pd.Series(logreg.coef_[0])

coeff_df.sort_values(by='Correlation', ascending=False)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10

from sklearn.tree import DecisionTreeClassifier,plot_tree
import matplotlib.pyplot as plt

clf = DecisionTreeClassifier()
clf.fit(X_train, y_train)
y_pred = clf.predict(X_test)
print(clf.score(X_train, y_train))

plot_tree(clf,max_depth=2)
plt.show()