2015-04-22 22:35:16 +00:00
|
|
|
from __future__ import absolute_import
|
|
|
|
from __future__ import print_function
|
|
|
|
|
|
|
|
import numpy as np
|
|
|
|
import pandas as pd
|
|
|
|
|
|
|
|
from keras.models import Sequential
|
|
|
|
from keras.layers.core import Dense, Dropout, Activation
|
|
|
|
from keras.layers.normalization import BatchNormalization
|
|
|
|
from keras.layers.advanced_activations import PReLU
|
|
|
|
from keras.utils import np_utils, generic_utils
|
|
|
|
|
|
|
|
from sklearn.preprocessing import LabelEncoder
|
|
|
|
from sklearn.preprocessing import StandardScaler
|
|
|
|
|
|
|
|
'''
|
|
|
|
This demonstrates how to reach a score of 0.4890 (local validation)
|
|
|
|
on the Kaggle Otto challenge, with a deep net using Keras.
|
|
|
|
|
|
|
|
Compatible Python 2.7-3.4
|
|
|
|
|
|
|
|
Recommended to run on GPU:
|
|
|
|
Command: THEANO_FLAGS=mode=FAST_RUN,device=gpu,floatX=float32 python kaggle_otto_nn.py
|
2015-04-22 22:39:06 +00:00
|
|
|
On EC2 g2.2xlarge instance: 19s/epoch. 6-7 minutes total training time.
|
2015-04-22 22:35:16 +00:00
|
|
|
|
|
|
|
Best validation score at epoch 21: 0.4881
|
|
|
|
|
|
|
|
Try it at home:
|
|
|
|
- with/without BatchNormalization (BatchNormalization helps!)
|
|
|
|
- with ReLU or with PReLU (PReLU helps!)
|
|
|
|
- with smaller layers, largers layers
|
|
|
|
- with more layers, less layers
|
|
|
|
- with different optimizers (SGD+momentum+decay is probably better than Adam!)
|
2015-05-12 20:07:50 +00:00
|
|
|
|
|
|
|
Get the data from Kaggle: https://www.kaggle.com/c/otto-group-product-classification-challenge/data
|
2015-04-22 22:35:16 +00:00
|
|
|
'''
|
|
|
|
|
|
|
|
np.random.seed(1337) # for reproducibility
|
|
|
|
|
|
|
|
def load_data(path, train=True):
|
|
|
|
df = pd.read_csv(path)
|
|
|
|
X = df.values.copy()
|
|
|
|
if train:
|
|
|
|
np.random.shuffle(X) # https://youtu.be/uyUXoap67N8
|
|
|
|
X, labels = X[:, 1:-1].astype(np.float32), X[:, -1]
|
|
|
|
return X, labels
|
|
|
|
else:
|
|
|
|
X, ids = X[:, 1:].astype(np.float32), X[:, 0].astype(str)
|
|
|
|
return X, ids
|
|
|
|
|
|
|
|
def preprocess_data(X, scaler=None):
|
|
|
|
if not scaler:
|
|
|
|
scaler = StandardScaler()
|
|
|
|
scaler.fit(X)
|
|
|
|
X = scaler.transform(X)
|
|
|
|
return X, scaler
|
|
|
|
|
2015-05-10 10:02:36 +00:00
|
|
|
def preprocess_labels(labels, encoder=None, categorical=True):
|
2015-04-22 22:35:16 +00:00
|
|
|
if not encoder:
|
|
|
|
encoder = LabelEncoder()
|
|
|
|
encoder.fit(labels)
|
|
|
|
y = encoder.transform(labels).astype(np.int32)
|
|
|
|
if categorical:
|
|
|
|
y = np_utils.to_categorical(y)
|
|
|
|
return y, encoder
|
|
|
|
|
|
|
|
def make_submission(y_prob, ids, encoder, fname):
|
|
|
|
with open(fname, 'w') as f:
|
|
|
|
f.write('id,')
|
2015-04-24 18:05:09 +00:00
|
|
|
f.write(','.join([str(i) for i in encoder.classes_]))
|
2015-04-22 22:35:16 +00:00
|
|
|
f.write('\n')
|
|
|
|
for i, probs in zip(ids, y_prob):
|
|
|
|
probas = ','.join([i] + [str(p) for p in probs.tolist()])
|
|
|
|
f.write(probas)
|
|
|
|
f.write('\n')
|
|
|
|
print("Wrote submission to file {}.".format(fname))
|
|
|
|
|
|
|
|
|
|
|
|
print("Loading data...")
|
|
|
|
X, labels = load_data('train.csv', train=True)
|
|
|
|
X, scaler = preprocess_data(X)
|
|
|
|
y, encoder = preprocess_labels(labels)
|
|
|
|
|
|
|
|
X_test, ids = load_data('test.csv', train=False)
|
2015-04-22 23:34:01 +00:00
|
|
|
X_test, _ = preprocess_data(X_test, scaler)
|
2015-04-22 22:35:16 +00:00
|
|
|
|
|
|
|
nb_classes = y.shape[1]
|
|
|
|
print(nb_classes, 'classes')
|
|
|
|
|
|
|
|
dims = X.shape[1]
|
|
|
|
print(dims, 'dims')
|
|
|
|
|
|
|
|
print("Building model...")
|
|
|
|
|
|
|
|
model = Sequential()
|
|
|
|
model.add(Dense(dims, 512, init='glorot_uniform'))
|
|
|
|
model.add(PReLU((512,)))
|
|
|
|
model.add(BatchNormalization((512,)))
|
|
|
|
model.add(Dropout(0.5))
|
|
|
|
|
|
|
|
model.add(Dense(512, 512, init='glorot_uniform'))
|
|
|
|
model.add(PReLU((512,)))
|
|
|
|
model.add(BatchNormalization((512,)))
|
|
|
|
model.add(Dropout(0.5))
|
|
|
|
|
|
|
|
model.add(Dense(512, 512, init='glorot_uniform'))
|
|
|
|
model.add(PReLU((512,)))
|
|
|
|
model.add(BatchNormalization((512,)))
|
|
|
|
model.add(Dropout(0.5))
|
|
|
|
|
|
|
|
model.add(Dense(512, nb_classes, init='glorot_uniform'))
|
|
|
|
model.add(Activation('softmax'))
|
|
|
|
|
|
|
|
model.compile(loss='categorical_crossentropy', optimizer="adam")
|
|
|
|
|
|
|
|
print("Training model...")
|
|
|
|
|
2015-05-12 20:07:50 +00:00
|
|
|
model.fit(X, y, nb_epoch=20, batch_size=128, validation_split=0.15)
|
2015-04-22 22:35:16 +00:00
|
|
|
|
|
|
|
print("Generating submission...")
|
|
|
|
|
|
|
|
proba = model.predict_proba(X_test)
|
|
|
|
make_submission(proba, ids, encoder, fname='keras-otto.csv')
|
|
|
|
|