Feed-forward neural network not training with Keras function generators

machine_learning
deep_learning
data_science
python

#1

I am using feed-forward neural network for a classification task. My data is 1 million examples for 9 classes (imbalanced). Due to memory constraint in Keras, I used function generator to generate batches automatically with a batch size of 200. I trained a simple model with 3 hidden layers with ReLU activation function. Input layer is 39 dimensional MFCCs and output is 9 classes. This model worked fine when I used a subset of this huge data (Sure!) but now, while using function generator i.e model.fit_generator, I see that training accuracy is just wandering around and validation accuracy is just too low. Looks like the model is not learning at all. What might be the possible reasons for this behaviour?

about data

Data : Speech

subset of huge data mentioned was totally clean but I generated 1 million examples from 1300 examples using data augmentation techniques like equalization, time stretch, time compression, noises, reverb etc


#2

Hi @mounika84,

As you have large number of datapoints, try to build a deeper network with more number of hidden layers and nodes. You can also change the optimizers while compiling the model.


#3

@PulkitS Thanks, I have also tried training with deeper network. I used a 9 layered network with 500 units in each, ReLU actiavtion function and Adam optimizer. Still, no change in behaviour of the model.![re_3|690x215]


#4

Hi @mounika84,

Looks like the model is underfitting as the training accuracy is very less. Can you please share the code that you have used also how many datapoints are there in the training set?


#5
@PulkitS : Please find this code 
import numpy as np
import os
import random
import keras
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation
from keras.utils import np_utils
from keras.optimizers import SGD,Adam
from keras.wrappers.scikit_learn import KerasClassifier
from sklearn.model_selection import cross_val_score
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import StratifiedKFold
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline
from matplotlib import pyplot
from sklearn.model_selection import train_test_split
from keras.models import load_model
from keras.models import model_from_json
from keras.callbacks import Callback
from keras.callbacks import ModelCheckpoint
from keras.callbacks import EarlyStopping
from sklearn.metrics import classification_report, confusion_matrix
from numpy.random import seed
seed(1)    

THEANO_FLAGS='device=cuda,floatX=float32'
epochs=200
exp_no=3
train_file_name='train_without_time_equalized_reverb_noise.txt'
lines=open(train_file_name).read().splitlines()
#train_generator(lines,labels)
labels={'user_1':0,'user_2':1,'user_3':2,'user_4':3,'user_5':4,'user_6':5,'user_7':6,'user_8':7,'user_9':8}
nb_classes=9
batch_size=200
test_file_name='val_without_time_equalized_reverb_noise.txt'
test_lines=open(test_file_name).read().splitlines()    

def get_input(path):
  feat=np.load(path)
  return(feat)    

def get_output(path,label_file=None):
   spkr_name=path.split('/')[-2]
   lab=labels[spkr_name]
   lab=np.int64(lab)
   #lab=keras.utils.to_categorical(lab,nb_classes)
   return(lab)    

def train_generator(files,labels,batch_size=200):
   while True:
      # select files for the batch
      batch_paths=np.random.choice(a=files,size=batch_size)
      batch_input=[]
      batch_output=[]
      b=0
      # read in each input and get labels
      for input_path in batch_paths:
        b+=1
        input_name=get_input(input_path)
        # MVN
        mean=np.mean(input_name,axis=0)
        std=np.std(input_name,axis=0)+0.00001
        input_name=(input_name-mean)/std
        output=get_output(input_path,labels)
        output=np.repeat(output,input_name.shape[0])
        output=keras.utils.to_categorical(output,nb_classes)
        #print(input_name.shape)
        #print(output.shape)
        #batch_input+=[input_name]
        batch_input.append(input_name)
        #batch_output+=[output]
        batch_output.append(output)
        # return a tuple of (input,output) to feed the network
        batch_x=np.array(batch_input)
        batch_y=np.array(batch_output)
        yield(batch_x[b-1],batch_y[b-1])    

def validation_generator(files,labels,batch_size=1):
  while True:
    batch_paths=np.random.choice(a=files,size=batch_size)
    batch_input=[]
    batch_output=[]
    c=0
    for input_path in batch_paths:
       c+=1
       input_name=get_input(input_path)
       # MVN
       mean=np.mean(input_name)
       std=np.std(input_name)+0.00001
       input_name=(input_name-mean)/std
       output=get_output(input_path,labels)
       output=np.repeat(output,input_name.shape[0])
       output=keras.utils.to_categorical(output,nb_classes)
       batch_input.append(input_name)
       batch_output.append(output)
       batch_x=np.array(batch_input)
       batch_y=np.array(batch_output)
       yield(batch_x[c-1],batch_y[c-1])    


# model    

model=Sequential()
model.add(Dense(300,activation='relu',input_dim=39))
model.add(Dropout(0.2))
model.add(Dense(200,activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(100,activation='relu'))
model.add(Dense(9,activation='softmax'))    

print(model.summary())
sgd=SGD(lr=0.001,decay=1e-6,momentum=0.9,nesterov=True) # Stochastic Gradient Descent, decay is the learning rate scheduler    

model.compile(loss='categorical_crossentropy',optimizer=sgd,metrics=['accuracy'])
#model.compile(loss='categorical_crossentropy',optimizer='Adam',metrics=['accuracy'])    

model_dir='/media//results/'    

filepath=model_dir+'exp_'+str(exp_no)+'/weights.{epoch:02d}-{val_loss:.2f}-{val_acc:.2f}.hdf5' # Saving the model weights at each epoch that can be loaded later
checkpointer=ModelCheckpoint(filepath=filepath,monitor='val_loss',verbose=0,save_best_only=False,mode='auto')    

model.fit_generator(train_generator(lines,labels),steps_per_epoch=994931/batch_size,validation_data=validation_generator(test_lines,labels),validation_steps=4500/batch_size,epochs=epochs,verbose=1,shuffle=True,initial_epoch=0,callbacks=[checkpointer])

@PulkitS Number of training examples : 994931
test examples : 500 utterances per class i.e 4500

input : 994931 X 39
output : 9 units