import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import time
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import OneHotEncoder
from sklearn.preprocessing import PolynomialFeatures
from sklearn.utils import shuffle
from sklearn.linear_model import LinearRegression
import tensorflow as tf
import xgboost as xgb
import lightgbm as lgb

global path
path = ''

def readfiles():
    '''Read the files train_2016_v2.csv and properties_2016.csv'''
    print('Reading Files...')
    tic=time.time()
    train = pd.read_csv(path +'train_2016_v2.csv')
    train['transactiondate']=train['transactiondate'].apply(lambda x : float(x[5:7]))
    train['transactionyear']=0.0
    df = pd.read_csv(path +'properties_2016.csv',low_memory=False)
    print('Time elapsed: ',time.time() - tic)
    return train, df

def readfiles2017():
    '''Read the files train_2017.csv and properties_2017.csv'''
    print('Reading Files...')
    tic=time.time()
    train = pd.read_csv(path +'train_2017.csv')
    train['transactiondate']=train['transactiondate'].apply(lambda x : float(x[5:7]))
    train['transactionyear']=1.0
    df = pd.read_csv(path +'properties_2017.csv',low_memory=False)
    print('Time elapsed: ',time.time() - tic)
    return train, df

def preprocess(df):
    '''preprocess the data'''
    print('Preprocessing Data...')
    params = {
    'collist':[...], #featurenames hidden
    'numlist':[...], #featurenames hidden
    'temlist':[...], #featurenames hidden
    'addlist':[...], #featurenames hidden
    'enc':OneHotEncoder(handle_unknown='ignore')
         }
	#hidden 
	#perform preprocess on features including adding and dropping features and fillna().
    return df.reset_index(), params

def mergeframes(train, df):
    '''As the dataframe df does not contain the target logerror, we have to merge the two files to create the training data'''
    print('Merging DataFrames...')
    tic = time.time()
    dftrain = df.merge(train,how = 'inner',left_on = 'parcelid', right_on = 'parcelid')
    print('Done! Time elapsed:', time.time() - tic)
    return dftrain

def cropframes(df, hi=0.5, lo=-0.5):
    '''dropping the outliers'''
    b=df[df['logerror']<hi]
    return b[b['logerror']>lo]

def onehot(df, params):
    '''Fitting the OneHotEncoder and transforming to the data'''
    print('Fitting One Hot Encoding...')
    tic = time.time()
    enc = params['enc']
    collist = params['collist']
    numlist = params['numlist']
    try:
        remaining=[col for col in list(df.columns) if col not in ['parcelid']+numlist+collist]
    except:
        print('Input Dataframe Columns Error')
        return None
    df_onehot=pd.DataFrame(enc.fit_transform(df[collist].astype(int)).toarray(), index = df.index)
    df_onehot=pd.concat([df[['parcelid']], df[numlist], df_onehot, df[remaining]], axis=1)
    print('Done! Time elapsed:', time.time() - tic)
    return df_onehot
	
def onehottransform(df,params):
    '''One-Hot Transforming the data'''
    print('Transforming One Hot Encoding...')
    tic = time.time()
    enc = params['enc']
    collist = params['collist']
    numlist = params['numlist']
    remaining=[col for col in list(df.columns) if col not in ['parcelid']+numlist+collist]
    df_onehot=pd.DataFrame(enc.transform(df[collist].astype(int)).toarray(), index = df.index)
    df_onehot=pd.concat([df[['parcelid']], df[numlist], df_onehot, df[remaining]], axis=1)
    print('Done! Time elapsed:', time.time() - tic)
    return df_onehot
	
def traintestsplit(df, params, percent=0.98,random_state=0):
    '''Performing One Hot and train test split'''
    enc = params['enc']
    collist = params['collist']
    numlist = params['numlist']
    X = df[numlist+collist+['transactiondate','transactionyear']]
    Xonehot = onehot(df, params).drop(['parcelid','logerror'], axis=1)
    y = df['logerror']
    X_train, X_test, y_train, y_test = train_test_split(X,y, train_size=percent, test_size=1-percent, random_state=random_state)
    X_onehot_train, X_onehot_test, _, _ = train_test_split(Xonehot,y, train_size=percent, test_size=1-percent, random_state=random_state)
    return X_train, X_test, X_onehot_train, X_onehot_test, y_train, y_test
	
def neuralnetworkmodel(x):
    '''The model for neural network. x: placeholder, output a tensor'''
    numcol = int(x.get_shape()[1])
    init = tf.contrib.layers.xavier_initializer()
    activation = tf.nn.elu
    net = tf.contrib.layers.fully_connected(x,numcol,activation_fn=activation, weights_initializer=init)
    net = tf.contrib.layers.fully_connected(net,numcol//2,activation_fn=activation, weights_initializer=init)
    net = tf.contrib.layers.fully_connected(net,numcol//2,activation_fn=activation, weights_initializer=init)
    net = tf.contrib.layers.fully_connected(net,numcol//4,activation_fn=activation, weights_initializer=init)
    net = tf.contrib.layers.fully_connected(net,numcol//4,activation_fn=activation, weights_initializer=init)
    net = tf.contrib.layers.fully_connected(net,numcol//8,activation_fn=activation, weights_initializer=init)
    net = tf.contrib.layers.fully_connected(net,numcol//8,activation_fn=activation, weights_initializer=init)
    net = tf.contrib.layers.fully_connected(net,numcol//16,activation_fn=activation, weights_initializer=init)
    net = tf.contrib.layers.fully_connected(net,numcol//16,activation_fn=activation, weights_initializer=init)
    return tf.contrib.layers.fully_connected(net,1,activation_fn=None, weights_initializer=init)
	
def neuralnetwork(X_train,X_test, y_train, y_test, epochs=20,learning_rate=0.001, batch_size=128,save=''):
    '''Training the neural network'''
    print('Training neural network...')
    tic=time.time()
    numcol = X_train.shape[1]
    tf.reset_default_graph()
    x = tf.placeholder('float',shape=[None,numcol])
    pred=neuralnetworkmodel(x)
    output = tf.placeholder('float',shape=None)
    error = tf.losses.absolute_difference(labels=tf.expand_dims(output,1), predictions=pred)
    train_step = tf.train.AdamOptimizer(learning_rate = learning_rate).minimize(error)
    
    saver = tf.train.Saver()
    numberofdata = len(X_train)
    numberoftestdata = len(X_test)
    
    with tf.Session() as session:
        session.run(tf.global_variables_initializer())
        for i in range(epochs):
	    #Train the weights and record the training error
            trainerror=[]
            for j in range(numberofdata//batch_size):
                er, _= session.run([error,train_step],feed_dict={x:X_train[j*batch_size:(j+1)*batch_size], output:y_train[j*batch_size:(j+1)*batch_size]})
                trainerror.append(er)
	    #record the test error
            testerror = []
            for j in range(numberoftestdata//batch_size):
                testerror.append( session.run(error, feed_dict={x:X_test[j*batch_size:(j+1)*batch_size], output:y_test[j*batch_size:(j+1)*batch_size]}))
            #print('Epoch {:4}: Train error: {:12.8f}    Test error: {:12.8f}'.format(i+1, np.mean(trainerror), np.mean(testerror)))
        saver.save(session,'weights_nn'+save)
        print('Done! Time Elapsed: ', time.time() - tic)
    return 'weights_nn'+save 	

def get_neuralnetwork_result(X, weightname):
    '''Make predictions using our model. Input weightname is the name of the saved weights'''
    numcol = X.shape[1]
    tf.reset_default_graph()
    x = tf.placeholder('float',shape=[None,numcol])
    pred =neuralnetworkmodel(x)
    saver = tf.train.Saver()
    numberofdata=len(X)
    batch_size=10000
    
    with tf.Session() as session:
        session.run(tf.global_variables_initializer())
        saver.restore(session,weightname)
        predlist = np.array([])
        j=0
        while j<numberofdata:
            output = session.run(pred,feed_dict={x:X[j:j+batch_size]})
            predlist=np.append(predlist,output.reshape(-1))
            j=j+batch_size
    return predlist

def xgBoost(X_train, X_test, y_train, y_test, xgb_params, numround):
    print('Performing XGBoost...')
    tic = time.time()
    dtrain = xgb.DMatrix(X_train, y_train)
    dtest = xgb.DMatrix(X_test)
    model = xgb.train(dict(xgb_params, silent=1), dtrain, num_boost_round=numround)
    xgb_pred_train = model.predict(dtrain)
    xgb_pred_test = model.predict(dtest)
    print('Training error: {:8.4f}  Test error: {:8.4f}  Time elapsed: {:8.4f}'.format(
        np.mean(np.abs(xgb_pred_train - y_train)),
        np.mean(np.abs(xgb_pred_test - y_test)),
        time.time() - tic))
    return model
	
def lgBoost(X_train, X_test, y_train, y_test, lgbparams, numround):
    tic=time.time()
    print('Performing LightGBM...')
    d_train = lgb.Dataset(X_train, label=y_train)
    d_valid = lgb.Dataset(X_test, label=y_test)
    model = lgb.train(lgbparams, d_train, numround)
    pred_test = model.predict(X_test)
    pred_train = model.predict(X_train)
    print('Training error: {:8.4f}  Test error: {:8.4f}  Time elapsed: {:8.4f}'.format(
            np.mean(np.abs(pred_train - y_train)),
            np.mean(np.abs(pred_test - y_test)),
            time.time() - tic))
    return model
	
def ensemble_training(X_train, X_test, X_train_scaled, X_test_scaled, y_train, y_test,num=0):
    '''Train 5 different models'''
    weightname=neuralnetwork(X_train_scaled, X_test_scaled, y_train, y_test,epochs=10,save=str(num))
    xgb_params1 = {
        'eta': 0.020,
        'max_depth': 6,
        'subsample': 0.80,
        'objective': 'reg:linear',
        'eval_metric': 'mae',
        'base_score': np.median(y_train),
        'silent': 1
    }
    xgb_params2 = {
        'eta': 0.037,
        'max_depth': 5,
        'subsample': 0.80,
        'objective': 'reg:linear',
        'eval_metric': 'mae',
        'lambda': 0.8,   
        'alpha': 0.4, 
        'base_score': np.median(y_train),
        'silent': 1
    }
    lgbparams1 = {
        'max_bin' : 10,
        'learning_rate' : 0.03,
        'boosting_type' : 'gbdt',
        'objective' : 'regression',
        'metric' : 'l1',
        'sub_feature': 0.7,
        'bagging_fraction' : 0.85,
        'bagging_freq' : 40,
        'num_leaves' : 80,        
        'min_data' : 500,       
        'min_hessian' : 0.05,  
        'verbose' : 0
    }
    lgbparams2 = {
        'max_bin' : 10,
        'learning_rate' : 0.03,
        'boosting_type' : 'gbdt',
        'objective' : 'regression',
        'metric' : 'l1',
        'sub_feature': 0.5,    
        'bagging_fraction' : 0.85, 
        'bagging_freq' : 40,
        'num_leaves' : 512,       
        'min_data' : 500,      
        'min_hessian' : 1,   
        'verbose' : 0
    }
    xgb1_model = xgBoost(X_train, X_test, y_train, y_test, xgb_params1, numround=325)
    xgb2_model = xgBoost(X_train, X_test, y_train, y_test, xgb_params2, numround=125)
    lgb1_model = lgBoost(X_train, X_test, y_train, y_test, lgbparams1, numround=100)
    lgb2_model = lgBoost(X_train, X_test, y_train, y_test, lgbparams2, numround=175)
    return weightname, xgb1_model, xgb2_model, lgb1_model, lgb2_model
	
def get_predictions(X, X_scaled, weightname, xgb1_model, xgb2_model, lgb1_model, lgb2_model, showtext=False):
    '''Get predictions for the five different models. Return a DataFrame containing predictions'''
    if showtext:
        print('Beginning Prediction: Neural Network...')
    tic=time.time()
    neuralnetwork_pred = get_neuralnetwork_result(X_scaled, weightname)
    toc = time.time()
    if showtext:
        print('Done. Time Elapsed: {:8.4f}  Beginning Prediction: XGBoost...'.format(toc - tic))
    tic = toc
    xgb1_pred = xgb1_model.predict(xgb.DMatrix(X))
    toc=time.time()
    if showtext:
        print('Done. Time Elapsed: {:8.4f}  Beginning Prediction: XGBoost2...'.format(toc - tic))
    tic = toc
    xgb2_pred = xgb2_model.predict(xgb.DMatrix(X))
    toc=time.time()
    if showtext:
        print('Done. Time Elapsed: {:8.4f} Beginning Prediction: LightGBM...'.format(toc - tic))
    tic =toc
    lgb1_pred = lgb1_model.predict(X)
    toc=time.time()
    if showtext:
        print('Done. Time Elapsed: {:8.4f} Beginning Prediction: LightGBM2...'.format(toc - tic))
    tic =toc
    lgb2_pred = lgb2_model.predict(X)
    toc=time.time()    
    if showtext:
        print('Done. Time Elapsed: {:8.4f} '.format(toc - tic))
	
    pred = pd.DataFrame()
    pred['neuralnetwork'] = neuralnetwork_pred
    pred['xgboost'] = xgb1_pred
    pred['xgboost2'] = xgb2_pred
    pred['lightgbm'] = lgb1_pred
    pred['lightgbm2'] = lgb2_pred
    return pred
	
def quadratic_regression(pred, y, pred_test, y_test, lamb=1e-2, alpha=1e-3):
    '''Input a DataFrame of predictions and perform a quadratic regression'''
    poly = PolynomialFeatures(2)
    Xp=poly.fit_transform(np.array(pred))
    Xp_test=poly.transform(np.array(pred_test))
    
    #initialize the weights
    n_features = Xp.shape[1]
    n_deg_1_features = pred.shape[1]
    weight_init = np.zeros([n_features,1])
    weight_init[1:1+n_deg_1_features,0] = 1/n_deg_1_features
    
    #Model for quadratic regression (lasso regression on quadratic features)
    XP = tf.placeholder('float', shape=[None, n_features])
    YOUT = tf.placeholder('float', shape = None)
    W = tf.Variable(weight_init, dtype=tf.float32)
    b = tf.Variable(tf.zeros([1,1]))
    out = tf.add(tf.matmul(XP,W), b)
    loss_reg = lamb*tf.reduce_mean(tf.multiply(W,W))
    loss_data = tf.reduce_mean(tf.abs(out-tf.expand_dims(YOUT,1))) 
    loss = loss_data + loss_reg
    grad_step = tf.train.GradientDescentOptimizer(alpha).minimize(loss)
    
    dcount = 0
    lasterror = 999999
    with tf.Session() as sess:
        sess.run(tf.global_variables_initializer())
        for i in range(100000):
            _, er = sess.run([grad_step, loss], feed_dict={XP: Xp, YOUT: y})   
            testerror=sess.run(loss_data, feed_dict={XP: Xp_test, YOUT: y_test})
            if lasterror < testerror:
                dcount = dcount + 1
            else:
                dcount = 0
            lasterror = testerror
	    #early stopping
            if dcount > 5:
                break
        Wout=np.squeeze(sess.run(W))
        bout=np.squeeze(sess.run(b))
	#As the first quadratic weight is a constant, the bias b is added to the first quadratic weight.
        Wout[0]=Wout[0]+bout
        return Wout
		
def get_predictions_quadratic(pred, W):
    '''Getting prediction in the final quadratic model'''
    poly = PolynomialFeatures(2)
    Xp=poly.fit_transform(np.array(pred))
    return np.dot(Xp,W)
	
def finalstage(X_train, X_test, X_train_scaled, X_test_scaled, y_train, y_test,num=0):
    #Train the ensemble - five separate models
    weightname, xgb1_model, xgb2_model, lgb1_model, lgb2_model = ensemble_training(X_train, X_test, X_train_scaled, X_test_scaled, y_train, y_test,num=num)

    #Get the prediction using the train set
    pred = get_predictions(X_train, X_train_scaled, weightname, xgb1_model, xgb2_model, lgb1_model, lgb2_model)
    
    #Get the prediction using the test set
    pred_test = get_predictions(X_test, X_test_scaled, weightname, xgb1_model, xgb2_model, lgb1_model, lgb2_model)
    print('Performing ensemble training...')
    tic=time.time()
    
    #Train the quadratic regression using the test set A as the training set, and test set B as the test set 
    W=quadratic_regression(pred_test[:len(pred_test)//2], y_test[:len(y_test)//2], pred_test[len(pred_test)//2:], y_test[len(y_test)//2:], lamb=0, alpha = 3e-2)
    print('Done! Time Elapsed:',time.time() - tic)
    return weightname, xgb1_model, xgb2_model, lgb1_model, lgb2_model, W
	
train1, df1_ = readfiles()
train2, df2_ = readfiles2017()
df1 = df1_.copy()
df2 = df2_.copy()
df1, params1 = preprocess(df1)
df2, params2 = preprocess(df2)
dfall1=mergeframes(train1, df1)
dfall2=mergeframes(train2, df2)
dfall = pd.concat([dfall1, dfall2]).reset_index().drop('index', axis=1)
dfall = cropframes(dfall, lo=-0.35, hi=0.35)
X_train, X_test, X_onehot_train, X_onehot_test, y_train, y_test = traintestsplit(dfall, params2)
scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_onehot_train)
X_test_scaled = scaler.transform(X_onehot_test)
weightname, xgb1_model, xgb2_model, lgb1_model, lgb2_model, W = finalstage(X_train, X_test, X_train_scaled, X_test_scaled, y_train, y_test)
for tdate in [10.0,11.0,12.0]:
    for tyear in [0.0,1.0]:
        df2['transactiondate']=tdate
        df2['transactionyear']=tyear
        tic = time.time()
        Xfinal = onehottransform(df2,params2).drop('parcelid', axis=1)
        Xf = scaler.transform(Xfinal)
        pred = get_predictions(df2.drop('parcelid', axis=1), Xf, weightname, xgb1_model, xgb2_model, lgb1_model, lgb2_model, showtext=True)
        final_prediction=get_predictions_quadratic(pred, W)
        print('Time elapsed: ', time.time() - tic)