cmput656_full_data.py

# -*- coding: utf-8 -*-
"""CMPUT656 Full Data.ipynb

Automatically generated by Colaboratory.
"""

import pandas as pd
import os
import numpy as np
import matplotlib.pyplot as plt
# from keras.callbacks import Callback
from sklearn.metrics import confusion_matrix, f1_score, precision_score, recall_score, classification_report

# from google.colab import drive
# drive.mount('/drive')

# path = '/drive/My Drive/Colab Notebooks/'
# os.chdir(path)
# print(os.getcwd())

import tokenization

import tensorflow as tf
import tensorflow_hub as hub
from tensorflow.keras.utils import to_categorical
from sklearn import preprocessing
from sklearn.model_selection import train_test_split
from sklearn.model_selection import KFold
from sklearn.utils import shuffle
from sklearn.metrics import f1_score

print(tf.__version__)

device_name = tf.test.gpu_device_name()
print(device_name)
if device_name != '/device:GPU:0':
  raise SystemError('GPU device not found')
print('Found GPU at: {}'.format(device_name))

# gpu_info = !nvidia-smi
# gpu_info = '\n'.join(gpu_info)
# if gpu_info.find('failed') >= 0:
#   print('Not connected to a GPU')
# else:
#   print(gpu_info)

m_url = 'https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/4'
# m_url = "/home/rsaha/scratch/re_656_data/bert_en_uncased_L-12_H-768_A-12_4"
bert_layer = hub.KerasLayer(m_url, trainable=False)

vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case)



data_path = "/home/rsaha/projects/def-afyshe-ab/rsaha/projects/RE_656/Processed Data/"
relations_path = data_path + 'Input_all_29_relation.tsv'

train_data = pd.read_csv(relations_path, encoding='utf-8', sep = '\t')

print(train_data.shape[0])

#drop few tweets
# remove_n = round(0.4 * train_data.shape[0])
# drop_indices = np.random.choice(train_data.index, remove_n, replace=False)
# train_data = train_data.drop(drop_indices)
# train_data.shape

train_data.fillna("", inplace = True)

# Shuffle data so that there is a higher chance of the train and test data being from the same distribution.
train_data = shuffle(train_data, random_state = 1)

train_data.head()

# # Now read the rows, convert them into strings and then only keep the unique ones.
# sentences_and_labels =  np.array([[' '.join(map(str, row[:-1].tolist())).strip(), row[-1]] for row in train_data.iloc[:,:].values])

labels = train_data.iloc[:,-1].values
sentences = train_data.iloc[:,:-1].values.tolist()

sentences = [' '.join(sent).strip() for sent in sentences]
# c = 0
# for row in train_data.iloc[:,:].values:
#     label = row[-1]
#     c += 1
#     row_retrieved = row[:-1].tolist()
#     s = ' '.join(row_retrieved).strip()
#     del row_retrieved
#     # break
#     # s = ' '.join(map(str, row[:-1].tolist())).strip()

#     temp = np.array([s, label])
#     del label
#     del s
#     del row
#     sentences_and_labels.append(temp)
#     del temp

# # print(sentences_and_labels.shape)
# sentences = np.array(sentences_and_labels)[:, 0]
# del train_data
# labels = sentences_and_lables[:, 1]

# sentences = sentences_and_labels[:, 0]

label = preprocessing.LabelEncoder()
y = label.fit_transform(train_data['relation'])
label_mappings = integer_mapping = {i: l for i, l in enumerate(label.classes_)}
# y = to_categorical(y) # doing this later.

# label_mappings

print(y[:5])

print(sentences[:5])

X_train, X_val, y_train, y_val = train_test_split(sentences, y, stratify=y, random_state=0, test_size=0.4)   # Global training and test sets.

print(len(X_train))

from sklearn.utils.class_weight import compute_class_weight

# calculate class weight
# calculate class weight
weighting = compute_class_weight(class_weight = "balanced",
                                        classes = np.unique(y_train),
                                        y = y_train)
weights = {}

for index in range(len(weighting)):
    weights[index] = weighting[index]

print(weights)

# m_url = 'https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/2'
# NOTE: Changed this.
def bert_encode(texts, tokenizer, max_len=512):
    all_tokens = []
    all_masks = []
    all_segments = []
    
    for text in texts:
        text = tokenizer.tokenize(text)
        
        text = text[:max_len-2]
        input_sequence = ["[CLS]"] + text + ["[SEP]"]
        pad_len = max_len-len(input_sequence)
        
        tokens = tokenizer.convert_tokens_to_ids(input_sequence) + [0] * pad_len
        pad_masks = [1] * len(input_sequence) + [0] * pad_len
        segment_ids = [0] * max_len
        
        all_tokens.append(tokens)
        all_masks.append(pad_masks)
        all_segments.append(segment_ids)
        
    return np.array(all_tokens), np.array(all_masks), np.array(all_segments)

def build_model(bert_layer, max_len=512, seed=0):
    
    input_word_ids = tf.keras.Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
    input_mask = tf.keras.Input(shape=(max_len,), dtype=tf.int32, name="input_mask")
    input_type_ids = tf.keras.Input(shape=(max_len,), dtype=tf.int32, name="segment_ids")

    mlm_inputs = dict(
    input_word_ids=input_word_ids,
    input_mask=input_mask,
    input_type_ids=input_type_ids,
)

    
    # pooled_output, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])
    bert_layer_outputs = bert_layer(mlm_inputs)
    # print("Bert_layer outputs: ", bert_layer_outputs)
    pooled_output = bert_layer_outputs['pooled_output']
    sequence_output = bert_layer_outputs['sequence_output']
    print("Sequence output: ", sequence_output)
    #np.savez_compressed(f"bert_sequence_op_seed_{seed}.npz", sequence_output)
    
    print(tf.shape(sequence_output))
    clf_output = sequence_output[:, :, :]
    print(tf.shape(clf_output))
    input_shape = tf.shape(clf_output)
    lay = tf.keras.layers.Conv1D(filters=8, kernel_size=5, strides=1, padding="same", activation="relu", input_shape=input_shape[1:])(clf_output)
    lay = tf.keras.layers.MaxPooling1D(2, 2)(lay)
    #lay = tf.keras.layers.LSTM(2, return_sequences=True, dropout=0.2)(lay)
    lay = tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(8, return_sequences=True, dropout=0.2))(lay)
    lay = tf.keras.layers.Flatten()(lay)
    out = tf.keras.layers.Dense(29, activation='softmax')(lay)
    
    model = tf.keras.models.Model(inputs=[input_word_ids, input_mask, input_type_ids], outputs=out)
    #model.compile(tf.keras.optimizers.Adam(lr=2e-5), loss='categorical_crossentropy', metrics=['accuracy'])
    model.compile(tf.keras.optimizers.Adam(lr=2e-5), loss='sparse_categorical_crossentropy', metrics=['accuracy'])
    #model.compile(tf.keras.optimizers.RMSprop(lr=0.001), loss='sparse_categorical_crossentropy', metrics=['accuracy'])
    
    return model

"""### Obtaining Train, test splits.
###### In the train splits, we will have a separate validation split.
"""

checkpoint_path = "training_relations/cnn_bilstm/cp.ckpt"
checkpoint_dir = os.path.dirname(checkpoint_path)

def get_labels(y_pred):
    y_pred_label = np.zeros((len(y_pred),1))
    print(y_pred_label.shape)
    for index in range(len(y_pred)):
        y_pred_label[index] = np.argmax(y_pred[index])
    return y_pred_label

"""# Do not run the following cell if using checkpointed files."""

with tf.device('/device:GPU:0'):
    splits = 5 # For five fold cross-validation.
    #seeds = [i for i in range(splits)]  # Fix the seed value for reproducibility.
    seeds = [4] 
    
    val_dict = {}
    test_dict = {}

    # First get random train-test splits. Doesn't include validation, which will be obtained from the train set.
    for seed in seeds:
        x_t, x_test, y_t, y_test = train_test_split(X_train, y_train, random_state=seed, test_size=0.33)   # Global training and test sets.

        # Now get validation sets from each training set.
        # kf = KFold(n_splits=5, shuffle=False) # Setting shuffle=False because shuffled dataset already before.
        # fold_count = 0

        # for train_index, val_index in kf.split(x_t):
        #     #print(x_t.shape)
        #     #print(y_t.shape)
        #     x_train, x_val = x_t[train_index], x_t[val_index]   # Training and validation features.
        #     y_train, y_val = y_t[train_index], y_t[val_index]   # Training and validation labels.

        #     #encode train data
        #     max_len = 80
        #     train_input = bert_encode(x_train, tokenizer, max_len=max_len)
        #     train_labels = y_train
        #     x_val = bert_encode(x_val, tokenizer, max_len=max_len)


        #     model = build_model(bert_layer, max_len=max_len)
        #     model.summary()
        #     #checkpoint = tf.keras.callbacks.ModelCheckpoint('model.h5', monitor='val_accuracy', save_best_only=True, verbose=1)
        #     checkpoint = tf.keras.callbacks.ModelCheckpoint(filepath=checkpoint_path, save_weights_only=True, verbose=1)
        #     earlystopping = tf.keras.callbacks.EarlyStopping(monitor='val_accuracy', patience=5, verbose=1)
        #     train_sh = model.fit(
        #     train_input, train_labels,
        #     #validation_split=0.2,
        #     validation_data=(x_val, y_val),
        #     epochs=2,
        #     callbacks=[checkpoint, earlystopping],
        #     batch_size=16,
        #     verbose=1)

        #     # Validation sets can be used for hyperparamter tuning.  
        #     val_dict[str(seed) + str(fold_count)] = train_sh.history
        #     fold_count += 1

        #encode whole train data
        max_len = 80
        
        
        #train model on whole train data
        model = build_model(bert_layer, max_len=max_len, seed=seed)
        model.load_weights(checkpoint_path)
        model.summary()
        # train_input = bert_encode(x_t, tokenizer, max_len=max_len)
        train_labels = y_t
        #checkpoint = tf.keras.callbacks.ModelCheckpoint('model.h5', monitor='val_accuracy', save_best_only=True, verbose=1)
        checkpoint = tf.keras.callbacks.ModelCheckpoint(filepath=checkpoint_path, save_weights_only=True, verbose=1)
        earlystopping = tf.keras.callbacks.EarlyStopping(monitor='val_accuracy', patience=5, verbose=1)
        # train_sh = model.fit(
        # train_input, train_labels,
        # validation_split=0,
        # epochs=5,
        # callbacks=[checkpoint, earlystopping],
        # batch_size=16,
        # verbose=1)

        #encode test data 
        test_input = bert_encode(x_test, tokenizer, max_len=max_len)

        # Evaluate the model on the test data using `evaluate`
        print("Evaluate on test data for ", seed)
        results = model.evaluate(test_input, y_test, batch_size=128)

        #calculate F1-score
        y_pred = model.predict(test_input, verbose=1)
        y_pred_label = get_labels(y_pred)
        print("y_pred_label: ", y_pred_label)
        prediction_save_path = f"/home/rsaha/projects/def-afyshe-ab/rsaha/projects/RE_656/Seed Results/prediction_labels/seed_{seed}/"
        np.savez_compressed(prediction_save_path + "prediction_labels_last_epoch.npz", y_pred)
        
        f1_value = f1_score(y_test, y_pred_label, average='macro')
        results.append(f1_value)
        print("test loss, test acc, F1-score:", results)

        print("recall score", classification_report(y_test, y_pred_label))

        test_dict[seed] = results

with open('results full data seed=4 batch_size=16.txt','w') as data: 
    data.write(str(val_dict))
    data.write('\n')
    data.write(str(test_dict))

"""## The following section is for loading the checkpoint weights and then obtaining the confusion matrix."""

# cms = []

# #code for loading weights from checkpoint
# with tf.device('/device:GPU:0'):
#     results = []
#     splits = 5 # For five fold cross-validation.
#     #seeds = [i for i in range(splits)]  # Fix the seed value for reproducibility.
#     seeds = [4]#0, 1, 2, 3, 4]
    
#     val_dict = {}
#     test_dict = {}
    
#     # First get random train-test splits. Doesn't include validation, which will be obtained from the train set.
#     for seed in seeds:
#         x_t, x_test, y_t, y_test = train_test_split(sentences, y, random_state=seed, test_size=0.2)   # Global training and test sets.
#         print(x_test[:5])

#         # Evaluate the new model
#         max_len = 80
#         new_model = build_model(bert_layer, max_len=max_len)
#         new_model.summary()
#         # Loads the weights for new model
#         checkpoint_path = f"/content/drive/Shareddrives/CMPUT 656 Data and Results/Result 29 Relations/Seed {seed}/training_relations/cp.ckpt"
#         training_relations_path = f"/content/drive/Shareddrives/CMPUT 656 Data and Results/Result 29 Relations/Seed {seed}/training_relations/"
#         # /content/drive/Shareddrives/CMPUT 656 Data and Results/Result 29 Relations/Seed 0/training_relations/cp.ckpt.data-00000-of-00001
        

#         new_model.load_weights(checkpoint_path)

#         #encode test data
#         test_input = bert_encode(x_test, tokenizer, max_len=max_len)

#         # Evaluate the model on the test data using `evaluate`
#         print("Evaluate on test data for ", seed)
#         # results = new_model.evaluate(test_input, y_test, batch_size=16)

#         #calculate F1-score
#         y_pred = new_model.predict(test_input, batch_size=16, verbose=1)
#         y_pred_label = get_labels(y_pred)
#         f1_value = f1_score(y_test, y_pred_label, average='macro')
#         np.savez_compressed(training_relations_path +  "predictions.npz", y_pred_label)
#         cm = confusion_matrix(y_pred_label, y_test, labels=[i for i in label_mappings.keys()])
#         cms.append(cm)

#         results.append(f1_value)
#         print("Test loss, Test acc, F1-score:", results)

# x_test[:20]

# print(reduced_label_mappings[:20])

# for k, v in label_mappings.items():
#     print(k, v)

# reduced_label_mappings = {
#     0: 'None',
#     1: 'award=nominee',
#     2: 'author-works_written',
#     3: 'book-genre',
#     4: 'company-industry',
#     5: 'person-graduate',
#     6: 'actor-character',
#     7: 'director-film',
#     8: 'film-country',
#     9: 'film-genre',
#     10: 'film-language',
#     11: 'film-music',
#     12: 'film-production_company',
#     13: 'actor-film',
#     14: 'producer-film',
#     15: 'writer-film',
#     16: 'political_party-politician',
#     17: 'location-contains',
#     18: 'musician-album',
#     19: 'musician-origin',
#     20: 'person-place_of_death',
#     21: 'person-nationality',
#     22: 'person-parents',
#     23: 'person-place_of_birth',
#     24: 'person-profession',
#     25: 'person-religion',
#     26: 'person-spouse',
#     27: 'football_position-player',
#     28: 'sports_team-player'
# }

"""# Read in all the predictions for different seeds and make a confusion matrix by averaging them together. 

NOTE: The confusion matrix is normalized.
"""

# cms = []
# seeds = [0, 1, 2, 3, 4]
# for seed in seeds:
#     training_relations_path = f"/content/drive/Shareddrives/CMPUT 656 Data and Results/Result 29 Relations/Seed {seed}/training_relations/"
#     x_t, x_test, y_t, y_test = train_test_split(sentences, y, random_state=seed, test_size=0.2)
#     del x_t
#     del x_test
#     del y_t
#     predictions = np.load(training_relations_path + "predictions.npz", allow_pickle=True)['arr_0'].tolist()
#     cms.append(confusion_matrix(predictions, y_test, labels=[i for i in label_mappings.keys()]))

# plt.clf()
# averaged_cms = np.mean(cms, axis=0)
# averaged_cms = averaged_cms.astype('float') / averaged_cms.sum(axis=1)[:, np.newaxis]
# cms_df = pd.DataFrame(averaged_cms, index = [value for value in reduced_label_mappings.values()],
#                       columns=[value for value in reduced_label_mappings.values()])
# fig, ax = plt.subplots(figsize=(15, 10))
# heat = sns.heatmap(cms_df, vmin=0.0, vmax=1.0, cbar_kws={'label': 'Normalized value'})
# heat.figure.axes[-1].yaxis.label.set_size(20)
# yticks = [i.upper() for i in cms_df.index]
# xticks = [i.upper() for i in cms_df.columns]
# plt.yticks(plt.yticks()[0], labels=yticks, rotation=0)
# plt.xticks(plt.xticks()[0], labels=xticks, rotation=270)
# plt.title("Confusion matrix for all 29 relations - CoMemNet-BiLSTM", fontsize=20)

# plt.show()

# cm = confusion_matrix(y_pred_label, y_test, labels=[i for i in label_mappings.keys()])
# cms.append(cm)

# import matplotlib.pyplot as plt
# import seaborn as sns
# averaged_cms = np.mean(cms, axis=0)
# averaged_cms = averaged_cms.astype('float') / averaged_cms.sum(axis=1)[:, np.newaxis]
# cms_df = pd.DataFrame(averaged_cms, index = [value for value in reduced_label_mappings.values()],
#                       columns=[value for value in reduced_label_mappings.values()])
# fig, ax = plt.subplots(figsize=(15, 10))
# heat = sns.heatmap(cms_df, vmin=0.0, vmax=1.0, cbar_kws={'label': 'Normalized value'})
# heat.figure.axes[-1].yaxis.label.set_size(20)
# yticks = [i.upper() for i in cms_df.index]
# xticks = [i.upper() for i in cms_df.columns]
# plt.yticks(plt.yticks()[0], labels=yticks, rotation=0)
# plt.xticks(plt.xticks()[0], labels=xticks, rotation=270)
# plt.title("Confusion matrix for all 29 relations", fontsize=20)

# plt.show()

# averaged_cms.shape

"""# Gradio app to demo the model."""

# !pip install -q gradio
# !pip install sentencepiece

# import tokenization
# import pandas as pd
# import os
# import numpy as np
# import matplotlib.pyplot as plt
# import seaborn as sns
# # from keras.callbacks import Callback
# from sklearn.metrics import confusion_matrix, f1_score, precision_score, recall_score

# import tensorflow as tf
# import tensorflow_hub as hub
# from tensorflow.keras.utils import to_categorical
# from sklearn import preprocessing
# from sklearn.model_selection import train_test_split
# from sklearn.model_selection import KFold
# from sklearn.utils import shuffle
# from sklearn.metrics import f1_score
# import gradio as gr

# m_url = 'https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/2'
# bert_layer = hub.KerasLayer(m_url, trainable=False)

# from google.colab import drive
# drive.mount('/content/drive')

# def get_labels(y_pred):
#     y_pred_label = np.zeros((len(y_pred),1))
#     print(y_pred_label.shape)
#     for index in range(len(y_pred)):
#         y_pred_label[index] = np.argmax(y_pred[index])
#     return y_pred_label

# vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
# do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
# tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case)

# def bert_encode(texts, tokenizer, max_len=512):
#     all_tokens = []
#     all_masks = []
#     all_segments = []
    
#     for text in texts:
#         text = tokenizer.tokenize(text)
        
#         text = text[:max_len-2]
#         input_sequence = ["[CLS]"] + text + ["[SEP]"]
#         pad_len = max_len-len(input_sequence)
        
#         tokens = tokenizer.convert_tokens_to_ids(input_sequence) + [0] * pad_len
#         pad_masks = [1] * len(input_sequence) + [0] * pad_len
#         segment_ids = [0] * max_len
        
#         all_tokens.append(tokens)
#         all_masks.append(pad_masks)
#         all_segments.append(segment_ids)
        
#     return np.array(all_tokens), np.array(all_masks), np.array(all_segments)

# def build_model(bert_layer, max_len=512):
#     input_word_ids = tf.keras.Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
#     input_mask = tf.keras.Input(shape=(max_len,), dtype=tf.int32, name="input_mask")
#     segment_ids = tf.keras.Input(shape=(max_len,), dtype=tf.int32, name="segment_ids")
    
#     pooled_output, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])
    
#     print(tf.shape(sequence_output))
#     clf_output = sequence_output[:, :, :]
#     print(tf.shape(clf_output))
    
#     lay = tf.keras.layers.Conv1D(filters=8, kernel_size=5, strides=1, padding="same", activation="relu")(clf_output)
#     lay = tf.keras.layers.MaxPooling1D(2, 2)(lay)
#     #lay = tf.keras.layers.LSTM(2, return_sequences=True, dropout=0.2)(lay)
#     lay = tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(8, return_sequences=True, dropout=0.2))(lay)
#     lay = tf.keras.layers.Flatten()(lay)
#     out = tf.keras.layers.Dense(29, activation='softmax')(lay)
    
#     model = tf.keras.models.Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)
#     #model.compile(tf.keras.optimizers.Adam(lr=2e-5), loss='categorical_crossentropy', metrics=['accuracy'])
#     model.compile(tf.keras.optimizers.Adam(lr=2e-5), loss='sparse_categorical_crossentropy', metrics=['accuracy'])
#     #model.compile(tf.keras.optimizers.RMSprop(lr=0.001), loss='sparse_categorical_crossentropy', metrics=['accuracy'])
    
#     return model

# reduced_label_mappings = {
#     0: 'None',
#     1: 'award-nominee',
#     2: 'author-works_written',
#     3: 'book-genre',
#     4: 'company-industry',
#     5: 'person-graduate',
#     6: 'actor-character',
#     7: 'director-film',
#     8: 'film-country',
#     9: 'film-genre',
#     10: 'film-language',
#     11: 'film-music',
#     12: 'film-production_company',
#     13: 'actor-film',
#     14: 'producer-film',
#     15: 'writer-film',
#     16: 'political_party-politician',
#     17: 'location-contains',
#     18: 'musician-album',
#     19: 'musician-origin',
#     20: 'person-place_of_death',
#     21: 'person-nationality',
#     22: 'person-parents',
#     23: 'person-place_of_birth',
#     24: 'person-profession',
#     25: 'person-religion',
#     26: 'person-spouse',
#     27: 'football_position-player',
#     28: 'sports_team-player'
# }

# with tf.device('/device:GPU:0'):
#     results = []
#     splits = 5 # For five fold cross-validation.
#     #seeds = [i for i in range(splits)]  # Fix the seed value for reproducibility.
#     seeds = [4]#0, 1, 2, 3, 4]
    
#     val_dict = {}
#     test_dict = {}
    
#     # First get random train-test splits. Doesn't include validation, which will be obtained from the train set.
#     for seed in seeds:
#         x_t, x_test, y_t, y_test = train_test_split(sentences, y, random_state=seed, test_size=0.2)   # Global training and test sets.

#         # Evaluate the new model
#         max_len = 80
#         new_model = build_model(bert_layer, max_len=max_len)
#         new_model.summary()
#         # Loads the weights for new model
#         checkpoint_path = f"/content/drive/Shareddrives/CMPUT 656 Data and Results/Result 29 Relations/Seed {seed}/training_relations/cp.ckpt"
#         training_relations_path = f"/content/drive/Shareddrives/CMPUT 656 Data and Results/Result 29 Relations/Seed {seed}/training_relations/"
#         # /content/drive/Shareddrives/CMPUT 656 Data and Results/Result 29 Relations/Seed 0/training_relations/cp.ckpt.data-00000-of-00001
        

#         new_model.load_weights(checkpoint_path)

#         #encode test data
#         test_input = bert_encode(x_test, tokenizer, max_len=max_len)

#         # Evaluate the model on the test data using `evaluate`
#         print("Evaluate on test data for ", seed)
#         # results = new_model.evaluate(test_input, y_test, batch_size=16)

#         #calculate F1-score
#         y_pred = new_model.predict(test_input, batch_size=16, verbose=1)
#         y_pred_label = get_labels(y_pred)
#         f1_value = f1_score(y_test, y_pred_label, average='macro')
#         np.savez_compressed(training_relations_path +  "predictions.npz", y_pred_label)
#         cm = confusion_matrix(y_pred_label, y_test, labels=[i for i in label_mappings.keys()])
#         cms.append(cm)

#         results.append(f1_value)
#         print("Test loss, Test acc, F1-score:", results)

# seed = 0
# checkpoint_path = f"/content/drive/Shareddrives/CMPUT 656 Data and Results/Result 29 Relations/Seed {seed}/training_relations/cp.ckpt"
# max_len = 80

# new_model = build_model(bert_layer, max_len=max_len)
# new_model.load_weights(checkpoint_path)

# def predict_relation(sentence):
#     print(sentence)
#     # df = pd.DataFrame.from_dict(
#     #     {
#     #         "Pclass": [passenger_class + 1],
#     #         "Sex": [0 if is_male else 1],
#     #         "Age": [age],
#     #         "Company": [
#     #             (1 if "Sibling" in company else 0) + (2 if "Child" in company else 0)
#     #         ],
#     #         "Fare": [fare],
#     #         "Embarked": [embark_point + 1],
#     #     }
#     # )
    
#     test_input = bert_encode(np.array([sentence]), tokenizer, max_len=max_len)
#     y_pred = new_model.predict(test_input, batch_size=16, verbose=1)
#     y_pred_label = get_labels(y_pred)
#     relations = [rel for rel in reduced_label_mappings.values()]
#     probabilities = y_pred.tolist()[0]
#     result_dict = {}
#     for k, v in zip(relations, probabilities):
#         result_dict[k] = v
#     return result_dict
#     # return y_pred_label, y_pred

# [rel for rel in reduced_label_mappings.values()]

# predict_relation("TV works Role(s) Yugo Kanno Composer")

# iface = gr.Interface(
#     predict_relation,
#     inputs="text",
#     outputs="label",
#     interpretation="default",
#     title="CoMemNet - Relation Extractor", description="NOTE: Model is trained on Wikipedia table data and not continuous text."
# )

# iface.launch(debug=False)

# !git clone https://huggingface.co/spaces/simpleParadox/RE_656

"""# Validation Test"""

# data = pd.read_csv(relations_path, encoding='utf-8', sep = '\t')

# drop_indices

# val_data = data.iloc[drop_indices]
# val_data.shape

# val_data.fillna("", inplace = True)

# val_data.head()

# val_labels = val_data.iloc[:,-1].values
# val_sentences = val_data.iloc[:,:-1].values.tolist()

# val_sentences = [' '.join(sent).strip() for sent in val_sentences]

# y_val = label.transform(val_data['relation'])
# val_label_mappings = integer_mapping = {i: l for i, l in enumerate(label.classes_)}

# val_label_mappings

# #code for loading weights from checkpoint
# with tf.device('/device:GPU:0'):
#     results = []
#     val_dict = {}
#     test_dict = {}
    

#     # Evaluate the new model
#     max_len = 80
#     x_val = val_sentences
    
#     new_model = build_model(bert_layer, max_len=max_len)
#     new_model.summary()
#     # Loads the weights for new model
#     checkpoint_path = "training_relations/cp.ckpt"
#     training_relations_path = "training_relations/"
#     # /content/drive/Shareddrives/CMPUT 656 Data and Results/Result 29 Relations/Seed 0/training_relations/cp.ckpt.data-00000-of-00001
        

#     new_model.load_weights(checkpoint_path)

#     #encode test data
#     print('Encoding data...')
#     val_input = bert_encode(x_val, tokenizer, max_len=max_len)
#     print('Encoding Done...')

#     # Evaluate the model on the val data using `evaluate`
#     results = new_model.evaluate(val_input, y_val, batch_size=16)

#     #calculate F1-score
#     y_pred = new_model.predict(val_input, batch_size=16, verbose=1)
#     y_pred_label = get_labels(y_pred)
#     f1_value = f1_score(y_val, y_pred_label, average='macro')
        
#     results.append(f1_value)
#     print("Test loss, Test acc, F1-score:", results)