# -*- coding: utf-8 -*- """ Created on Mon Sep 13 16:45:43 2021 @author: A752018 """ # Same as Mid-term report (LATEX) & Atos Scientific community # ----------------------------------------------------------------------------- # Data # ----------------------------------------------------------------------------- dossier = 'C:\\Users\\A752018\Documents\\Présentations\\2021 09 - Phd day\\data' os.chdir(dossier) os.getcwd() mode_save = True var_clust = 'cluster_unif_scale' list_base = pickle.load(open((dossier + '\\data_PhDay'), 'rb')) # pickle.dump(list_base, open(('data_PhDDay'), 'wb')) # ----------------------------------------------------------------------------- # PROPOSED APPROACH # ----------------------------------------------------------------------------- signal = 0 data = list_base[signal].copy().T # Echelle temporelle tps = max(data['TOA']) - min(data['TOA']) # 198191129.69084877 -> 3,3031854948474790312 mins round(tps / 6.606e+7, 2) data = data[data['TOA'] < 15300000] data['LEVEL_watt'] = data['LEVEL'].apply(f) # Calcul de la dTOA initiale data = data.sort_values(by = ['TOA']) data.index = range(0, len(data)) x = np.array(data['TOA']) data['dtoa_initiale'] = pd.DataFrame(np.diff(x, append = x[-1] )) # -- 2) Clustering print("Start of clustering") df = clustering_hdb(data = data, epsilon_value = 0) data = df.copy() data['ID'] = data.index del df print("End of clustering") # -- 3) Pre-precess OT len(data[var_clust].unique()) data_without_outliers = (data).query(var_clust + ' != -1') pd.DataFrame(data_without_outliers[var_clust].value_counts().sort_index()) x_dist_toa = data_without_outliers['TOA'] nbr_bins = SEUIL_bins(data_without_outliers) bins_toa = np.linspace(x_dist_toa.min(), x_dist_toa.max(), nbr_bins) # --- 4) OT print("Start of OT") Z_bin_ot_max_toa, matrice_idx_ot_max_toa = POT_computation(data = data_without_outliers, bins = bins_toa, var_clust = var_clust, var_dist = 'TOA', distance = np.abs, hist_values = 'Max', reg = 1, reg_m = 1, mode = 'simple', method = 'sinkhorn') print("End of OT") # Evaluation print("Start of EVAL_metric_dendo") # Vérification si on doit réduire la taille des gros clusters pour appliquer les métriques et gagner en temps d'éxécution names = pd.DataFrame(data[var_clust].value_counts().sort_index()) names = names[names[var_clust] > 1500] if(names.empty != True): data_without_outliers_kms = MEF_reduction_data(data = data_without_outliers, var_clust = var_clust, method = 'kmeans++', n_groups = 1500) matrice_metric_max_toa = EVAL_metrics_dendo(data = data_without_outliers_kms, var_clust = var_clust, matrice_idx = matrice_idx_ot_max_toa) else: matrice_metric_max_toa = EVAL_metrics_dendo(data = data_without_outliers, var_clust = var_clust, matrice_idx = matrice_idx_ot_max_toa) k1_ot_max_toa, k_list1_ot_max_toa, k1_dist_ot_max_toa = EVAL_model_decision(Z = Z_bin_ot_max_toa, matrice_metric = matrice_metric_max_toa) print("End of EVAL_metric_dendo") # Regroupments data = MEF_regroupment(data = data, k = k1_ot_max_toa, label_var = 'cluster_final', matrice_idx = matrice_idx_ot_max_toa, var_clust = var_clust, dtoa_clus = 'dTOA_reconstruite_ot_max_toa') data = data.T stat = EVAL_metrics_regroupement(data = data, labels_pred = 'cluster_final') # --- PLOTS # True label PLOT_simple(data.T, var_coloration = 'ID_radar', signal = signal, title = True, var1 = 'FN', var2 = 'DI', var3 = 'LEVEL', var4 = 'FN', var5 = 'DI', label_var1 = 'Frequency (MHz)', label_var2 = 'Pulse width (ns)', label_var3 = 'Level (dBm)', label_var4 = 'Frequency (MHz)', label_var5 = 'Pulse width (ns)', save = mode_save, save_name = 'true_label') # Initiale data PLOT_simple(data.T, var_coloration = None, signal = signal, title = True, var1 = 'FN', var2 = 'DI', var3 = 'LEVEL', var4 = 'TOA', var5 = 'dtoa_initiale', label_var1 = 'Frequency (MHz)', label_var2 = 'Pulse width (ns)', label_var3 = 'Level (dBm)', label_var4 = 'Time of arrival (µs)', label_var5 = 'Difference of Time of Arrival (µs)', save = mode_save, save_name = 'true_label') # Plot clustering PLOT_simple(data_without_outliers.T, var_coloration = var_clust, signal = signal, title = True, var1 = 'FN', var2 = 'DI', var3 = 'LEVEL', var4 = 'FN', var5 = 'DI', label_var1 = 'Frequency (MHz)', label_var2 = 'Pulse width (ns)', label_var3 = 'Level (dBm)', label_var4 = 'Frequency (MHz)', label_var5 = 'Pulse width (ns)', save = mode_save, save_name = 'true_label') # Histograms PLOT_histogram(data = data_without_outliers.T, signal = signal, bins = bins_toa, var_hist = 'TOA', weights = True, var_cluster = var_clust, mode_save = mode_save, save_name = 'histo_toa', nbr_plot = 6) # Dendro PLOT_dendo(data = data_without_outliers, signal = signal, linkage_matrix = Z_bin_ot_max_toa, var_clust = var_clust, value = True, title = True, distance_cut = k1_dist_ot_max_toa, save = mode_save, save_name = 'dendo_ot_max_toa') # Plot final PLOT_simple(data.T, var_coloration = 'cluster_final', signal = signal, title = True, var1 = 'FN', var2 = 'DI', var3 = 'LEVEL', var4 = 'TOA', var5 = 'dTOA_reconstruite_ot_max_toa', label_var1 = 'Frequency (MHz)', label_var2 = 'Pulse width (ns)', label_var3 = 'Level (dBm)', label_var4 = 'Time of arrival (µs)', label_var5 = 'Difference of Time of Arrival (µs)', save = mode_save, save_name = 'true_label') # ----------------------------------------------------------------------------- # APPLICATION CASE # ----------------------------------------------------------------------------- signal = 1 data = list_base[signal].copy().T data = data[['FN', 'TOA', 'LEVEL', 'DI', 'ID_radar']] var_clust = 'cluster_unif' # --- Clustering df = clustering_hdb(data = data, epsilon_value = 0) # data = pd.merge(data, df, left_on = ['FN', 'DI', 'TOA', 'LEVEL'], right_on = ['FN', 'DI', 'TOA', 'LEVEL']) # data.drop_duplicates(inplace = True) # data['ID'] = data.index data = df.copy() del df data['LEVEL_watt'] = data['LEVEL'].apply(f) # -- Supression des outliers data_without_outliers = (data).query(var_clust + ' != -1') # -- Création des histogrammes x_dist_toa = data_without_outliers['TOA'] nbr_bins = SEUIL_bins(data_without_outliers) bins_toa = np.linspace(x_dist_toa.min(), x_dist_toa.max(), nbr_bins) # --- Transport optimal print("Start of OT") Z_bin_ot_max_toa, matrice_idx_ot_max_toa = POT_computation(data = data_without_outliers, bins = bins_toa, var_clust = var_clust, var_dist = 'TOA', distance = np.abs, hist_values = 'Max', reg = 1, reg_m = 1, mode = 'simple', method = 'sinkhorn') print("End of OT") # --- 2.1.3) Evaluation pour couper le dendogramme + réduction taille base de données print("Start of EVAL_metric_dendo") # Vérification si on doit réduire la taille des gros clusters pour appliquer les métriques et gagner en temps d'éxécution names = pd.DataFrame(data_without_outliers[var_clust].value_counts().sort_index()) names = names[names[var_clust] > 1500] if(names.empty != True): data_without_outliers_kms = MEF_reduction_data(data = data_without_outliers, var_clust = var_clust, method = "kmeans++", n_groups = 1500) matrice_metric_max_toa = EVAL_metrics_dendo(data = data_without_outliers_kms, var_clust = var_clust, matrice_idx = matrice_idx_ot_max_toa) else: matrice_metric_max_toa = EVAL_metrics_dendo(data = data_without_outliers, var_clust = var_clust, matrice_idx = matrice_idx_ot_max_toa) k1_ot_max_toa, k_list1_ot_max_toa, k1_dist_ot_max_toa = EVAL_model_decision(Z = Z_bin_ot_max_toa, matrice_metric = matrice_metric_max_toa) print("End of EVAL_metric_dendo") # --- Regroupements data['var_data'] = data[var_clust].copy() data = MEF_regroupment(data = data, k = k1_ot_max_toa, label_var = 'cluster_final', matrice_idx = matrice_idx_ot_max_toa, var_clust = 'var_data', dtoa_clus = 'dTOA_reconstruite_ot_max_toa') data = data.T stat = EVAL_metrics_regroupement(data = data, labels_pred = 'cluster_final') # --- PLOTS # True label PLOT_simple(data.T, var_coloration = 'ID_radar', signal = signal, title = True, var1 = 'FN', var2 = 'DI', var3 = 'LEVEL', var4 = 'FN', var5 = 'DI', label_var1 = 'Frequency (MHz)', label_var2 = 'Pulse width (ns)', label_var3 = 'Level (dBm)', label_var4 = 'Frequency (MHz)', label_var5 = 'Pulse width (ns)', save = mode_save, save_name = 'true_label') # Initiale data PLOT_simple(data.T, var_coloration = None, signal = signal, title = True, var1 = 'FN', var2 = 'DI', var3 = 'LEVEL', var4 = 'TOA', var5 = 'dtoa_initiale', label_var1 = 'Frequency (MHz)', label_var2 = 'Pulse width (ns)', label_var3 = 'Level (dBm)', label_var4 = 'Time of arrival (µs)', label_var5 = 'Difference of Time of Arrival (µs)', save = mode_save, save_name = 'true_label') # Plot clustering PLOT_simple(data_without_outliers.T, var_coloration = var_clust, signal = signal, title = True, var1 = 'FN', var2 = 'DI', var3 = 'LEVEL', var4 = 'FN', var5 = 'DI', label_var1 = 'Frequency (MHz)', label_var2 = 'Pulse width (ns)', label_var3 = 'Level (dBm)', label_var4 = 'Frequency (MHz)', label_var5 = 'Pulse width (ns)', save = mode_save, save_name = 'true_label') PLOT_simple(data.T, var_coloration = var_clust, signal = signal, title = True, var1 = 'FN', var2 = 'DI', var3 = 'LEVEL', var4 = 'FN', var5 = 'DI', label_var1 = 'Frequency (MHz)', label_var2 = 'Pulse width (ns)', label_var3 = 'Level (dBm)', label_var4 = 'Frequency (MHz)', label_var5 = 'Pulse width (ns)', save = mode_save, save_name = 'true_label') # Histograms PLOT_histogram(data = data_without_outliers.T, signal = signal, bins = bins_toa, var_hist = 'TOA', weights = True, var_cluster = var_clust, mode_save = mode_save, save_name = 'histo_toa', nbr_plot = 6) # Dendro PLOT_dendo(data = data_without_outliers, signal = signal, linkage_matrix = Z_bin_ot_max_toa, var_clust = var_clust, value = True, title = True, distance_cut = k1_dist_ot_max_toa, save = mode_save, save_name = 'dendo_ot_max_toa') # Plot final PLOT_simple(data.T, var_coloration = 'cluster_final', signal = signal, title = True, var1 = 'FN', var2 = 'DI', var3 = 'LEVEL', var4 = 'FN', var5 = 'DI', label_var1 = 'Frequency (MHz)', label_var2 = 'Pulse width (ns)', label_var3 = 'Level (dBm)', label_var4 = 'Frequency (MHz)', label_var5 = 'Pulse width (ns)', save = mode_save, save_name = 'true_label')