implemented plotting functionalities for discrepancy

linfa/discrepancy.py

@@ -22,35 +22,55 @@ def __init__(self, model_name,
  model_folder='./', 
  surrogate=None, 
  device='cpu'):
-
- self.device = device
- self.input_size = input_size
- self.output_size = output_size
+
  self.model_name = model_name
  self.model_folder = model_folder
- self.is_trained = False
 
- # Assign LF model
- self.lf_model = lf_model
+ if(var_grid_in is None):
+
+ self.device = None
+ self.input_size = None
+ self.output_size = None
+ self.dnn_arch = None
+ self.is_trained = None
+ self.lf_model = None
+ self.var_grid_in = None
+ self.var_grid_out = None
+ self.var_in_avg = None
+ self.var_in_std = None
+ self.var_out_avg = None
+ self.var_out_std = None
+ self.surrogate = None
 
- # Store variable grid locations
- self.var_grid_in=var_grid_in
- # Output variables - multiple noisy observations 
- # are available for each combination of variables
- self.var_grid_out=var_grid_out
-
- # Input/output statistics
- self.var_in_avg = torch.mean(var_grid_in,dim=0)
- if(len(self.var_grid_in) == 1):
- self.var_in_std = torch.zeros_like(self.var_in_avg)
  else:
- self.var_in_std = torch.std(var_grid_in,dim=0)
- # If there are multiple outputs, we will define one file for each output
- self.var_out_avg = torch.mean(var_grid_out)
- self.var_out_std = torch.std(var_grid_out)
 
- # Create surrogate
- self.surrogate = FNN(input_size, output_size, arch=dnn_arch, device=self.device, init_zero=True) if surrogate is None else surrogate
+ self.device = device
+ self.input_size = input_size
+ self.output_size = output_size
+ self.dnn_arch = dnn_arch
+ self.is_trained = False
+
+ # Assign LF model
+ self.lf_model = lf_model
+
+ # Store variable grid locations
+ self.var_grid_in=var_grid_in
+ # Output variables - multiple noisy observations 
+ # are available for each combination of variables
+ self.var_grid_out=var_grid_out
+
+ # Input/output statistics
+ self.var_in_avg = torch.mean(var_grid_in,dim=0)
+ if(len(self.var_grid_in) == 1):
+ self.var_in_std = torch.zeros_like(self.var_in_avg)
+ else:
+ self.var_in_std = torch.std(var_grid_in,dim=0)
+ # If there are multiple outputs, we will define one file for each output
+ self.var_out_avg = torch.mean(var_grid_out)
+ self.var_out_std = torch.std(var_grid_out)
+
+ # Create surrogate
+ self.surrogate = FNN(input_size, output_size, arch=self.dnn_arch, device=self.device, init_zero=True) if surrogate is None else surrogate
 
  def surrogate_save(self):
  """Save surrogate model to [self.name].sur and [self.name].npz
@@ -60,7 +80,19 @@ def surrogate_save(self):
 
  """
  # Save model state dictionary
- torch.save(self.surrogate.state_dict(), self.model_folder +'/'+ self.model_name + '.sur')
+ dict_to_save = {}
+ dict_to_save['weights'] = self.surrogate.state_dict()
+ dict_to_save['grid_in'] = self.var_grid_in
+ dict_to_save['grid_stats_in'] = [self.var_in_avg,self.var_in_std]
+ dict_to_save['grid_out'] = self.var_grid_out
+ dict_to_save['grid_stats_out'] = [self.var_out_avg,self.var_out_std]
+ dict_to_save['trained'] = self.is_trained
+ dict_to_save['input_size'] = self.input_size
+ dict_to_save['output_size'] = self.output_size
+ dict_to_save['dnn_arch'] = self.dnn_arch
+ dict_to_save['device'] = self.device
+ # Save entire dictionary
+ torch.save(dict_to_save, self.model_folder +'/'+ self.model_name + '.sur')
 
  def surrogate_load(self):
  """Load surrogate model from [self.name].sur and [self.name].npz
@@ -69,7 +101,18 @@ def surrogate_load(self):
  None
  """
  # Read back the state dictionary from file
- self.surrogate.load_state_dict(torch.load(self.model_folder +'/'+ self.model_name + '.sur'))
+ load_dict = torch.load(self.model_folder +'/'+ self.model_name + '.sur')
+ self.var_grid_in = load_dict['grid_in']
+ self.var_in_avg,self.var_in_std = load_dict['grid_stats_in']
+ self.var_grid_out = load_dict['grid_out']
+ self.var_out_avg,self.var_out_std = load_dict['grid_stats_out']
+ self.is_trained = load_dict['trained']
+ self.input_size = load_dict['input_size']
+ self.output_size = load_dict['output_size']
+ self.dnn_arch = load_dict['dnn_arch']
+ self.device = load_dict['device']
+ self.surrogate = FNN(self.input_size, self.output_size, arch=self.dnn_arch, device=self.device, init_zero=True)
+ self.surrogate.load_state_dict(load_dict['weights'])
 
  def update(self, batch_x, max_iters=10000, lr=0.01, lr_exp=0.999, record_interval=50, store=True, reg=False, reg_penalty=0.0001):
  """Train surrogate model with pre-grid.

linfa/plot_disc.py

@@ -3,35 +3,83 @@
 from linfa.discrepancy import Discrepancy
 import numpy as np
 import os
+import argparse
+
+def scale_limits(min,max,factor):
+ if(min>max):
+ temp = max
+ max = min
+ min = temp
+ center = 0.5*(min+max)
+ range = max - min
+ return center - factor*0.5*range, center + factor*0.5*range
+
+def plot_disr_histograms(lf_file,lf_dicr_file,lf_discr_noise_file):
+
+ # Read result files
+ lf_model = np.loadtxt(lf_file)
+ lf_model_plus_disc = np.loadtxt(lf_dicr_file)
+ lf_model_plus_disc_plus_noise = np.loadtxt(lf_discr_noise_file)
+
+ # Plot histograms
+ plt.hist(lf_model, label = 'LF', alpha =0.5)
+ plt.hist(lf_model_plus_disc, label = 'LF + disc', alpha =0.5)
+ plt.hist(lf_model_plus_disc_plus_noise, label = 'LF + disc + noise', alpha = 0.5)
+ plt.xlabel('Coverage')
+ plt.ylabel('Frequency')
+ plt.legend()
+ plt.show()
+
+def plot_discr_surface_2d(file_path,data_file,num_1d_grid_points,data_limit_factor):
 
-# directory = 'results'
-# iteration = '25000'
-# experiment = 'test_lf_with_disc_hf_data_prior_TP1'
-# filename1 = 'outputs_lf'
-# filename2 = 'outputs_lf+discr'pi
-# filename3 = 'outputs_lf+discr+noise'
+ exp_name = os.path.basename(file_path)
+ dir_name = os.path.dirname(file_path) 
+
+ print(exp_name)
+ print(dir_name)
+
+ # Create new discrepancy
+ dicr = Discrepancy(model_name=exp_name, 
+ model_folder=dir_name,
+ lf_model=None,
+ input_size=None,
+ output_size=None,
+ var_grid_in=None,
+ var_grid_out=None)
+ dicr.surrogate_load()
+
+ # Get the number of dimensions for the aux variable
+ num_dim = dicr.var_grid_in.size(1)
+ if(num_dim == 2):
+ min_dim_1 = torch.min(dicr.var_grid_in[:,0])
+ max_dim_1 = torch.max(dicr.var_grid_in[:,0])
+ min_dim_2 = torch.min(dicr.var_grid_in[:,1])
+ max_dim_2 = torch.max(dicr.var_grid_in[:,1])
+ min_dim_1,max_dim_1 = scale_limits(min_dim_1,max_dim_1,data_limit_factor)
+ min_dim_2,max_dim_2 = scale_limits(min_dim_2,max_dim_2,data_limit_factor)
+
+ test_grid_1 = torch.linspace(min_dim_1,max_dim_1,num_1d_grid_points)
+ test_grid_2 = torch.linspace(min_dim_2,max_dim_2,num_1d_grid_points)
+ grid_t,grid_p = torch.meshgrid(test_grid_1, test_grid_2, indexing='ij')
+ test_grid = torch.cat((grid_t.reshape(-1,1),grid_p.reshape(-1,1)),1)
+
+ res = dicr.forward(test_grid)
 
-# path1 = os.path.join(directory, experiment, experiment + '_' + filename1 + '_' + iteration)
-# path2 = os.path.join(directory, experiment, experiment + '_' + filename2 + '_' + iteration)
-# path3 = os.path.join(directory, experiment, experiment + '_' + filename3 + '_' + iteration)
+ x = test_grid[:,0].cpu().detach().numpy()
+ y = test_grid[:,1].cpu().detach().numpy()
+ z = res.cpu().detach().numpy().flatten()
 
-# lf_model = np.loadtxt(path1)
-# lf_model_plus_disc = np.loadtxt(path2)
-# lf_model_plus_disc_plus_noise = np.loadtxt(path3)
+ print(x.shape,y.shape,z.shape)
 
-<<<<<<< Updated upstream
-# plt.hist(lf_model[:,0], label = 'LF')
-# plt.hist(lf_model_plus_disc[:,0], label = 'LF + disc')
-# plt.hist(lf_model_plus_disc_plus_noise[:,0], label = 'LF + disc + noise')
-# plt.xlabel('Coverage')
-# plt.ylabel('Frequency')
-# plt.legend()
-# plt.show()
+ ax = plt.figure().add_subplot(projection='3d')
+ ax.plot_trisurf(x,y,z,linewidth=0.2, antialiased=True)
+ plt.show()
 
-# shape: number of ___?___ (15) x Batch size (5000)
-# print(np.shape(lf_model))
+ else:
+ print('ERROR. Invalid number of dimensions. Should be 2. Instead is ',num_dim)
+ exit(-1)
 
-def plot_discrepancy(file_path,train_grid_in,train_grid_out,test_grid):
+def eval_discrepancy_custom_grid(file_path,train_grid_in,train_grid_out,test_grid):
 
  exp_name = os.path.basename(file_path)
  dir_name = os.path.dirname(file_path) 
@@ -58,34 +106,122 @@ def plot_discrepancy(file_path,train_grid_in,train_grid_out,test_grid):
 # =========
 if __name__ == '__main__':
 
- file_path = './tests/results/test_lf_with_disc_hf_data_TP1/test_lf_with_disc_hf_data_TP1'
- obs_file = './tests/results/test_lf_with_disc_hf_data_TP1/test_lf_with_disc_hf_data_TP1_data'
- train_grid_in = torch.from_numpy(np.loadtxt(obs_file).reshape(1,-1)[:,:2])
- train_grid_out = torch.from_numpy(np.loadtxt(obs_file).reshape(1,-1)[:,2:])
-
- # Create a testing grid 
- t_test = torch.linspace(350.0,450.0,20)
- p_test = torch.linspace(1.0,5.0,20)
- grid_t,grid_p = torch.meshgrid(t_test, p_test, indexing='ij')
- test_grid = torch.cat((grid_t.reshape(-1,1),grid_p.reshape(-1,1)),1)
+ # Init parser
+ parser = argparse.ArgumentParser(description='.')
+
+ # folder name
+ parser.add_argument('-f', '--folder',
+ action=None,
+ # nargs='+',
+ const=None,
+ default='./',
+ type=str,
+ required=False,
+ help='Folder with experiment results',
+ metavar='',
+ dest='folder_name')
+
+
+ # folder name
+ parser.add_argument('-n', '--name',
+ action=None,
+ # nargs='+',
+ const=None,
+ default='./',
+ type=str,
+ required=True,
+ help='Name of numerical experiment',
+ metavar='',
+ dest='exp_name')
+
+ # iteration number = 1
+ parser.add_argument('-i', '--iter',
+ action=None,
+ # nargs='+',
+ const=None,
+ default=1,
+ type=int,
+ choices=None,
+ required=True,
+ help='Iteration number',
+ metavar='',
+ dest='step_num')
+
+ # plot format
+ parser.add_argument('-p', '--picformat',
+ action=None,
+ const=None,
+ default='png',
+ type=str,
+ choices=['png','pdf','jpg'],
+ required=False,
+ help='Output format for picture',
+ metavar='',
+ dest='img_format')
+
+ # Enable dark mode for pictures
+ parser.add_argument('-d', '--dark',
+ action='store_true',
+ default=False,
+ required=False,
+ help='Generate pictures for dark background',
+ dest='use_dark_mode')
+
+ # Enable dark mode for pictures
+ parser.add_argument('-m', '--mode',
+ action=None,
+ const=None,
+ default='histograms',
+ type=str,
+ choices=['histograms','discr_surface'],
+ required=False,
+ help='Type of plot/result to generate',
+ metavar='',
+ dest='result_mode')
 
- res = plot_discrepancy(file_path,train_grid_in,train_grid_out,test_grid)
-
- print(res)
-
-
- # Draw histograms of lf+discr
- # Draw histograms of lf
- # Draw histograms of lf+discr+noise
-=======
-plt.hist(lf_model, label = 'LF', alpha =0.5)
-plt.hist(lf_model_plus_disc, label = 'LF + disc', alpha =0.5)
-plt.hist(lf_model_plus_disc_plus_noise, label = 'LF + disc + noise', alpha = 0.5)
-plt.xlabel('Coverage')
-plt.ylabel('Frequency')
-plt.legend()
-plt.show()
-
-# shape: number of no TP-pairs x Batch size (5000)
-print(np.shape(lf_model))
->>>>>>> Stashed changes
+ # folder name
+ parser.add_argument('-z', '--num_points',
+ action=None,
+ # nargs='+',
+ const=None,
+ default=10,
+ type=int,
+ required=False,
+ help='Number of on-dimensional test grid points (same in every dimension)',
+ metavar='',
+ dest='num_1d_grid_points')
+
+ # folder name
+ parser.add_argument('-y', '--limfactor',
+ action=None,
+ # nargs='+',
+ const=None,
+ default=1.0,
+ type=float,
+ required=False,
+ help='Factor for test grid limits from data file',
+ metavar='',
+ dest='data_limit_factor')
+
+ # Parse Commandline Arguments
+ args = parser.parse_args()
+
+ # Set file name/path for lf and discr results
+ out_dir = args.folder_name + args.exp_name + '/'
+ lf_file = out_dir + args.exp_name + '_outputs_lf_' + str(args.step_num)
+ lf_dicr_file = out_dir + args.exp_name + '_outputs_lf+discr_' + str(args.step_num) 
+ lf_discr_noise_file = out_dir + args.exp_name + '_outputs_lf+discr+noise_' + str(args.step_num)
+ discr_sur_file = out_dir + args.exp_name
+ data_file = out_dir + args.exp_name + '_data'
+
+ # out_info = args.exp_name + '_' + str(args.step_num)
+
+ if(args.result_mode == 'histograms'):
+ plot_disr_histograms(lf_file,lf_dicr_file,lf_discr_noise_file)
+ elif(args.result_mode == 'discr_surface'):
+ plot_discr_surface_2d(discr_sur_file,data_file,args.num_1d_grid_points,args.data_limit_factor)
+ else:
+ print('ERROR. Invalid execution mode')
+ exit(-1)
+
+

linfa/tests/test_discr_09_prior_TP15.py

@@ -84,6 +84,7 @@ def run_test():
 
  # Create new discrepancy
  exp.surrogate = Discrepancy(model_name=exp.name, 
+ model_folder=exp.output_dir, 
  lf_model=exp.model.solve_t,
  input_size=exp.model.var_in.size(1),
  output_size=1,