serveur

.gitattributes

 *.zip filter=lfs diff=lfs merge=lfs -text
 *.bin filter=lfs diff=lfs merge=lfs -text
+*imagenet filter=lfs diff=lfs merge=lfs -text
+*zip filter=lfs diff=lfs merge=lfs -text

backpack.py

@@ -11,6 +11,7 @@ class BackPack(nn.Module):
     def __init__(self, image_size, QF, folder_model, c_coeffs, rho_0, entropy, N, nets, ecdf_list, attack):
         super(BackPack, self).__init__()
         self.net_decompress = IDCT8_Net(image_size, QF, folder_model)
+        self.device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
         self.N = N
         self.entropy = torch.tensor(entropy)
         self.im_size = image_size
@@ -20,7 +21,7 @@ class BackPack(nn.Module):
         self.spatial_cover = self.net_decompress.forward(
             torch.reshape(self.c_coeffs, (1, 1, self.im_size, self.im_size)))/255
         self.nets = nets
-        proba_cover = [torch.nn.Softmax(dim=1)(net.forward(self.spatial_cover.cuda().float()))[0, 1]
+        proba_cover = [torch.nn.Softmax(dim=1)(net.forward(self.spatial_cover.to(self.device).float()))[0, 1]
                        for net in nets]
         self.proba_cover = np.array(
             [ecdf(x.cpu().detach().numpy()) for x, ecdf in zip(proba_cover, ecdf_list)])
@@ -53,7 +54,7 @@ class BackPack(nn.Module):
             self.c_coeffs+b, (self.N, 1, self.im_size, self.im_size))
 
         spatial_image_soft = self.net_decompress.forward(stego_soft)/255
-        logits = [torch.reshape(net.forward(spatial_image_soft.cuda().float()), (1, self.N, 2))
+        logits = [torch.reshape(net.forward(spatial_image_soft.to(self.device).float()), (1, self.N, 2))
                   for net in self.nets]
         logits = torch.cat(logits)
         probas_soft = torch.nn.Softmax(dim=2)(logits)[:, :, -1]
@@ -70,7 +71,7 @@ class BackPack(nn.Module):
             stego_hard = torch.reshape(
                 self.c_coeffs+b_hard, (self.N, 1, self.im_size, self.im_size))
             spatial_image_hard = self.net_decompress.forward(stego_hard)/255
-            logits = [torch.reshape(net.forward(spatial_image_hard.cuda().float()), (1, self.N, 2))
+            logits = [torch.reshape(net.forward(spatial_image_hard.to(self.device).float()), (1, self.N, 2))
                       for net in self.nets]
             logits = torch.cat(logits)
             probas_hard = torch.nn.Softmax(dim=2)(logits)[:, :, -1]

experience_512_75_0_4_SGE_efnetB0s1_xunet_srnet/description.txt

-Jun-10-2021Launch of the protocol, starting from iteration 6 to 7
+Jul-18-2021Launch of the protocol, starting from iteration 1 to 2
 Number of CPUs called : 10
  
 PARAMETERS 
@@ -6,15 +6,15 @@ Image characteristics
 - QF = 75
 - Image size = 512
 - Embedding rate = 0.4 bpnzAC
-- Cover images are taken in folder /gpfswork/rech/srp/commun/JPEG_75_512/c_coeffs/
-- Stego images are taken in folder /gpfswork/rech/srp/commun/JPEG_75_512/J_UNI_0_4_npy/
-- Cost maps are taken in folder /gpfswork/rech/srp/commun/JPEG_75_512/costs/
+- Cover images are taken in folder ./experience_512_75_0_4_SGE_efnetB0s1_xunet_srnet/cover/c_coeffs/
+- Stego images are taken in folder ./experience_512_75_0_4_SGE_efnetB0s1_xunet_srnet/data_adv_0/J_UNI_0_4_npy/
+- Cost maps are taken in folder ./experience_512_75_0_4_SGE_efnetB0s1_xunet_srnet/cover/costs/
  
 Protocol setup 
 - Strategy =minmax 
  
 Model description 
-- The 3 model architectures are efnet,xunet,srnet with the following setup :
+- The 3 model architectures are xunet,srnet,efnet with the following setup :
      - Efficient-net version is b0 pretrained on image-net 
      - First conv stem is with stride = 1 
  
@@ -24,26 +24,26 @@ Training setup
 - Train size = 4000
 - Valid size = 1000
 - Test size = 5000
-- Files permutation, which order determines train, valid and test sets is /gpfswork/rech/srp/commun/python3/tifs_protocol_efficientnet/models/permutation_files.npy
-- Model efnet is trained during 30 epochs 
-- Pair training is not used 
-- Batch size is 8 
-- Curriculum is used : the embedding rate starts from 1.0 and decreases every two epochs by factor 0.9 to reach target embedding rate 0.4
- 
+- Files permutation, which order determines train, valid and test sets is ./models/permutation_files.npy
 - Model xunet is trained during 30 epochs 
 - Pair training is used 
-- Batch size is 2*32 
+- Batch size is 2*30 
 - Curriculum is used : the embedding rate starts from 1.2 and decreases every two epochs by factor 0.9 to reach target embedding rate 0.4
  
-- Model srnet is trained during 30 epochs 
+- Model srnet is trained during 40 epochs 
 - Pair training is used 
 - Batch size is 2*16 
-- Curriculum is used : the embedding rate starts from 1.0 and decreases every two epochs by factor 0.9 to reach target embedding rate 0.4
+- Curriculum is used : the embedding rate starts from 1.4 and decreases every two epochs by factor 0.9 to reach target embedding rate 0.4
+ 
+- Model efnet is trained during 30 epochs 
+- Pair training is not used 
+- Batch size is 16 
+- Curriculum is used : the embedding rate starts from 1.2 and decreases every two epochs by factor 0.9 to reach target embedding rate 0.4
  
 Attack setup 
 - The smoothing function is SGE 
-- Maximum number of steps is 2000 
-- Number of samples is 1 
-- Tau is initialized with value 10.0 and decreases by factor 0.5 when needed
+- Maximum number of steps is 200 
+- Number of samples is 2 
+- Tau is initialized with value 5.0 and decreases by factor 0.5 when needed
 - The exit condition is required to be respected with precision = 0.01
  

job.slurm

@@ -4,15 +4,20 @@ module load pytorch-gpu/py3/1.7.1
 	
 python main.py --label='70' \
 	--begin_step=1 \
+	--permutation_files='./models/permutation_files.npy' \
+	--serveur='no' \
 	--number_steps=1 \
 	--folder_model='./models/' \
-	--data_dir_prot='./experiment/' \
-	--data_dir_cover= \
-	--data_dir_stego_0= \
-	--cost_dir= \
+	--data_dir_prot='./experience_512_75_0_4_SGE_efnetB0s1_xunet_srnet/' \
+	--data_dir_cover='./experience_512_75_0_4_SGE_efnetB0s1_xunet_srnet/cover/c_coeffs/' \
+	--data_dir_stego_0='./experience_512_75_0_4_SGE_efnetB0s1_xunet_srnet/data_adv_0/J_UNI_0_4_npy/' \
+	--cost_dir='./experience_512_75_0_4_SGE_efnetB0s1_xunet_srnet/cover/costs/' \
 	--strategy='minmax' \
 	--image_size=512 \
 	--QF=75 \
+	--train_size=4000 \
+    --valid_size=1000 \
+    --test_size=5000 \
 	--emb_rate=0.4 \
 	--model='xunet,srnet,efnet' \
 	--version_eff='b0' \
@@ -41,7 +46,9 @@ python main.py --label='70' \
 	--N_samples=2 \
 	--attack='SGE' \
 	--attack_last='no' \
-	--lr=0.01
+	--lr=0.01 \
+	--batch_adv=10000
+
 
 
 

main.py

@@ -68,7 +68,7 @@ def run_iteration(iteration_step, label, data_dir_prot, data_dir_cover, data_dir
                   image_size, QF, folder_model, permutation_files, version_eff, stride, n_loops,
                   model, train_size, valid_size, test_size, attack, attack_last, emb_rate,
                   batch_adv, n_iter_max_backpack, N_samples, tau_0, precision, lr,
-                  num_of_threads, training_dictionnary, spatial, strategy):
+                  num_of_threads, training_dictionnary, spatial, strategy, serveur):
 
     n_images = train_size+valid_size+test_size
     models = model.split(',')
@@ -80,19 +80,27 @@ def run_iteration(iteration_step, label, data_dir_prot, data_dir_cover, data_dir
                              batch_adv, n_iter_max_backpack, N_samples, tau_0, precision, lr,
                              num_of_threads, training_dictionnary, spatial))
 
-    if(iteration_step > 0):
+    if(iteration_step==0):
+        create_folder(data_dir_prot, ['data_adv_0', 'cover'])
+
+    else:
 
         # GENERATE ADV DATA BASE OF THE BEST LAST CLASSIFIER
         directory_adv = data_dir_prot+'data_adv_'+str(iteration_step)+'/'
-        create_folder(directory_adv, ['adv_final', 'adv_cost', ])
+        create_folder(directory_adv, ['adv_final', 'adv_cost'])
 
         print('Generating adv step ' + str(iteration_step))
         num_batch = n_images // batch_adv
         command = 'script_attack.py ' + \
             custom_command('attack', iteration_step, model)
+        
+        if(serveur):
+            command += ' --idx_start=$SLURM_ARRAY_TASK_ID'
             run_job('attack', label, command, iteration_step,
                     gpu=True, num_batch=num_batch)
             wait(label)
+        else:
+            os.system('python '+ command + ' --idx_start=0')
 
         # EVALUATION OF ALL THE CLASSIFIERS ON THE NEW ADV DATA BASE
         for i in range(iteration_step):
@@ -110,9 +118,14 @@ def run_iteration(iteration_step, label, data_dir_prot, data_dir_cover, data_dir
                 command = 'script_evaluate_classif.py' + custom_command('classif', iteration_step, my_model) \
                     + ' --iteration_f=' + \
                     str(i)+' --iteration_adv='+str(iteration_step)
+
+                if(serveur):
                     run_job('eval_'+str(my_model), label,
                         command, iteration_step, gpu=True)
+                else:
+                    os.system('python '+ command)
                 
+        if(serveur):
             wait(label)
 
         # GENERATION OF THE TRAIN DATA BASE
@@ -126,9 +139,14 @@ def run_iteration(iteration_step, label, data_dir_prot, data_dir_cover, data_dir
                       'train_'+my_model+'_'+str(iteration_step)])
         command = 'script_train.py' + \
             custom_command('train', iteration_step, my_model)
+        
+        if(serveur):
             run_job('train_'+my_model, label, command, iteration_step,
                     num_of_threads=num_of_threads, gpu=True)
+        else:
+            os.system('python '+ command)
 
+    if(serveur):
         wait(label)
 
     # EVALUATE NEW CLASSIFIERS ON ALL STEGO DATA BASES AND ON COVER
@@ -145,9 +163,14 @@ def run_iteration(iteration_step, label, data_dir_prot, data_dir_cover, data_dir
             command = 'script_evaluate_classif.py' + custom_command('classif', iteration_step, my_model) \
                 + ' --iteration_f='+str(iteration_step) + \
                 ' --iteration_adv='+str(i)
+
+            if(serveur):
                 run_job('eval_'+str(model), label,
                     command, iteration_step, gpu=True)
+            else:
+                os.system('python '+ command)
 
+    if(serveur):
         wait(label)
 
     for my_model in models:
@@ -157,7 +180,7 @@ def run_iteration(iteration_step, label, data_dir_prot, data_dir_cover, data_dir
     return(True)
 
 
-def run_protocol(begin_step, number_steps, label, data_dir_prot, data_dir_cover, data_dir_stego_0, cost_dir,
+def run_protocol(begin_step, serveur, number_steps, label, data_dir_prot, data_dir_cover, data_dir_stego_0, cost_dir,
                  image_size, QF, folder_model, permutation_files, version_eff, stride, n_loops,
                  model, train_size, valid_size, test_size, attack, attack_last, emb_rate,
                  batch_adv, n_iter_max_backpack, N_samples, tau_0, precision, lr, strategy,
@@ -185,7 +208,7 @@ def run_protocol(begin_step, number_steps, label, data_dir_prot, data_dir_cover,
                       image_size, QF, folder_model, permutation_files, version_eff, stride, n_loops,
                       model, train_size, valid_size, test_size, attack, attack_last, emb_rate,
                       batch_adv, n_iter_max_backpack, N_samples, tau_0, precision, lr,
-                      num_of_threads, training_dictionnary, spatial, strategy)
+                      num_of_threads, training_dictionnary, spatial, strategy, serveur)
 
         iteration_step += 1
 
@@ -194,6 +217,7 @@ if __name__ == '__main__':
 
     argparser = argparse.ArgumentParser(sys.argv[0])
     argparser.add_argument('--begin_step', type=int)
+    argparser.add_argument('--serveur', type=str, default='no')
     argparser.add_argument('--number_steps', type=int, default=10)
     argparser.add_argument('--folder_model', type=str,
                            help='The path to the folder where the architecture of models are saved')
@@ -233,7 +257,7 @@ if __name__ == '__main__':
 
     # FOR ADVERSARIAL COST MAP
     argparser.add_argument('--attack', type=str)
-    argparser.add_argument('--attack_last', type=str)
+    argparser.add_argument('--attack_last', type=str,default='no')
     argparser.add_argument('--lr', type=float)
     argparser.add_argument('--batch_adv', type=int, default=100)
     argparser.add_argument('--n_iter_max_backpack', type=int)
@@ -277,4 +301,6 @@ if __name__ == '__main__':
 
     params = argparser.parse_args()
 
+    params.serveur = params.serveur=='yes'
+
     run_protocol(**vars(params))