Merge branch 'master' of github.com:bootphon/abnet3

Author: Rachine

abnet3/dataloader.py

@@ -11,8 +11,9 @@
 import h5features
 import argparse
 
-from abnet3.utils import read_feats
+from abnet3.utils import read_feats, EmbeddingObserver
 from abnet3.model import *
+from abnet3.integration import BiWeightedDeepLearnt
 
 
 class EmbedderBuilder:
@@ -64,6 +65,9 @@ def embed(self):
         if self.network_path is not None:
             self.network.load_network(self.network_path)
         self.network.eval()
+
+        if self.cuda:
+            self.network.cuda()
         print("Done loading network weights")
 
         with h5features.Reader(self.feature_path, 'features') as fh:
@@ -112,6 +116,9 @@ def embed(self):
             self.network.load_network(self.network_path)
         self.network.eval()
 
+        if self.cuda:
+            self.network.cuda()
+
         with h5features.Reader(self.feature_path, 'features') as fh:
             features = fh.read()
 
@@ -140,3 +147,75 @@ def embed(self):
 
         with h5features.Writer(self.output_path+'.phn') as fh:
             fh.write(data_phn, 'features')
+
+class MultimodalEmbedder(EmbedderBuilder):
+    """
+    Embedder class for multimodal siamese network
+    """
+
+    def __init__(self, *args, **kwargs):
+        super(MultimodalEmbedder, self).__init__(*args, **kwargs)
+        self.observers = [] #tuples list, of the form (EmbedderObserver,
+                                                      #function to get the data,
+                                                      #path to be saved)
+
+        if isinstance(self.network.integration_unit, BiWeightedDeepLearnt):
+            print("Placing observer to save learnt attention weights")
+            self.observers.append(EmbeddingObserver(
+                                 self.network.integration_unit.get_weights,
+                                 self.output_path+"attention_weights.features"))
+
+    def embed(self):
+        """
+        Embed method to embed features based on a saved network
+        """
+
+        if self.network_path is not None:
+            self.network.load_network(self.network_path)
+        self.network.eval()
+
+        if self.cuda:
+            self.network.cuda()
+
+        items = None
+        times = None
+        features_list = []
+        for path in self.feature_path:
+            with h5features.Reader(path, 'features') as fh:
+                features = fh.read()
+                features_list.append(features.features())
+                check_items = features.items()
+                check_times = features.labels()
+            if not items:
+                items = check_items
+            if not times:
+                times = check_times
+
+        print("Done loading input feature file")
+
+        zipped_feats = zip(*features_list)
+        embeddings = []
+        for feats in zipped_feats:
+            modes_list = []
+            for feat in feats:
+                if feat.dtype != np.float32:
+                    feat = feat.astype(np.float32)
+                feat_torch = Variable(torch.from_numpy(feat), volatile=True)
+                if self.cuda:
+                    feat_torch = feat_torch.cuda()
+                modes_list.append(feat_torch)
+            emb, _ = self.network(modes_list, modes_list)
+            emb = emb.cpu()
+            embeddings.append(emb.data.numpy())
+
+            #Register activity on observer
+            for observer in self.observers:
+                observer.register_status()
+
+        data = h5features.Data(items, times, embeddings, check=True)
+        with h5features.Writer(self.output_path + "embedded.features") as fh:
+            fh.write(data, 'features')
+
+        #Save observer registers
+        for observer in self.observers:
+            observer.save(items, times)

abnet3/embedder.py

@@ -12,7 +12,7 @@
 import os
 import h5py
 import shutil
-import tempfile
+import argparse
 
 from abnet3.utils import read_vad_file, read_feats, Features_Accessor
 
@@ -341,6 +341,27 @@ def load_mean_variance(self, file_path):
 
         return {'mean': mean_var[0], 'variance': mean_var[1]}
 
+
+    def normalize(self, input_features, output_features):
+        print("Normalizing")
+        if self.norm_per_file:
+            self.mean_var_norm_per_file(input_features, output_features,
+                                        vad_file=self.vad_file)
+        else:
+            if self.load_mean_variance_path is not None:
+                params = self.load_mean_variance(
+                    file_path=self.load_mean_variance_path)
+            else:
+                params = None
+            mean, variance = self.mean_variance_normalisation(
+                input_features, output_features, params=params,
+                vad_file=self.vad_file
+            )
+            if self.save_mean_variance_path is not None:
+                self.save_mean_variance(
+                    mean, variance,
+                    output_file=self.save_mean_variance_path)
+
     def generate(self):
 
         functions = {
@@ -370,23 +391,7 @@ def generate(self):
             if self.normalization:
                 print("Normalizing")
                 h5_temp2 = tempdir + '/temp2'
-                if self.norm_per_file:
-                    self.mean_var_norm_per_file(h5_temp1, h5_temp2,
-                                                vad_file=self.vad_file)
-                else:
-                    if self.load_mean_variance_path is not None:
-                        params = self.load_mean_variance(
-                            file_path=self.load_mean_variance_path)
-                    else:
-                        params = None
-                    mean, variance = self.mean_variance_normalisation(
-                        h5_temp1, h5_temp2, params=params,
-                        vad_file=self.vad_file
-                    )
-                    if self.save_mean_variance_path is not None:
-                        self.save_mean_variance(
-                            mean, variance,
-                            output_file=self.save_mean_variance_path)
+                self.normalize(h5_temp1, h5_temp2)
             else:
                 h5_temp2 = h5_temp1
             if self.stack:
@@ -397,3 +402,93 @@ def generate(self):
                 shutil.copy(h5_temp2, self.output_path)
         finally:
             shutil.rmtree(tempdir)
+
+
+
+def main_wav(args):
+
+    features_generator = FeaturesGenerator(
+        files=args.wav_dir,
+        output_path=args.output_path,
+        method=args.method,
+        n_filters=args.n_filters,
+        save_mean_variance_path=args.save_mean_var,
+        load_mean_variance_path=args.load_mean_var,
+        vad_file=args.vad,
+        normalization=args.normalization,
+        stack=True,
+        norm_per_file=args.norm_per_file,
+        norm_per_channel=args.norm_per_channel,
+    )
+
+    features_generator.generate()
+
+def main_normalize(args):
+    features_generator = FeaturesGenerator(
+        save_mean_variance_path=args.save_mean_var,
+        load_mean_variance_path=args.load_mean_var,
+        vad_file=args.vad,
+        normalization=True,
+        norm_per_file=args.norm_per_file,
+        norm_per_channel=args.norm_per_channel,
+    )
+
+    features_generator.normalize(
+        args.input_features,
+        args.output_features
+    )
+
+def main():
+    parser = argparse.ArgumentParser()
+
+    subparsers = parser.add_subparsers(help='sub-command help')
+    
+    parser_wav = subparsers.add_parser("wav")
+    parser_normalize = subparsers.add_parser("norm")
+
+
+    parser_wav.add_argument("wav_dir", help="Path to wav directory")
+    parser_wav.add_argument("output_path", help="Path to output h5f file")
+    parser_wav.add_argument("method", choices=["mfcc", "fbanks"],
+                        help="which features to generate")
+    parser_wav.add_argument("--vad", help="Path to vad file "
+                                      "(CSV, seconds with header)")
+    parser_wav.add_argument("--normalization", "-n", action="store_true")
+    parser_wav.add_argument("--norm-per-file", action="store_true",
+                        help="Independent normalization for each file")
+    parser_wav.add_argument("--norm-per-channel", action="store_true",
+                        help="Normalize each channel independently")
+    parser_wav.add_argument("--n-filters", type=int, default=40)
+    parser_wav.add_argument("--save-mean-var", type=str,
+                        help="Path to emplacement where mean / var"
+                             "will be saved")
+    parser_wav.add_argument("--load-mean-var", type=str,
+                        help="Path to emplacement where mean / var"
+                             "are saved. Will be used to compute test features")
+
+    parser_wav.set_defaults(func=main_wav)
+
+    parser_normalize.add_argument("input_features", help="Path to input h5f file")
+    parser_normalize.add_argument("output_features", help="Path to output h5f file")
+    parser_normalize.add_argument("--vad", help="Path to vad file "
+                                      "(CSV, seconds with header)")
+    parser_normalize.add_argument("--norm-per-file", action="store_true",
+                        help="Independent normalization for each file")
+    parser_normalize.add_argument("--norm-per-channel", action="store_true",
+                        help="Normalize each channel independently")
+    parser_normalize.add_argument("--save-mean-var", type=str,
+                        help="Path to emplacement where mean / var"
+                             "will be saved")
+    parser_normalize.add_argument("--load-mean-var", type=str,
+                        help="Path to emplacement where mean / var"
+                             "are saved. Will be used to compute test features")
+    
+    parser_normalize.set_defaults(func=main_normalize)
+
+    args = parser.parse_args()
+    if args.func:
+        args.func(args)
+
+
+if __name__ == '__main__':
+    main()

abnet3/features.py

@@ -139,6 +139,9 @@ def run_single_experiment(self, single_experiment=None, gpu_id=0):
 
         os.makedirs(single_experiment['pathname_experience'], exist_ok=True)
 
+        with open(os.path.join(single_experiment['pathname_experience'], 'exp.yml'), 'w') as f:
+            yaml.dump(single_experiment, f, default_flow_style=False)
+
         features_prop = single_experiment['features']
         features_class = getattr(abnet3.features, features_prop['class'])
         arguments = features_prop['arguments']
@@ -170,7 +173,8 @@ def run_single_experiment(self, single_experiment=None, gpu_id=0):
         dataloader_prop = single_experiment['dataloader']
         dataloader_class = getattr(abnet3.dataloader, dataloader_prop['class'])
         arguments = dataloader_prop['arguments']
-        arguments['pairs_path'] = sampler.directory_output
+        if not 'pairs_path' in arguments:
+            arguments['pairs_path'] = sampler.directory_output
         arguments['features_path'] = features.output_path
         dataloader = dataloader_class(**arguments)