Merge pull request #1 from eshaz/multi-threading

Multi Threading

Author: eshaz

README.md

@@ -3,16 +3,16 @@
 `synaudio` is a JavaScript library that finds the synchronization point between two similar audio clips.
   * Synchronize two audio clips by finding the sample offset with the [Pearson correlation coefficient](https://en.wikipedia.org/wiki/Pearson_correlation_coefficient).
   * Correlation algorithm implemented as WebAssembly SIMD.
-  * Built in Web Worker implementation for easy multithreading.
+  * Built in Web Worker implementations for concurrency.
 
 ---
 
 * [Installing](#installing)
 * [Usage](#usage)
 * [API](#api)
   * [Options](#options)
-  * [Types](#types)
   * [Methods](#methods)
+  * [Types](#types)
 
 ## Installing
 
@@ -32,7 +32,7 @@
    });
    ```
 
-1. Call `sync` or `syncWorker` method on the instance to find the synchronization point in samples between two audio clips.
+1. Call the `sync`, `syncWorker`, or `syncWorkerConcurrent` method on the instance to find the synchronization point in samples between two audio clips.
 
    * See the [API](#api) section below for details on these methods.
 
@@ -50,12 +50,13 @@
    }
 
    const {
-     sampleOffset,  // position relative to `base` where `comparison` matches best
-     correlation    // covariance coefficient of the match [ ranging -1 (worst) to 1 (best) ]
-   } = await synAudio.syncWorker(
-         base,      // audio data to use a base for the comparison
-         comparison // audio data to compare against the base
-       );
+     sampleOffset, // position relative to `base` where `comparison` matches best
+     correlation, //  covariance coefficient of the match [ ranging -1 (worst) to 1 (best) ]
+   } = await synAudio.syncWorkerConcurrent(
+     base, //        audio data to use a base for the comparison
+     comparison, //  audio data to compare against the base
+     4 //            number of threads to spawn
+   );
    ```
 
 ## API
@@ -64,22 +65,74 @@
 
 Class that that finds the synchronization point between two similar audio clips.
 
-### Options
-
 ```js
-const synAudio = new SynAudio({
-  correlationSampleSize: 5000,
-  initialGranularity: 16,
+new SynAudio({
+  correlationSampleSize: 1234,
+  initialGranularity: 1
 });
 ```
 
+### Options
+```ts
+declare interface SynAudioOptions {
+  correlationSampleSize?: number; // default 11025
+  initialGranularity?: number; // default 16
+}
+```
 * `correlationSampleSize` *optional, defaults to 11025*
   * Number of samples to compare while finding the best offset
   * Higher numbers will increase accuracy at the cost of slower computation
 * `initialGranularity` *optional, defaults to 16*
   * Number of samples to jump while performing the first pass search
   * Higher numbers will decrease accuracy at the benefit of much faster computation
 
+
+### Methods
+```ts
+declare class SynAudio {
+  constructor(options?: SynAudioOptions);
+
+  public async sync(
+    base: PCMAudio,
+    comparison: PCMAudio
+  ): Promise<SynAudioResult>;
+
+  public async syncWorker(
+    base: PCMAudio,
+    comparison: PCMAudio
+  ): Promise<SynAudioResult>;
+
+  public async syncWorkerConcurrent(
+    base: PCMAudio,
+    comparison: PCMAudio,
+    threads?: number // default 1
+  ): Promise<SynAudioResult>;
+}
+```
+
+* `sync(base: PCMAudio, comparison: PCMAudio): SynAudioResult`
+  * Executes on the main thread.
+  * Parameters
+    * `base` Audio data to compare against
+    * `comparison` Audio data to use as a comparison
+  * Returns
+    * `SynAudioResult` containing the `correlation` and `sampleOffset`
+* `syncWorker(base: PCMAudio, comparison: PCMAudio): SynAudioResult`
+  * Execute in a separate thread as a web worker.
+  * Parameters
+    * `base` Audio data to compare against
+    * `comparison` Audio data to use as a comparison
+  * Returns
+    * `SynAudioResult` containing the `correlation` and `sampleOffset`
+* `syncWorkerConcurrent(base: PCMAudio, comparison: PCMAudio, threads: number): SynAudioResult`
+  * Splits the incoming data into chunks and spawns multiple workers that execute concurrently.
+  * Parameters
+    * `base` Audio data to compare against
+    * `comparison` Audio data to use as a comparison
+    * `threads` Number of threads to spawn *optional, defaults to 1*
+  * Returns
+    * `SynAudioResult` containing the `correlation` and `sampleOffset`
+
 ### Types
 
 ```ts
@@ -105,21 +158,4 @@ interface SynAudioResult {
   * Correlation coefficient of the `base` and `comparison` audio at the `sampleOffset`
   * Ranging from -1 (worst) to 1 (best)
 * `sampleOffset`
-  * Number of samples relative to `base` where `comparison` has the highest correlation
-
-### Methods
-
-* `sync(base: PCMAudio, comparison: PCMAudio): SynAudioResult`
-  * Executes on the main thread.
-  * Parameters
-    * `base` Audio data to compare against
-    * `comparison` Audio data to use as a comparison
-  * Returns
-    * `SynAudioResult` containing the `correlation` and `sampleOffset`
-* `syncWorker(base: PCMAudio, comparison: PCMAudio): SynAudioResult`
-  * Execute in a separate thread as a web worker
-  * Parameters
-    * `base` Audio data to compare against
-    * `comparison` Audio data to use as a comparison
-  * Returns
-    * `SynAudioResult` containing the `correlation` and `sampleOffset`
+  * Number of samples relative to `base` where `comparison` has the highest correlation

index.d.ts

@@ -9,21 +9,24 @@ declare interface SynAudioResult {
 }
 
 declare interface SynAudioOptions {
-  correlationSampleSize?: number;
-  initialGranularity?: number;
+  correlationSampleSize?: number; // default 11025
+  initialGranularity?: number; // default 16
 }
 
 declare class SynAudio {
   constructor(options?: SynAudioOptions);
 
-  public async sync(
+  public sync(base: PCMAudio, comparison: PCMAudio): Promise<SynAudioResult>;
+
+  public syncWorker(
     base: PCMAudio,
     comparison: PCMAudio
   ): Promise<SynAudioResult>;
 
-  public async syncWorker(
+  public syncWorkerConcurrent(
     base: PCMAudio,
-    comparison: PCMAudio
+    comparison: PCMAudio,
+    threads?: number // default 1
   ): Promise<SynAudioResult>;
 }
 

package-lock.json

@@ -1,7 +1,12 @@
 {
   "name": "synaudio",
-  "version": "0.0.1",
+  "version": "0.1.0",
   "description": "Library that finds the synchronization point between two similar audio clips.",
+  "files": [
+    "index.js",
+    "index.d.ts",
+    "src/SynAudio.js"
+  ],
   "keywords": [
     "wasm",
     "simd",

package.json

@@ -23,6 +23,7 @@ import Worker from "web-worker";
 const simd=async()=>WebAssembly.validate(new Uint8Array([0,97,115,109,1,0,0,0,1,5,1,96,0,1,123,3,2,1,0,10,10,1,8,0,65,0,253,15,253,98,11]))
 
 const wasmModule = new WeakMap();
+const webworkerSource = new WeakMap();
 
 /* WASM strings are embeded during the build */
 const simdWasm = String.raw`dynEncode0064dÅ×ÑedddereÄnããããããããããdfsegÉÒÚjÑÉÑÓÖÝfdfgfedjleãd¥äìhokfmÇÓÖÖÉÐÅØÉddoÃÃÌÉÅÔÃÆÅ×Égdnwewhvãgàkßjá„e„j„e„j¬†v¥hӅwf¤„e¥d°qd„f¥f¬qd„f¥eυx¥d„eυy„e¥eՅz„e¥àՅo„e¥f؆{„d΅|„e¥hφ}¥fÚ¥eΆf¥bccckՅp„f¥eՅ~„e¥h­…tg¤„e„sЅq¥d…jf¤f¤„tqd„d„d„e„qÎ¥fØέ„d„q¥fØ΄|­Õqd¥d…f„}¥h³h¤„p…n„d…jg¤„j„j„rΆuadfd„jadfdaHeaofd„j„uadft„jadftaHeaoft„j¥„Î…j„f¥l΅f„n¥fφnqdoo„~h¤„d„f¥fØΆj„d„f„qÎ¥fØÎadfd„jadfdaHeaofdo„o†j„eªqeo„j¥ãׅf„zh¤„d„j¥fØΆn„d„j„qÎ¥fØΎfd„nŽfdöœfd„j¥eօjo„f„yªqd„e„jυn„j¥f؆q„r΅u„d…jg¤„j„qΆf„j„uΆŽfd„fŽfdöœfd„f„Žfh„fŽfhöœfh„j¥l΅j„n¥fφnqdoo„r„{΅r„x„s¥eΆs«qdoo„v„wυof¤„i¥f¬qd„h„o„h„o¬†p¥d°qd„i¥eυx„p¥eՅy„p¥àՅh„p¥f؆z„g΅{„p¥hφ|¥fÚ¥eΆf¥bccckՅi„f¥eՅ}¥d…r„p¥h­…~¥d…sg¤„p„sЅq¥d…jf¤f¤„~qd„g„g„p„qÎ¥fØέ„g„q¥fØ΄{­Õqd¥d…f„|¥h³h¤„i…n„g…jg¤„j„j„rΆtadfd„jadfdaHeaofd„j„tadft„jadftaHeaoft„j¥„Î…j„f¥l΅f„n¥fφnqdoo„}h¤„g„f¥fØΆj„g„f„qÎ¥fØÎadfd„jadfdaHeaofdo„p„h†jªqeo„j¥eօf„yh¤„g„j¥fØΆn„g„j„qÎ¥fØΎfd„nŽfdöœfd„f…jo„f„pªqd„p„jυn„j¥f؆q„r΅t„g…jg¤„j„qΆf„j„tΆuŽfd„fŽfdöœfd„f„uŽfh„fŽfhöœfh„j¥l΅j„n¥fφnqdoo„r„z΅r„x„s¥eΆs«qdoo¥d…i„m¥dšfd„l¥dšfd„e„oυpf¤„o¥d°qd„o¥gՅff¤„v„w¥ã×Άe¥g­h¤¥d…hpeo„d…j„o¥àՆh…ng¤„€„jŽfd„jŽfh„jŽfl„jŽfp…€„j¥t΅j„n¥hφnqdoo„fh¤„d„h¥fØ΅j„f…ng¤„€„jŽfd…€„j¥h΅j„n¥eφnqdoof¤„e¥g­h¤¥d…epeo„g…j„o¥àՆe…ng¤„„jŽfd„jŽfh„jŽfl„jŽfp…„j¥t΅j„n¥hφnqdoo„f©qd„g„e¥fØ΅jg¤„„jŽfd…„j¥h΅j„f¥eφfqdoo„„o†Œ…Ž„p¥d®h¤„k¥f؅r„Œ§ddä#ö…Š„o¥hυs„Žaw…ˆ„o…„o¥i¬…q„d…e¥d…hg¤„d„h¥fØΎfd…‚„d„h„oÎ¥fØΎfd…f¤„qh¤apdddddddddddddddd…ƒapdddddddddddddddd…„apdddddddddddddddd……peo„€„aw…‰¥d…n„e…j„g…fapdddddddddddddddd……apdddddddddddddddd…„apdddddddddddddddd…ƒg¤„ƒ„jaddd„‰aIe††„faddd„ˆaIe†‡aJeaHe…ƒ„j¥t΅j„f¥t΅f„…„‡„‡aJeaHe……„„„†„†aJeaHe…„„s„n¥hΆn®qdoo„ƒaƒg„ƒaƒf„ƒaƒd„ƒaƒeööö„Šù„„aƒg„„aƒf„„aƒd„„aƒeööö„Šù„…aƒg„…aƒf„…aƒd„…aƒeööö„Šù†‹„Âh¤„m„hšfd„l„‹œfd„‹…„h…io„€„‚„…€„e„r΅e„p„h„kΆh®qdoo„k¥e؆e„iΆf„p„f„p¬…k„i„eφe¥d„e¥d®…hf¤„o¥d°h¤¨dddddddd…€peo„v„w¥ã×΅ifã„o¥gՆe©h¤¨dddddddd…€„hpeo„d„h¥fØ΅j¨dddddddd…€„e…fg¤„€„jŽfd…€„j¥h΅j„f¥eφfqdo„e„hÎo…e„i¥g­qd„d„e¥fØ΅j„h„v΄eτwυfg¤„€„jŽfd„jŽfh„jŽfl„jŽfp…€„j¥t΅j„f¥hφfqdoo„h„k¬h¤„Œ§ddä#ö…Š„o¥hυi„d„h¥fØ΅e„Žaw…ˆ„o…„o¥i¬…pg¤„d„h¥fØΎfd…‚„d„h„oÎ¥fØΎfd…Œf¤„ph¤apdddddddddddddddd…ƒapdddddddddddddddd…„apdddddddddddddddd……peo„€„aw…‰¥d…n„e…j„g…fapdddddddddddddddd……apdddddddddddddddd…„apdddddddddddddddd…ƒg¤„ƒ„jaddd„‰aIe††„faddd„ˆaIe†‡aJeaHe…ƒ„j¥t΅j„f¥t΅f„…„‡„‡aJeaHe……„„„†„†aJeaHe…„„i„n¥hΆn®qdoo„ƒaƒg„ƒaƒf„ƒaƒd„ƒaƒeööö„Šù„„aƒg„„aƒf„„aƒd„„aƒeööö„Šù„…aƒg„…aƒf„…aƒd„…aƒeööö„Šù†‹„Âh¤„m„hšfd„l„‹œfd„‹…o„€„‚„Œ…€„e¥h΅e„k„h¥eΆh«qdoood~sØÅÖËÉØÃÊÉÅØÙÖÉ×ek×ÍÑȕ–œ`;
@@ -67,11 +68,6 @@ export default class SynAudio {
         const pageSize = 64 * 1024;
         const floatByteLength = Float32Array.BYTES_PER_ELEMENT;
 
-        if (a.sampleRate !== b.sampleRate)
-          throw new Error(
-            "SynAudio: sample rates of both audio data must be the same"
-          );
-
         const memory = new WebAssembly.Memory({
           initial:
             ((a.samplesDecoded * a.channelData.length +
@@ -142,36 +138,93 @@ export default class SynAudio {
     };
   }
 
+  async syncWorkerConcurrent(a, b, threads = 1) {
+    const promises = [];
+    const lengths = [];
+
+    // split a buffer into equal chunks for threads
+    // overlap at the end of the buffer by correlation sample size
+    let offset = 0;
+    for (let i = 1; i <= threads; i++) {
+      const aBufferLength = Math.ceil(a.samplesDecoded / threads);
+      const correlationSampleOverlap =
+        i === threads ? 0 : this._correlationSampleSize;
+
+      const aSplit = {
+        channelData: [],
+      };
+
+      for (const channel of a.channelData) {
+        const cutChannel = channel.subarray(
+          offset,
+          offset + aBufferLength + correlationSampleOverlap
+        );
+        aSplit.channelData.push(cutChannel);
+        aSplit.samplesDecoded = cutChannel.length;
+      }
+
+      const actualLength =
+        aBufferLength < a.samplesDecoded ? aBufferLength : a.samplesDecoded;
+      lengths.push(actualLength);
+      offset += actualLength;
+
+      promises.push(this.syncWorker(aSplit, b));
+    }
+
+    const results = await Promise.all(promises);
+
+    // find the result with the highest correlation and calculate the offset relative to the input data
+    let bestResultIdx = 0;
+    let bestCorrelation = -1;
+    for (let i = 0; i < results.length; i++)
+      if (results[i].correlation > bestCorrelation) {
+        bestResultIdx = i;
+        bestCorrelation = results[i].correlation;
+      }
+
+    return {
+      correlation: results[bestResultIdx].correlation,
+      sampleOffset:
+        results[bestResultIdx].sampleOffset +
+        lengths.slice(0, bestResultIdx).reduce((acc, len) => acc + len, 0),
+    };
+  }
+
   async syncWorker(a, b) {
-    const webworkerSourceCode =
-      "'use strict';" +
-      `(${((SynAudioWorker, correlationSampleSize, initialGranularity) => {
-        self.onmessage = ({ data: { module, a, b } }) => {
-          const worker = new SynAudioWorker(
-            Promise.resolve(module),
-            correlationSampleSize,
-            initialGranularity
-          );
-
-          worker.sync(a, b).then((results) => {
-            self.postMessage(results);
-          });
-        };
-      }).toString()})(${this.SynAudioWorker.toString()}, ${
-        this._correlationSampleSize
-      }, ${this._initialGranularity})`;
-
-    let type = "text/javascript",
-      source;
-
-    try {
-      // browser
-      source = URL.createObjectURL(new Blob([webworkerSourceCode], { type }));
-    } catch {
-      // nodejs
-      source = `data:${type};base64,${Buffer.from(webworkerSourceCode).toString(
-        "base64"
-      )}`;
+    let source = webworkerSource.get(this);
+
+    if (!source) {
+      const webworkerSourceCode =
+        "'use strict';" +
+        `(${((SynAudioWorker, correlationSampleSize, initialGranularity) => {
+          self.onmessage = ({ data: { module, a, b } }) => {
+            const worker = new SynAudioWorker(
+              Promise.resolve(module),
+              correlationSampleSize,
+              initialGranularity
+            );
+
+            worker.sync(a, b).then((results) => {
+              self.postMessage(results);
+            });
+          };
+        }).toString()})(${this.SynAudioWorker.toString()}, ${
+          this._correlationSampleSize
+        }, ${this._initialGranularity})`;
+
+      let type = "text/javascript";
+
+      try {
+        // browser
+        source = URL.createObjectURL(new Blob([webworkerSourceCode], { type }));
+      } catch {
+        // nodejs
+        source = `data:${type};base64,${Buffer.from(
+          webworkerSourceCode
+        ).toString("base64")}`;
+      }
+
+      webworkerSource.set(this, source);
     }
 
     const worker = new Worker(source, { name: "SynAudio" });