removed comment about streams

shmh40 · shmh40 · commit ea726caa69fb · 2025-06-27T15:12:01.000Z
diff --git a/src/weathergen/datasets/masking.py b/src/weathergen/datasets/masking.py
@@ -71,7 +71,6 @@ def mask_source(
             )
 
         # Handle the special case where all tokens are masked
-        # NOTE: not going to handle different streams correctly.
         if rate == 1.0:
             token_lens = [len(t) for t in tokenized_data]
             self.perm_sel = [np.ones(l, dtype=bool) for l in token_lens]

Original file line number	Diff line number	Diff line change
`@@ -71,7 +71,6 @@ def mask_source(`
`71`	`71`	`)`
`72`	`72`
`73`	`73`	`# Handle the special case where all tokens are masked`
`74`		`- # NOTE: not going to handle different streams correctly.`
`75`	`74`	`if rate == 1.0:`
`76`	`75`	`token_lens = [len(t) for t in tokenized_data]`
`77`	`76`	`self.perm_sel = [np.ones(l, dtype=bool) for l in token_lens]`