Merge pull request #1500 from quantopian/make-metadata-optional

MAINT: Adds option for minute bar writer to not write metadata

Author: Andrew Daniels

tests/data/test_minute_bars.py

@@ -33,7 +33,6 @@
     Timedelta,
     NaT,
     date_range,
-    isnull,
 )
 
 from zipline.data.bar_reader import NoDataOnDate
@@ -971,8 +970,15 @@ def test_truncate_between_data_points(self):
         # Truncate to first day with data.
         self.writer.truncate(days[0])
 
+        # Refresh the reader since truncate update the metadata.
+        self.reader = BcolzMinuteBarReader(self.dest)
+
         self.assertEqual(self.writer.last_date_in_output_for_sid(sid), days[0])
 
+        cal = self.trading_calendar
+        _, last_close = cal.open_and_close_for_session(days[0])
+        self.assertEqual(self.reader.last_available_dt, last_close)
+
         minute = minutes[0]
 
         open_price = self.reader.get_value(sid, minute, 'open')
@@ -995,28 +1001,6 @@ def test_truncate_between_data_points(self):
 
         self.assertEquals(50.0, volume_price)
 
-        minute = minutes[1]
-
-        open_price = self.reader.get_value(sid, minute, 'open')
-
-        self.assertTrue(isnull(open_price))
-
-        high_price = self.reader.get_value(sid, minute, 'high')
-
-        self.assertTrue(isnull(high_price))
-
-        low_price = self.reader.get_value(sid, minute, 'low')
-
-        self.assertTrue(isnull(low_price))
-
-        close_price = self.reader.get_value(sid, minute, 'close')
-
-        self.assertTrue(isnull(close_price))
-
-        volume_price = self.reader.get_value(sid, minute, 'volume')
-
-        self.assertEqual(0.0, volume_price)
-
     def test_truncate_all_data_points(self):
 
         tds = self.market_opens.index
@@ -1044,51 +1028,15 @@ def test_truncate_all_data_points(self):
         # day with minute data.
         self.writer.truncate(self.test_calendar_start)
 
+        # Refresh the reader since truncate update the metadata.
+        self.reader = BcolzMinuteBarReader(self.dest)
+
         self.assertEqual(
             self.writer.last_date_in_output_for_sid(sid),
             self.test_calendar_start,
         )
 
-        minute = minutes[0]
-
-        open_price = self.reader.get_value(sid, minute, 'open')
-
-        self.assertTrue(isnull(open_price))
-
-        high_price = self.reader.get_value(sid, minute, 'high')
-
-        self.assertTrue(isnull(high_price))
-
-        low_price = self.reader.get_value(sid, minute, 'low')
-
-        self.assertTrue(isnull(low_price))
-
-        close_price = self.reader.get_value(sid, minute, 'close')
-
-        self.assertTrue(isnull(close_price))
-
-        volume_price = self.reader.get_value(sid, minute, 'volume')
-
-        self.assertEquals(0.0, volume_price)
-
-        minute = minutes[1]
-
-        open_price = self.reader.get_value(sid, minute, 'open')
-
-        self.assertTrue(isnull(open_price))
-
-        high_price = self.reader.get_value(sid, minute, 'high')
-
-        self.assertTrue(isnull(high_price))
-
-        low_price = self.reader.get_value(sid, minute, 'low')
-
-        self.assertTrue(isnull(low_price))
-
-        close_price = self.reader.get_value(sid, minute, 'close')
-
-        self.assertTrue(isnull(close_price))
-
-        volume_price = self.reader.get_value(sid, minute, 'volume')
-
-        self.assertEqual(0.0, volume_price)
+        cal = self.trading_calendar
+        _, last_close = cal.open_and_close_for_session(
+            self.test_calendar_start)
+        self.assertEqual(self.reader.last_available_dt, last_close)

zipline/data/minute_bars.py

@@ -317,6 +317,10 @@ class BcolzMinuteBarWriter(object):
 
         Defaults to supporting 15 years of NYSE equity market data.
         see: http://bcolz.blosc.org/opt-tips.html#informing-about-the-length-of-your-carrays # noqa
+    write_metadata : bool, optional
+        If True, writes the minute bar metadata (on init of the writer).
+        If False, no metadata is written (existing metadata is
+        retained). Default is True.
 
     Notes
     -----
@@ -373,7 +377,8 @@ def __init__(self,
                  minutes_per_day,
                  default_ohlc_ratio=OHLC_RATIO,
                  ohlc_ratios_per_sid=None,
-                 expectedlen=DEFAULT_EXPECTEDLEN):
+                 expectedlen=DEFAULT_EXPECTEDLEN,
+                 write_metadata=True):
 
         self._rootdir = rootdir
         self._start_session = start_session
@@ -391,15 +396,16 @@ def __init__(self,
         self._minute_index = _calc_minute_index(
             self._schedule.market_open, self._minutes_per_day)
 
-        metadata = BcolzMinuteBarMetadata(
-            self._default_ohlc_ratio,
-            self._ohlc_ratios_per_sid,
-            self._calendar,
-            self._start_session,
-            self._end_session,
-            self._minutes_per_day,
-        )
-        metadata.write(self._rootdir)
+        if write_metadata:
+            metadata = BcolzMinuteBarMetadata(
+                self._default_ohlc_ratio,
+                self._ohlc_ratios_per_sid,
+                self._calendar,
+                self._start_session,
+                self._end_session,
+                self._minutes_per_day,
+            )
+            metadata.write(self._rootdir)
 
     @property
     def first_trading_day(self):
@@ -795,6 +801,11 @@ def truncate(self, date):
                 )
                 shutil.move(tmp_path, sid_path)
 
+        # Update end session in metadata.
+        metadata = BcolzMinuteBarMetadata.read(self._rootdir)
+        metadata.end_session = date
+        metadata.write(self._rootdir)
+
 
 class BcolzMinuteBarReader(MinuteBarReader):
     """