switched from panel to xarray.DataArray

pgportfolio/marketdata/datamatrices.py

@@ -53,12 +53,15 @@ def __init__(self, start, end, period, batch_size=50, volume_average_days=30, bu
             raise ValueError("market {} is not valid".format(market))
         self.__period_length = period
         # portfolio vector memory, [time, assets]
-        self.__PVM = pd.DataFrame(index=self.__global_data.minor_axis,
-                                  columns=self.__global_data.major_axis)
+        panelxr = self.__global_data
+        self.__PVM = pd.DataFrame(index=panelxr.dim_2.values,
+                                  columns=panelxr.dim_1.values)
+
         self.__PVM = self.__PVM.fillna(1.0 / self.__coin_no)
 
         self._window_size = window_size
-        self._num_periods = len(self.__global_data.minor_axis)
+        self._num_periods = len(panelxr.dim_2.values)
+
         self.__divide_data(test_portion, portion_reversed)
 
         self._portion_reversed = portion_reversed
@@ -170,6 +173,8 @@ def setw(w):
 
     # volume in y is the volume in next access period
     def get_submatrix(self, ind):
+        p=self.__global_data
+        a = p.values[:, :, ind:ind+self._window_size+1]
         return self.__global_data.values[:, :, ind:ind+self._window_size+1]
 
     def __divide_data(self, test_portion, portion_reversed):

pgportfolio/marketdata/globaldatamatrix.py

@@ -3,9 +3,10 @@
 from __future__ import print_function
 
 from pgportfolio.marketdata.coinlist import CoinList
+import xarray as xr
 import numpy as np
 import pandas as pd
-from pgportfolio.tools.data import panel_fillna
+from pgportfolio.tools.data import panel_fillna, serial_data_fillna
 from pgportfolio.constants import *
 import sqlite3
 from datetime import datetime
@@ -69,8 +70,7 @@ def get_global_panel(self, start, end, period=300, features=('close',)):
         self.__checkperiod(period)
 
         time_index = pd.to_datetime(list(range(start, end+1, period)),unit='s')
-        panel = pd.Panel(items=features, major_axis=coins, minor_axis=time_index, dtype=np.float32)
-
+        panelxr = xr.DataArray(data=np.NaN, coords=[features, coins, time_index])
         connection = sqlite3.connect(DATABASE_DIR)
         try:
             for row_number, coin in enumerate(coins):
@@ -114,12 +114,15 @@ def get_global_panel(self, start, end, period=300, features=('close',)):
                     serial_data = pd.read_sql_query(sql, con=connection,
                                                     parse_dates=["date_norm"],
                                                     index_col="date_norm")
-                    panel.loc[feature, coin, serial_data.index] = serial_data.squeeze()
-                    panel = panel_fillna(panel, "both")
+
+                    serial_data_squeezed = serial_data.squeeze()
+                    serial_data_squeezed = serial_data_squeezed.reindex(time_index)
+                    serial_data_squeezed = serial_data_fillna(serial_data_squeezed, "both")
+                    panelxr.loc[feature, coin, :] = serial_data_squeezed
         finally:
             connection.commit()
             connection.close()
-        return panel
+        return panelxr
 
     # select top coin_number of coins by volume from start to end
     def select_coins(self, start, end):
@@ -162,7 +165,7 @@ def __checkperiod(self, period):
         elif period == DAY:
             return
         else:
-            raise ValueError('peroid has to be 5min, 15min, 30min, 2hr, 4hr, or a day')
+            raise ValueError('period has to be 5min, 15min, 30min, 2hr, 4hr, or a day {}'.format(period))
 
     # add new history data into the database
     def update_data(self, start, end, coin):

pgportfolio/tools/data.py

@@ -120,9 +120,20 @@ def panel_fillna(panel, type="bfill"):
     frames = {}
     for item in panel.items:
         if type == "both":
-            frames[item] = panel.loc[item].fillna(axis=1, method="bfill").\
-                fillna(axis=1, method="ffill")
+            frames[item] = panel.loc[item].fillna(axis=1, method="bfill").fillna(axis=1, method="ffill")
         else:
             frames[item] = panel.loc[item].fillna(axis=1, method=type)
     return pd.Panel(frames)
 
+
+def serial_data_fillna(serial_data, type="bfill"):
+    """
+    fill nan
+    :param serial_data: the series to be filled
+    :param type: bfill or ffill
+    """
+    if type == "both":
+        ret_val = serial_data.fillna(method="bfill").fillna(method="ffill")
+    else:
+        ret_val = serial_data.fillna(method=type)
+    return ret_val