DM-11814: Use embedded DB in table data support

Author: loitly

buildScript/depends.gincl

@@ -29,6 +29,7 @@ ext {
               ':servlet-api',
               ':websocket-api']
 
+  db_lib = [':h2-1.4.194', ':sqlite-jdbc-3.20.0', ':hsqldb']
   mysql_lib = ':mysql-connector-java-5.1.6-bin'
   starlink_lib = [':rayreg', ':regclient', ':stil']
   xbeanfinder_lib = [':xbean-finder-4.1', ':asm-5.0.3']
@@ -90,6 +91,7 @@ repositories {
           "$libPath/oidc",
           "$libPath/json",
           "$libPath/log4j",
+          "$libPath/db",
           "$libPath/mysql",
           "$libPath/sdss",
           "$libPath/spring",

config/app.config

@@ -12,7 +12,9 @@ work.directory = "/hydra/workarea"
 stats.log.dir = '${catalina.base}/logs'
 alerts.dir = "/hydra/alerts"
 
-debug.mode = false
+debug.mode = true
+
+application.db.url = "jdbc:h2:/hydra/server/tomcat/temp/application;CACHE_SIZE=262144"
 
 /*  keep vis.shared.mem.size small if pct is used */
 pct.vis.shared.mem.size = 0.4

config/common.prop

@@ -11,6 +11,7 @@ BuildDate=@build_time@
 # app.prop env sensitive properties
 #==========================================================
 
+DbAdapter.type = hsql
 
 # a directory for storing intermediate files
 # default to System.getProperty("java.io.tmpdir") + "/workarea"

config/web.xml

@@ -90,6 +90,15 @@
     </servlet-mapping>
 
 
+    <servlet>
+        <servlet-name>H2Console</servlet-name>
+        <servlet-class>org.h2.server.web.WebServlet</servlet-class>
+        <load-on-startup>1</load-on-startup>
+    </servlet>
+    <servlet-mapping>
+        <servlet-name>H2Console</servlet-name>
+        <url-pattern>/admin/db/*</url-pattern>
+    </servlet-mapping>
 <!--    Resources                         -->
 
     <resource-ref>

jars/db/h2-1.4.194.jar

@@ -5,7 +5,7 @@ apply from: "$fireflyPath/buildScript/gwt_webapp.gincl"
 
 dependencies {
   compile project(':xstream'), project(':simbad'), project(':firefly_data')
-  webappLib mysql_lib, FIREFLY_RUNTIME + ":$jar.baseName"
+  webappLib db_lib, mysql_lib, FIREFLY_RUNTIME + ":$jar.baseName"
 }
 
 sourceSets {

jars/db/hsqldb.jar

@@ -169,6 +169,13 @@ <h2>
                     {tbl_id: 'test-tbl'});
             firefly.showTable('table-1', tblReq);
 
+            // using column expressions
+            var tblReqXpr =  Object.assign({}, tblReq, {inclCols: 'ra + dec as "radec", ra, dec, ln(dec) as LnOfDec, power(ra, 2) as ra_sq'});
+            tblReqXpr.META_INFO.tbl_id = 'tblwithXpr';
+            tblReqXpr.META_INFO.title = 'table with expressions';
+            tblReqXpr.tbl_id = 'tblwithXpr';
+            firefly.showTable('table-1', tblReqXpr);
+
             trace1 = {
                 tbl_id: 'test-tbl',
                 x: 'tables::ra',

jars/db/sqlite-jdbc-3.20.0.jar

@@ -13,6 +13,7 @@
         window.firefly = {
             app: {
                 template: 'FireflyViewer',
+                views: 'tables',
                 options : {
                     MenuItemKeys: {maskOverlay:true},
                 }

src/firefly/build.gradle

@@ -73,7 +73,7 @@ public static void save(File file, DataGroup dataGroup)
         PrintWriter out = null;
         try {
             out = new PrintWriter(new BufferedWriter(new FileWriter(file)));
-            save(out, dataGroup);
+            save(out, dataGroup, false);
         } finally {
             if (out != null) out.close();
         }
@@ -88,15 +88,27 @@ public static void save(File file, DataGroup dataGroup)
      */
     public static void save(OutputStream stream, DataGroup dataGroup)
             throws IOException {
-        save(new PrintWriter(stream), dataGroup);
+        save(new PrintWriter(stream), dataGroup, false);
     }
 
-    private static void save(PrintWriter out, DataGroup dataGroup)
-        throws IOException {
+    /**
+     * save the catalogs to a stream, stream is not closed
+     *
+     * @param stream the output stream to write to
+     * @param dataGroup data group
+     * @param ignoreSysMeta ignore meta use by system.
+     * @throws IOException on error
+     */
+    public static void save(OutputStream stream, DataGroup dataGroup, boolean ignoreSysMeta)
+            throws IOException {
+        save(new PrintWriter(stream), dataGroup, ignoreSysMeta);
+    }
+
+    private static void save(PrintWriter out, DataGroup dataGroup, boolean ignoreSysMeta) throws IOException {
         List<DataType> headers = Arrays.asList(dataGroup.getDataDefinitions());
         int totalRow = dataGroup.size();
 
-        IpacTableUtil.writeAttributes(out, dataGroup.getKeywords());
+        IpacTableUtil.writeAttributes(out, dataGroup.getKeywords(), ignoreSysMeta);
         IpacTableUtil.writeHeader(out, headers);
 
         for (int i = 0; i < totalRow; i++) {

src/firefly/html/demo/ffapi-highlevel-charttest.html

@@ -5,17 +5,25 @@
 
 import edu.caltech.ipac.util.StringUtils;
 
+import javax.validation.constraints.NotNull;
 import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.SortedSet;
+import java.util.TreeSet;
 
 public class TableServerRequest extends ServerRequest implements Serializable, DataEntry, Cloneable {
 
+    public static final String TBL_FILE_PATH = "tblFilePath";       // this meta if exists contains source of the data
+    public static final String TBL_FILE_TYPE = "tblFileType";       // this meta if exists contains storage type, ipac, h2, sqlite, etc
+    public static final String DATASET_ID = "datasetID";        // this meta if exists contains the ID of the dataset returned.
+
     public static final String DECIMATE_INFO = "decimate";
-    public static final String TBL_FILE_PATH = "tblFilePath";
+    public static final String SQL_FROM = "sqlFrom";
+
     public static final String TBL_ID = "tbl_id";
     public static final String TITLE = "title";
     public static final String FILTERS = "filters";
@@ -25,8 +33,7 @@ public class TableServerRequest extends ServerRequest implements Serializable, D
     public static final String INCL_COLUMNS = "inclCols";
     public static final String FIXED_LENGTH = "fixedLength";
     public static final String META_INFO = "META_INFO";
-    public static final String SYS_PARAMS = "|" + StringUtils.toString(new String[]{FILTERS,SORT_INFO,PAGE_SIZE,START_IDX,INCL_COLUMNS,FIXED_LENGTH,META_INFO,TBL_ID,DECIMATE_INFO}, "|") + "|";
-
+    public static final List<String> SYS_PARAMS = Arrays.asList(new String[]{INCL_COLUMNS,FIXED_LENGTH,META_INFO,TBL_ID,DECIMATE_INFO,SQL_FROM});
     public static final String TBL_INDEX = "tbl_index";     // the table to show if it's a multi-table file.
 
     private int pageSize;
@@ -51,6 +58,9 @@ public Map<String, String> getMeta() {
     }
 
     public String getMeta(String key) {
+        if (metaInfo == null) {
+
+        }
         return metaInfo == null ? null : metaInfo.get(key);
     }
 
@@ -75,7 +85,7 @@ public String getTblTitle() {
 
     @Override
     public boolean isInputParam(String paramName) {
-        return !SYS_PARAMS.contains("|" + paramName + "|");
+        return !SYS_PARAMS.contains(paramName);
     }
 
     public int getStartIndex() {
@@ -145,12 +155,23 @@ public void setFilters(List<String> filters) {
         }
     }
 
+    /**
+     * use this when you want to get full data in its
+     * natural order.
+     */
+    public void keepBaseParamOnly() {
+        setFilters(null);
+        setPageSize(Integer.MAX_VALUE);
+        setSortInfo(null);
+        removeParam(INCL_COLUMNS);
+    }
+
     @Override
     public ServerRequest newInstance() {
         return new TableServerRequest();
     }
 
-    //====================================================================
+//====================================================================
 
 //====================================================================
 
@@ -230,6 +251,39 @@ public static String toFilterStr(List<String> l) {
         return StringUtils.toString(l,";");
     }
 
+    /**
+     * returns only the parameters used for searching. This excludes all system parameter, including
+     * filtering, sorting, and paging.
+     * @return
+     */
+    @NotNull
+    public SortedSet<Param> getSearchParams() {
+        TreeSet<Param> params = new TreeSet<>();
+        for (Param p : getParams()) {
+            if (!SYS_PARAMS.contains(p.getName()) ||
+                    p.getName().equals(SORT_INFO)) {
+                params.add(p);
+            }
+        }
+        return params;
+    }
+
+    /**
+     * returns the parameters used to modified this dataset.  This includes filtering and sorting.
+     * @return
+     */
+    @NotNull
+    public SortedSet<Param> getDataSetParam() {
+        TreeSet<Param> params = new TreeSet<>();
+        if (filters != null && filters.size() > 0) {
+            params.add(new Param(FILTERS, toFilterStr(filters)));
+        }
+        if (getSortInfo() != null) {
+            params.add(new Param(SORT_INFO, getSortInfo().toString()));
+        }
+        return params;
+    }
+
 //====================================================================
 //
 //====================================================================
@@ -238,9 +292,6 @@ public static String toFilterStr(List<String> l) {
     protected boolean addPredefinedAttrib(Param param) {
         if (param == null || param.getName() == null || param.getValue() == null) return false;
 
-//        if (param.getName().equals(SORT_INFO)) {
-//            sortInfo = SortInfo.parse(param.getValue());
-//    }
         switch (param.getName()) {
             case PAGE_SIZE:
                 pageSize = StringUtils.getInt(param.getValue());

src/firefly/html/firefly-dev.html

@@ -26,7 +26,7 @@ public interface TableDataView extends HasAccessInfos, Serializable {
     public static final String ROW_DESELECT_ALL = "rowDeselectAll";
     public static final String MODEL_LOADED = "modelLoaded";
 
-    public static final String ROWID = "ROWID";
+    public static final String ROWID = "ROW_IDX";
     public enum Align {LEFT, RIGHT, CENTER}
 
     DatasetInfoConverter getDatasetInfoConverter();

src/firefly/java/edu/caltech/ipac/astro/IpacTableWriter.java

@@ -4,7 +4,6 @@
 package edu.caltech.ipac.firefly.server;
 
 import edu.caltech.ipac.firefly.server.cache.EhcacheProvider;
-import edu.caltech.ipac.firefly.server.filters.CommonFilter;
 import edu.caltech.ipac.firefly.server.query.SearchProcessorFactory;
 import edu.caltech.ipac.firefly.server.util.Logger;
 import edu.caltech.ipac.firefly.server.visualize.VisContext;
@@ -88,7 +87,6 @@ public static void init(String contextPath, String contextName, String webappCon
             ServerContext.contextName = contextName;
 
             configInit();
-
             initVisSearchPath();
 
             PERM_FILE_PATH_STR = getPermWorkDir().getPath();

src/firefly/java/edu/caltech/ipac/firefly/data/TableServerRequest.java

@@ -129,12 +129,6 @@ public ServerRequest inspectRequest(ServerRequest request) {
         return req;
     }
 
-    @Override
-    protected String getWspaceSaveDirectory() {
-        return "/" + WorkspaceManager.SEARCH_DIR + "/" + WspaceMeta.CATALOGS;
-
-    }
-
     protected File loadDynDataFile(TableServerRequest request) throws IOException, DataAccessException {
         CatalogRequest req = QueryUtil.assureType(CatalogRequest.class, request);
         return searchGator(req);

src/firefly/java/edu/caltech/ipac/firefly/data/table/TableDataView.java

@@ -6,14 +6,29 @@
 import edu.caltech.ipac.astro.DataGroupQueryStatement;
 import edu.caltech.ipac.firefly.core.EndUserException;
 import edu.caltech.ipac.firefly.data.CatalogRequest;
+import edu.caltech.ipac.firefly.data.FileInfo;
 import edu.caltech.ipac.firefly.data.ServerRequest;
 import edu.caltech.ipac.firefly.data.TableServerRequest;
 import edu.caltech.ipac.firefly.data.table.TableMeta;
 import edu.caltech.ipac.firefly.server.ServerContext;
+import edu.caltech.ipac.firefly.server.db.DbAdapter;
+import edu.caltech.ipac.firefly.server.db.DbInstance;
+import edu.caltech.ipac.firefly.server.db.TableDbUtil;
+import edu.caltech.ipac.firefly.server.db.spring.JdbcFactory;
+import edu.caltech.ipac.firefly.server.db.spring.mapper.DataGroupUtil;
+import edu.caltech.ipac.firefly.server.query.DataAccessException;
+import edu.caltech.ipac.firefly.server.query.DbProcessor;
 import edu.caltech.ipac.firefly.server.query.ParamDoc;
+import edu.caltech.ipac.firefly.server.query.SearchManager;
 import edu.caltech.ipac.firefly.server.query.SearchProcessorImpl;
+import edu.caltech.ipac.firefly.server.util.StopWatch;
+import edu.caltech.ipac.firefly.server.util.ipactable.DataGroupPart;
+import edu.caltech.ipac.firefly.server.util.ipactable.TableDef;
 import edu.caltech.ipac.util.AppProperties;
+import edu.caltech.ipac.util.DataGroup;
 import edu.caltech.ipac.util.DataType;
+import edu.caltech.ipac.util.StringUtils;
+import org.apache.commons.codec.digest.DigestUtils;
 
 import java.io.File;
 import java.io.IOException;
@@ -26,7 +41,9 @@
 
 
 /**
- * @author Trey Roby
+ * This is a wrapper around GatorDDImpl.  It uses the original code to retrieve Gator's DD
+ * information, but stores it into a database.  Unlike our single-results per database,
+ * this database will contains all of the DD information retrieved by GatorDDImpl.
  */
 @SearchProcessorImpl(id = "GatorDD", params =
         {@ParamDoc(name = CatalogRequest.CATALOG, desc = "the catalog name to search"),
@@ -39,7 +56,63 @@
                 @ParamDoc(name = CatalogRequest.SERVICE_ROOT, desc = "the part of the URL string that specifies the service and first params. " +
                         "optional: almost never used")
         })
-public class GatorDD extends BaseGator {
+public class GatorDD extends DbProcessor {
+
+    public FileInfo createDbFile(TableServerRequest treq) throws DataAccessException {
+        DbAdapter dbAdapter = DbAdapter.getAdapter(treq);
+        File dbFile = new File(ServerContext.getPermWorkDir(), "GatorDD." + dbAdapter.getName());
+        DataGroup dd = new DataGroup("DD for GatorDD", new DataType[]{
+                new DataType("name", String.class),
+                new DataType("description", String.class),
+                new DataType("units", String.class),
+                new DataType("indx", String.class),
+                new DataType("dbtype", String.class),
+                new DataType("tableflg", Integer.class),
+                new DataType("sel", String.class)
+        });
+        TableDbUtil.createDDTbl(dbFile, dd, dbAdapter);
+        TableDbUtil.setDbMetaInfo(treq, DbAdapter.getAdapter(treq), dbFile);
+        return new FileInfo(dbFile);
+    }
+
+    @Override
+    protected DataGroupPart getDataset(TableServerRequest treq, File dbFile) throws DataAccessException {
+        DbAdapter dbAdapter = DbAdapter.getAdapter(treq);
+        DbInstance dbInstance =  dbAdapter.getDbInstance(dbFile);
+        String tblName = "ds_" + DigestUtils.md5Hex(StringUtils.toString(treq.getSearchParams(), "|"));
+
+        String tblExists = String.format("select count(*) from %s", tblName);
+        try {
+            JdbcFactory.getSimpleTemplate(dbInstance).queryForInt(tblExists);
+        } catch (Exception e) {
+            // does not exists.. fetch data and populate
+            fetchDataIntoTable(treq, tblName, dbFile, dbAdapter);
+        }
+
+        treq.setParam(TableServerRequest.SQL_FROM, tblName);
+        treq.setPageSize(Integer.MAX_VALUE);
+        String sql = String.format("%s %s %s", dbAdapter.selectPart(treq), dbAdapter.fromPart(treq), dbAdapter.wherePart(treq));
+        sql = dbAdapter.translateSql(sql);
+
+        DataGroup dg = TableDbUtil.runQuery(dbAdapter, dbFile, sql);
+        TableDef tm = new TableDef();
+        tm.setStatus(DataGroupPart.State.COMPLETED);
+        return new DataGroupPart(tm, dg, treq.getStartIndex(), dg.size());
+    }
+
+    private void fetchDataIntoTable(TableServerRequest treq, String tblName, File dbFile, DbAdapter dbAdapter) throws DataAccessException {
+        TableServerRequest ntreq = (TableServerRequest) treq.cloneRequest();
+        ntreq.keepBaseParamOnly();
+        DataGroupPart dgp = new GatorDDImpl().getData(treq);
+        TableDbUtil.createDataTbl(dbFile, dgp.getData(), dbAdapter, tblName);
+    }
+}
+
+
+/**
+ * @author Trey Roby
+ */
+class GatorDDImpl extends BaseGator {
 
     private static final String DEF_DD_SERVICE = AppProperties.getProperty("irsa.gator.service.dd",
             "/cgi-bin/Gator/nph-dd");