IRSA-975: NED searching seems not to work

- column names must be enclose by double-quotes
- remove caching done at the search processor’s level
- introduce SharedDbProcessor: results from same processor share the same database
- convert IBE’s template into a search processor
- convert XYGenericProcessor into an EmbeddedDbProcessor
- use firefly’s config file where applicable.
- limit log message to 2k characters

Author: loitly

config/log4j.properties

@@ -1,7 +1,7 @@
 log4j.rootLogger=INFO, console
 log4j.appender.console=org.apache.log4j.ConsoleAppender
 log4j.appender.console.layout=org.apache.log4j.PatternLayout
-log4j.appender.console.layout.ConversionPattern=%d{MM/dd HH:mm:ss} %5p - %m [%t] (%c{2}#%M:%L)%n
+log4j.appender.console.layout.ConversionPattern=%d{MM/dd HH:mm:ss} %5p - %.2000m [%t] (%c{2}#%M:%L)%n
 
 # ${host.name} is defined as an evironment variable at tomcat startup... not a log4j variable.
 log4j.appender.@app-name@_stats=org.apache.log4j.DailyRollingFileAppender
@@ -14,15 +14,15 @@ log4j.appender.@app-name@_brief=org.apache.log4j.DailyRollingFileAppender
 log4j.appender.@app-name@_brief.File=${catalina.base}/logs/@[email protected]
 log4j.appender.@app-name@_brief.DatePattern='.'yyyy-ww
 log4j.appender.@app-name@_brief.layout=org.apache.log4j.PatternLayout
-log4j.appender.@app-name@_brief.layout.ConversionPattern=%d{MM/dd HH:mm:ss} %5p - %m [%t] (%c{2}#%M:%L)%n
+log4j.appender.@app-name@_brief.layout.ConversionPattern=%d{MM/dd HH:mm:ss} %5p - %.2000m [%t] (%c{2}#%M:%L)%n
 
 log4j.appender.@app-name@=org.apache.log4j.DailyRollingFileAppender
 log4j.appender.@[email protected]=${catalina.base}/logs/@[email protected]
 log4j.appender.@[email protected]='.'yyyy-ww
 log4j.appender.@[email protected]=org.apache.log4j.PatternLayout
 log4j.appender.@[email protected]=\
 %d{MM/dd HH:mm:ss} %5p   Thread: %t  %c{2}#%M:%L\
-%n        %m%n
+%n        %.2000m%n
 
 
 # this renderer print the object into multiple lines

src/firefly/html/demo/ffapi-highlevel-charttest.html

@@ -184,7 +184,7 @@ <h3>Default chart for 'allwise' table group below</h3>
             firefly.showChart('defaultChart', {tbl_group: 'allwise'});
 
             // using column expressions
-            var tblReqXpr =  Object.assign({}, tblReq, {inclCols: 'ra + dec as radec, ra, dec, ln(dec) as LnOfDec, power(ra, 2) as ra_sq'});
+            var tblReqXpr =  Object.assign({}, tblReq, {inclCols: '"ra" + "dec" as "radec", "ra", "dec", ln("dec") as "LnOfDec", power("ra", 2) as "ra_sq"'});
             tblReqXpr.META_INFO.tbl_id = 'tblwithXpr';
             tblReqXpr.META_INFO.title = 'table with expressions';
             tblReqXpr.tbl_id = 'tblwithXpr';

src/firefly/java/edu/caltech/ipac/firefly/server/catquery/CatMasterTableQuery.java

@@ -25,9 +25,6 @@
 import edu.caltech.ipac.util.DataObject;
 import edu.caltech.ipac.util.DataType;
 import edu.caltech.ipac.util.StringUtils;
-import edu.caltech.ipac.util.cache.Cache;
-import edu.caltech.ipac.util.cache.CacheManager;
-import edu.caltech.ipac.util.cache.StringKey;
 
 import java.io.File;
 import java.io.IOException;
@@ -257,17 +254,7 @@ public static DataGroup getBaseGatorData(String originalFilename) throws IOExcep
     }
 
     protected File loadDataFile(TableServerRequest request) throws IOException, DataAccessException {
-        File retFile;
-
-        StringKey key = new StringKey(CatMasterTableQuery.class.getName(), getUniqueID(request));
-        Cache cache = CacheManager.getCache(Cache.TYPE_TEMP_FILE);
-        retFile = (File) cache.get(key);
-        if (retFile == null) {
-            retFile = getMasterCatalogFile(request);
-            cache.put(key, retFile);
-        }
-
-        return retFile;
+        return getMasterCatalogFile(request);
     }
 
 }

src/firefly/java/edu/caltech/ipac/firefly/server/catquery/GatorDD.java

@@ -19,7 +19,9 @@
 import edu.caltech.ipac.firefly.server.query.EmbeddedDbProcessor;
 import edu.caltech.ipac.firefly.server.query.ParamDoc;
 import edu.caltech.ipac.firefly.server.query.SearchProcessorImpl;
+import edu.caltech.ipac.firefly.server.query.SharedDbProcessor;
 import edu.caltech.ipac.firefly.server.util.ipactable.DataGroupPart;
+import edu.caltech.ipac.firefly.server.util.ipactable.DataGroupReader;
 import edu.caltech.ipac.firefly.server.util.ipactable.TableDef;
 import edu.caltech.ipac.util.AppProperties;
 import edu.caltech.ipac.util.DataGroup;
@@ -52,69 +54,23 @@
                 @ParamDoc(name = CatalogRequest.SERVICE_ROOT, desc = "the part of the URL string that specifies the service and first params. " +
                         "optional: almost never used")
         })
-public class GatorDD extends EmbeddedDbProcessor {
+public class GatorDD extends SharedDbProcessor {
 
-    public FileInfo createDbFile(TableServerRequest treq) throws DataAccessException {
-
-        DbAdapter dbAdapter = DbAdapter.getAdapter(treq);
-        File dbFile = new File(ServerContext.getTempWorkDir(), String.format("GatorDD-%s.%s", FileUtil.getHostname(), dbAdapter.getName()));
+    public DataGroup fetchData(TableServerRequest treq) throws DataAccessException {
+        TableServerRequest ntreq = (TableServerRequest) treq.cloneRequest();
+        ntreq.keepBaseParamOnly();
         try {
-            if (dbFile.createNewFile()) {
-                // created for the first time... populate dd and meta tables
-                DataGroup dd = new DataGroup("DD for GatorDD", new DataType[]{
-                        new DataType("name", String.class),
-                        new DataType("description", String.class),
-                        new DataType("units", String.class),
-                        new DataType("indx", String.class),
-                        new DataType("dbtype", String.class),
-                        new DataType("tableflg", Integer.class),
-                        new DataType("sel", String.class)
-                });
-                EmbeddedDbUtil.createDDTbl(dbFile, dd, dbAdapter, "data");
-                EmbeddedDbUtil.setDbMetaInfo(treq, DbAdapter.getAdapter(treq), dbFile);
-            }
+            File results = new GatorDDImpl().loadDataFile(treq);
+            return DataGroupReader.read(results);
         } catch (IOException e) {
-            // should not happen.
+            throw new DataAccessException(e.getMessage(), e);
         }
-        return new FileInfo(dbFile);
-    }
-
-    @Override
-    protected DataGroupPart getResultSet(TableServerRequest treq, File dbFile) throws DataAccessException {
-        DbAdapter dbAdapter = DbAdapter.getAdapter(treq);
-        DbInstance dbInstance =  dbAdapter.getDbInstance(dbFile);
-        String tblName = treq.getParam(CatalogRequest.CATALOG);
-
-        String tblExists = String.format("select count(*) from %s", tblName);
-        try {
-            JdbcFactory.getSimpleTemplate(dbInstance).queryForInt(tblExists);
-        } catch (Exception e) {
-            // DD for this catalog does not exists.. fetch data and populate
-            fetchDataIntoTable(treq, tblName, dbFile, dbAdapter);
-        }
-
-        treq.setParam(TableServerRequest.SQL_FROM, tblName);
-        treq.setPageSize(Integer.MAX_VALUE);
-        String sql = String.format("%s %s %s", dbAdapter.selectPart(treq), dbAdapter.fromPart(treq), dbAdapter.wherePart(treq));
-        sql = dbAdapter.translateSql(sql);
-
-        DataGroup dg = EmbeddedDbUtil.runQuery(dbAdapter, dbFile, sql, "data");
-        TableDef tm = new TableDef();
-        tm.setStatus(DataGroupPart.State.COMPLETED);
-        return new DataGroupPart(tm, dg, treq.getStartIndex(), dg.size());
     }
 
     @Override
     public boolean doLogging() {
         return false;
     }
-
-    private void fetchDataIntoTable(TableServerRequest treq, String tblName, File dbFile, DbAdapter dbAdapter) throws DataAccessException {
-        TableServerRequest ntreq = (TableServerRequest) treq.cloneRequest();
-        ntreq.keepBaseParamOnly();
-        DataGroupPart dgp = new GatorDDImpl().getData(treq);
-        EmbeddedDbUtil.createDataTbl(dbFile, dgp.getData(), dbAdapter, tblName);
-    }
 }
 
 

src/firefly/java/edu/caltech/ipac/firefly/server/db/BaseDbAdapter.java

@@ -28,7 +28,7 @@
  * @version $Id: DbInstance.java,v 1.3 2012/03/15 20:35:40 loi Exp $
  */
 abstract public class BaseDbAdapter implements DbAdapter {
-    private static long MAX_IDLE_TIME = 1000 * 60 * 5;      // will be cleaned up if idle more than 5 minutes.
+    private static long MAX_IDLE_TIME = 1000 * 60 * 15;      // will be purged up if idle more than 5 minutes.
     private static Map<String, EmbeddedDbInstance> dbInstances = new HashMap<>();
     private static Logger.LoggerImpl LOGGER = Logger.getLogger();
 
@@ -75,7 +75,7 @@ public String createDataSql(DataType[] dtTypes, String tblName) {
         tblName = StringUtils.isEmpty(tblName) ? "data" : tblName;
         List<String> coldefs = new ArrayList<>();
         for(DataType dt : dtTypes) {
-            coldefs.add( String.format("\"%s\" %s", dt.getKeyName().toUpperCase(),getDataType(dt.getDataType())));
+            coldefs.add( String.format("\"%s\" %s", dt.getKeyName(), getDataType(dt.getDataType())));       // add quotes to avoid reserved words clashes
         }
 
         return String.format("create table %s (%s)", tblName, StringUtils.toString(coldefs, ","));
@@ -103,13 +103,6 @@ public String selectPart(TableServerRequest treq) {
         return cols;
     }
 
-    public String fromPart(TableServerRequest treq) {
-        String from = treq.getParam(TableServerRequest.SQL_FROM);
-        from = from == null ? EmbeddedDbUtil.getResultSetID(treq) : from;
-        from = "from " + (StringUtils.isEmpty(from) ? "data" : from);
-        return from;
-    }
-
     public String wherePart(TableServerRequest treq) {
         String where = "";
         if (treq.getFilters() != null && treq.getFilters().size() > 0) {
@@ -171,12 +164,16 @@ public String getDataType(Class type) {
     }
 
     public DbInstance getDbInstance(File dbFile) {
+        return getDbInstance(dbFile, true);
+    }
+
+    public DbInstance getDbInstance(File dbFile, boolean create) {
         EmbeddedDbInstance ins = dbInstances.get(dbFile.getPath());
-        if (ins == null) {
+        if (ins == null && create) {
             ins = createDbInstance(dbFile);
             dbInstances.put(dbFile.getPath(), ins);
         }
-        ins.touch();
+        if (ins != null ) ins.touch();
         return ins;
 
     }

src/firefly/java/edu/caltech/ipac/firefly/server/db/DbAdapter.java

@@ -60,7 +60,6 @@ public interface DbAdapter {
     String getMetaSql(String forTable);
 
     String selectPart(TableServerRequest treq);
-    String fromPart(TableServerRequest treq);
     String wherePart(TableServerRequest treq);
     String orderByPart(TableServerRequest treq) ;
     String pagingPart(TableServerRequest treq) ;