Skip to content

Commit 02715b9

Browse files
loitlyloitly
committed
- converted StatisticsProcessor to DbProcessor
- added request locking for requests resulting in the same returned data. - delegate decimated_info and decimate_key to ipac-base processor
1 parent 98e4fb3 commit 02715b9

38 files changed

+520
-329
lines changed

‎config/app.config

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,6 @@ alerts.dir = "/hydra/alerts"
1414

1515
debug.mode = true
1616

17-
application.db.url = "jdbc:h2:/hydra/server/tomcat/temp/application;CACHE_SIZE=262144"
18-
1917
/* keep vis.shared.mem.size small if pct is used */
2018
pct.vis.shared.mem.size = 0.4
2119
vis.shared.mem.size = "100M"

‎config/log4j.properties

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -48,10 +48,9 @@ log4j.logger.statistics=DEBUG, @app-name@_stats
4848
log4j.additivity.statistics= false
4949

5050
# set to DEBUG to log Spring's SQL statement and execution
51-
log4j.logger.org.springframework.jdbc.core.JdbcTemplate=DEBUG
51+
log4j.logger.org.springframework.jdbc.core.JdbcTemplate=DEBUG, @app-name@
52+
log4j.logger.org.springframework.jdbc=WARN, @app-name@
5253

53-
# Hibernate logging level
54-
#log4j.logger.org.hibernate=debug
5554

5655
# ehcache logging level
5756
#log4j.logger.net.sf.ehcache=debug

‎src/firefly/java/edu/caltech/ipac/firefly/core/SearchDescResolver.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ public String getDesc(Request req) {
4444
}
4545

4646
}
47-
return retval.toString();
47+
return retval.length() > 100 ? retval.substring(0, 100) : retval.toString();
4848
}
4949

5050
public String getDownloadDesc(Request req) {

‎src/firefly/java/edu/caltech/ipac/firefly/data/ServerParams.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -169,7 +169,7 @@ public class ServerParams {
169169

170170
public static final String PACKAGE_REQUEST = "packageRequest";
171171
public static final String TABLE_SEARCH = "tableSearch";
172-
public static final String TABLE_FIND_IDX = "tableFindIndex";
172+
public static final String QUERY_TABLE = "queryTable";
173173
public static final String SELECTED_VALUES = "selectedValues";
174174
public static final String TABLE_SAVE = "tableSave";
175175
public static final String UPLOAD = "upload";

‎src/firefly/java/edu/caltech/ipac/firefly/data/TableServerRequest.java

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -269,7 +269,7 @@ public SortedSet<Param> getSearchParams() {
269269
}
270270

271271
/**
272-
* returns the parameters used to modified this dataset. This includes filtering and sorting.
272+
* returns the parameters used to modified this dataset. This includes filtering, sorting, and decimate info.
273273
* @return
274274
*/
275275
@NotNull
@@ -281,6 +281,9 @@ public SortedSet<Param> getDataSetParam() {
281281
if (getSortInfo() != null) {
282282
params.add(new Param(SORT_INFO, getSortInfo().toString()));
283283
}
284+
if (getDecimateInfo() != null) {
285+
params.add(new Param(DECIMATE_INFO, getDecimateInfo().toString()));
286+
}
284287
return params;
285288
}
286289

‎src/firefly/java/edu/caltech/ipac/firefly/server/ServerCommandAccess.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,7 @@ private static void initCommand() {
9595
_cmdMap.put(ServerParams.CREATE_DOWNLOAD_SCRIPT, new SearchServerCommands.CreateDownloadScript());
9696
_cmdMap.put(ServerParams.PACKAGE_REQUEST, new SearchServerCommands.PackageRequest());
9797
_cmdMap.put(ServerParams.TABLE_SEARCH, new SearchServerCommands.TableSearch());
98-
_cmdMap.put(ServerParams.TABLE_FIND_IDX, new SearchServerCommands.TableFindIndex());
98+
_cmdMap.put(ServerParams.QUERY_TABLE, new SearchServerCommands.QueryTable());
9999
_cmdMap.put(ServerParams.SELECTED_VALUES, new SearchServerCommands.SelectedValues());
100100
_cmdMap.put(ServerParams.JSON_SEARCH, new SearchServerCommands.JsonSearch());
101101

‎src/firefly/java/edu/caltech/ipac/firefly/server/ServerContext.java

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
package edu.caltech.ipac.firefly.server;
55

66
import edu.caltech.ipac.firefly.server.cache.EhcacheProvider;
7+
import edu.caltech.ipac.firefly.server.filters.CommonFilter;
78
import edu.caltech.ipac.firefly.server.query.SearchProcessorFactory;
89
import edu.caltech.ipac.firefly.server.util.Logger;
910
import edu.caltech.ipac.firefly.server.visualize.VisContext;
@@ -87,6 +88,7 @@ public static void init(String contextPath, String contextName, String webappCon
8788
ServerContext.contextName = contextName;
8889

8990
configInit();
91+
9092
initVisSearchPath();
9193

9294
PERM_FILE_PATH_STR = getPermWorkDir().getPath();
@@ -211,7 +213,7 @@ static private void initVisSearchPath() {
211213
* @return
212214
*/
213215
public static File getConfigFile(String fname) {
214-
216+
215217
File f = null;
216218
if (appConfigDir != null) {
217219
f = new File(appConfigDir, fname) ;

‎src/firefly/java/edu/caltech/ipac/firefly/server/catquery/GatorDD.java

Lines changed: 31 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -13,22 +13,17 @@
1313
import edu.caltech.ipac.firefly.server.ServerContext;
1414
import edu.caltech.ipac.firefly.server.db.DbAdapter;
1515
import edu.caltech.ipac.firefly.server.db.DbInstance;
16-
import edu.caltech.ipac.firefly.server.db.TableDbUtil;
16+
import edu.caltech.ipac.firefly.server.db.EmbeddedDbUtil;
1717
import edu.caltech.ipac.firefly.server.db.spring.JdbcFactory;
18-
import edu.caltech.ipac.firefly.server.db.spring.mapper.DataGroupUtil;
1918
import edu.caltech.ipac.firefly.server.query.DataAccessException;
20-
import edu.caltech.ipac.firefly.server.query.DbProcessor;
19+
import edu.caltech.ipac.firefly.server.query.EmbeddedDbProcessor;
2120
import edu.caltech.ipac.firefly.server.query.ParamDoc;
22-
import edu.caltech.ipac.firefly.server.query.SearchManager;
2321
import edu.caltech.ipac.firefly.server.query.SearchProcessorImpl;
24-
import edu.caltech.ipac.firefly.server.util.StopWatch;
2522
import edu.caltech.ipac.firefly.server.util.ipactable.DataGroupPart;
2623
import edu.caltech.ipac.firefly.server.util.ipactable.TableDef;
2724
import edu.caltech.ipac.util.AppProperties;
2825
import edu.caltech.ipac.util.DataGroup;
2926
import edu.caltech.ipac.util.DataType;
30-
import edu.caltech.ipac.util.StringUtils;
31-
import org.apache.commons.codec.digest.DigestUtils;
3227

3328
import java.io.File;
3429
import java.io.IOException;
@@ -56,36 +51,44 @@
5651
@ParamDoc(name = CatalogRequest.SERVICE_ROOT, desc = "the part of the URL string that specifies the service and first params. " +
5752
"optional: almost never used")
5853
})
59-
public class GatorDD extends DbProcessor {
54+
public class GatorDD extends EmbeddedDbProcessor {
6055

6156
public FileInfo createDbFile(TableServerRequest treq) throws DataAccessException {
57+
6258
DbAdapter dbAdapter = DbAdapter.getAdapter(treq);
6359
File dbFile = new File(ServerContext.getPermWorkDir(), "GatorDD." + dbAdapter.getName());
64-
DataGroup dd = new DataGroup("DD for GatorDD", new DataType[]{
65-
new DataType("name", String.class),
66-
new DataType("description", String.class),
67-
new DataType("units", String.class),
68-
new DataType("indx", String.class),
69-
new DataType("dbtype", String.class),
70-
new DataType("tableflg", Integer.class),
71-
new DataType("sel", String.class)
72-
});
73-
TableDbUtil.createDDTbl(dbFile, dd, dbAdapter);
74-
TableDbUtil.setDbMetaInfo(treq, DbAdapter.getAdapter(treq), dbFile);
60+
try {
61+
if (dbFile.createNewFile()) {
62+
// created for the first time... populate dd and meta tables
63+
DataGroup dd = new DataGroup("DD for GatorDD", new DataType[]{
64+
new DataType("name", String.class),
65+
new DataType("description", String.class),
66+
new DataType("units", String.class),
67+
new DataType("indx", String.class),
68+
new DataType("dbtype", String.class),
69+
new DataType("tableflg", Integer.class),
70+
new DataType("sel", String.class)
71+
});
72+
EmbeddedDbUtil.createDDTbl(dbFile, dd, dbAdapter);
73+
EmbeddedDbUtil.setDbMetaInfo(treq, DbAdapter.getAdapter(treq), dbFile);
74+
}
75+
} catch (IOException e) {
76+
// should not happen.
77+
}
7578
return new FileInfo(dbFile);
7679
}
7780

7881
@Override
7982
protected DataGroupPart getDataset(TableServerRequest treq, File dbFile) throws DataAccessException {
8083
DbAdapter dbAdapter = DbAdapter.getAdapter(treq);
8184
DbInstance dbInstance = dbAdapter.getDbInstance(dbFile);
82-
String tblName = "ds_" + DigestUtils.md5Hex(StringUtils.toString(treq.getSearchParams(), "|"));
85+
String tblName = treq.getParam(CatalogRequest.CATALOG);
8386

8487
String tblExists = String.format("select count(*) from %s", tblName);
8588
try {
8689
JdbcFactory.getSimpleTemplate(dbInstance).queryForInt(tblExists);
8790
} catch (Exception e) {
88-
// does not exists.. fetch data and populate
91+
// DD for this catalog does not exists.. fetch data and populate
8992
fetchDataIntoTable(treq, tblName, dbFile, dbAdapter);
9093
}
9194

@@ -94,17 +97,22 @@ protected DataGroupPart getDataset(TableServerRequest treq, File dbFile) throws
9497
String sql = String.format("%s %s %s", dbAdapter.selectPart(treq), dbAdapter.fromPart(treq), dbAdapter.wherePart(treq));
9598
sql = dbAdapter.translateSql(sql);
9699

97-
DataGroup dg = TableDbUtil.runQuery(dbAdapter, dbFile, sql);
100+
DataGroup dg = EmbeddedDbUtil.runQuery(dbAdapter, dbFile, sql);
98101
TableDef tm = new TableDef();
99102
tm.setStatus(DataGroupPart.State.COMPLETED);
100103
return new DataGroupPart(tm, dg, treq.getStartIndex(), dg.size());
101104
}
102105

106+
@Override
107+
public boolean doLogging() {
108+
return false;
109+
}
110+
103111
private void fetchDataIntoTable(TableServerRequest treq, String tblName, File dbFile, DbAdapter dbAdapter) throws DataAccessException {
104112
TableServerRequest ntreq = (TableServerRequest) treq.cloneRequest();
105113
ntreq.keepBaseParamOnly();
106114
DataGroupPart dgp = new GatorDDImpl().getData(treq);
107-
TableDbUtil.createDataTbl(dbFile, dgp.getData(), dbAdapter, tblName);
115+
EmbeddedDbUtil.createDataTbl(dbFile, dgp.getData(), dbAdapter, tblName);
108116
}
109117
}
110118

‎src/firefly/java/edu/caltech/ipac/firefly/server/catquery/GatorQuery.java

Lines changed: 21 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -302,36 +302,32 @@ protected void setColumnTips(TableMeta meta, ServerRequest request) {
302302
if (param.getName().equals(CatalogRequest.DBMS)) req.setParam(param);
303303
}
304304

305-
SearchManager sm = new SearchManager();
306-
DataGroupPart dgp = new DataGroupPart();
307-
308305
try {
309-
dgp = sm.getDataGroup(req);
310-
} catch (Exception e) {
311-
}
312-
313-
DataGroup dg = dgp.getData();
314-
if (dg != null) {
315-
for (int i = 0; i < dg.size(); i++) {
316-
DataObject dObj = dg.get(i);
317-
String tipStr = "";
318-
319-
String descStr = (String) dObj.getDataElement("description");
320-
if (!StringUtils.isEmpty(descStr) && !descStr.equalsIgnoreCase("null")) {
321-
tipStr += descStr;
322-
}
306+
DataGroupPart dgp = new SearchManager().getDataGroup(req);
307+
DataGroup dg = dgp.getData();
308+
if (dg != null) {
309+
for (int i = 0; i < dg.size(); i++) {
310+
DataObject dObj = dg.get(i);
311+
String tipStr = "";
312+
313+
String descStr = (String) dObj.getDataElement("DESCRIPTION");
314+
if (!StringUtils.isEmpty(descStr) && !descStr.equalsIgnoreCase("null")) {
315+
tipStr += descStr;
316+
}
323317

324-
String unitStr = (String) dObj.getDataElement("units");
325-
if (!StringUtils.isEmpty(unitStr) && !unitStr.equalsIgnoreCase("null")) {
326-
if (tipStr.length() > 0) {
327-
tipStr += " ";
318+
String unitStr = (String) dObj.getDataElement("UNITS");
319+
if (!StringUtils.isEmpty(unitStr) && !unitStr.equalsIgnoreCase("null")) {
320+
if (tipStr.length() > 0) {
321+
tipStr += " ";
322+
}
323+
tipStr += "(" + unitStr + ")";
328324
}
329-
tipStr += "(" + unitStr + ")";
330-
}
331325

332-
String nameStr = (String) dObj.getDataElement("name");
333-
meta.setAttribute(makeAttribKey(DESC_TAG, nameStr.toLowerCase()), tipStr);
326+
String nameStr = (String) dObj.getDataElement("NAME");
327+
meta.setAttribute(makeAttribKey(DESC_TAG, nameStr.toLowerCase()), tipStr);
328+
}
334329
}
330+
} catch (Exception e) {
335331
}
336332
}
337333

‎src/firefly/java/edu/caltech/ipac/firefly/server/db/BaseDbAdapter.java

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ public String createDataSql(DataType[] dtTypes, String tblName) {
6565
tblName = StringUtils.isEmpty(tblName) ? "data" : tblName;
6666
List<String> coldefs = new ArrayList<>();
6767
for(DataType dt : dtTypes) {
68-
coldefs.add( dt.getKeyName() + " " + getDataType(dt.getDataType()));
68+
coldefs.add( String.format("\"%s\" %s", dt.getKeyName().toUpperCase(),getDataType(dt.getDataType())));
6969
}
7070

7171
return String.format("create table if not exists %s (%s)", tblName, StringUtils.toString(coldefs, ","));
@@ -94,7 +94,7 @@ public String selectPart(TableServerRequest treq) {
9494
}
9595

9696
public String fromPart(TableServerRequest treq) {
97-
String from = TableDbUtil.getDatasetID(treq);
97+
String from = EmbeddedDbUtil.getDatasetID(treq);
9898
from = StringUtils.isEmpty(from) ? treq.getParam(TableServerRequest.SQL_FROM) : from;
9999
from = "from " + (StringUtils.isEmpty(from) ? "data" : from);
100100
return from;
@@ -163,4 +163,22 @@ public String getDataType(Class type) {
163163
return "varchar(1023)";
164164
}
165165
}
166+
167+
168+
static class EmbeddedDbInstance extends DbInstance {
169+
170+
public EmbeddedDbInstance(String name, String dbUrl, String driver) {
171+
super(false, null, dbUrl, null, null, driver, name);
172+
}
173+
174+
@Override
175+
public boolean equals(Object obj) {
176+
return StringUtils.areEqual(this.dbUrl,((EmbeddedDbInstance)obj).dbUrl);
177+
}
178+
179+
@Override
180+
public int hashCode() {
181+
return this.dbUrl.hashCode();
182+
}
183+
}
166184
}

‎src/firefly/java/edu/caltech/ipac/firefly/server/db/DbAdapter.java

Lines changed: 2 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -25,14 +25,6 @@ public interface DbAdapter {
2525
*/
2626
String getName();
2727

28-
/**
29-
* Some database may store the data in a different file than from what
30-
* treqed in the connection string.
31-
* @param dbFile
32-
* @return the storage file given the file in the connection string
33-
*/
34-
File getStorageFile(File dbFile);
35-
3628
/**
3729
* @param dbFile
3830
* @return a new DbInstance for the given dbFile
@@ -75,7 +67,7 @@ public interface DbAdapter {
7567
//
7668
//====================================================================
7769

78-
static String DEF_DB_TYPE = AppProperties.getProperty("DbAdapter.type", SQLITE);
70+
static String DEF_DB_TYPE = AppProperties.getProperty("DbAdapter.type", HSQL);
7971

8072
static DbAdapter getAdapter(TableServerRequest treq) {
8173
return getAdapter(treq.getMeta(TBL_FILE_TYPE));
@@ -91,7 +83,7 @@ static DbAdapter getAdapter(String type) {
9183
case HSQL:
9284
return new HsqlDbAdapter();
9385
default:
94-
return new SqliteDbAdapter(); // when an unrecognized type is given.
86+
return new HsqlDbAdapter(); // when an unrecognized type is given.
9587
}
9688
}
9789

0 commit comments

Comments
 (0)