Skip to content

Commit c7b96b8

Browse files
loitlyloitly
committed
- introduce session caching
- combined related tables into one database - generate dd and meta for all tables - combined statistics into one sql query. - fix StatisticsProcessorTest fail
1 parent 02715b9 commit c7b96b8

28 files changed

+434
-210
lines changed

‎config/app.config

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ work.directory = "/hydra/workarea"
1212
stats.log.dir = '${catalina.base}/logs'
1313
alerts.dir = "/hydra/alerts"
1414

15-
debug.mode = true
15+
debug.mode = false
1616

1717
/* keep vis.shared.mem.size small if pct is used */
1818
pct.vis.shared.mem.size = 0.4

‎config/log4j.properties

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,9 +48,11 @@ log4j.logger.statistics=DEBUG, @app-name@_stats
4848
log4j.additivity.statistics= false
4949

5050
# set to DEBUG to log Spring's SQL statement and execution
51-
log4j.logger.org.springframework.jdbc.core.JdbcTemplate=DEBUG, @app-name@
51+
log4j.logger.org.springframework.jdbc.core.JdbcTemplate=WARN, @app-name@
5252
log4j.logger.org.springframework.jdbc=WARN, @app-name@
5353

54+
# set to DEBUG to log hsqldb SQL statement and execution
55+
log4j.logger.org.hsqldb.jdbc=WARN, @app-name@
5456

5557
# ehcache logging level
5658
#log4j.logger.net.sf.ehcache=debug

‎src/firefly/java/edu/caltech/ipac/firefly/data/FileInfo.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ public class FileInfo implements HasAccessInfo, Serializable {
3434
//----------------------- Constructors ---------------------------------
3535
//======================================================================
3636

37-
private FileInfo() {}
37+
protected FileInfo() {}
3838

3939
public FileInfo(String internalFilename, String externalName, long sizeInBytes) {
4040
setInternalName(internalFilename);

‎src/firefly/java/edu/caltech/ipac/firefly/data/TableServerRequest.java

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ public class TableServerRequest extends ServerRequest implements Serializable, D
3333
public static final String INCL_COLUMNS = "inclCols";
3434
public static final String FIXED_LENGTH = "fixedLength";
3535
public static final String META_INFO = "META_INFO";
36-
public static final List<String> SYS_PARAMS = Arrays.asList(new String[]{INCL_COLUMNS,FIXED_LENGTH,META_INFO,TBL_ID,DECIMATE_INFO,SQL_FROM});
36+
public static final List<String> SYS_PARAMS = Arrays.asList(REQUEST_CLASS,INCL_COLUMNS,SORT_INFO,FILTERS,PAGE_SIZE,START_IDX,FIXED_LENGTH,META_INFO,TBL_ID,DECIMATE_INFO,SQL_FROM);
3737
public static final String TBL_INDEX = "tbl_index"; // the table to show if it's a multi-table file.
3838

3939
private int pageSize;
@@ -260,8 +260,7 @@ public static String toFilterStr(List<String> l) {
260260
public SortedSet<Param> getSearchParams() {
261261
TreeSet<Param> params = new TreeSet<>();
262262
for (Param p : getParams()) {
263-
if (!SYS_PARAMS.contains(p.getName()) ||
264-
p.getName().equals(SORT_INFO)) {
263+
if (!SYS_PARAMS.contains(p.getName())) {
265264
params.add(p);
266265
}
267266
}

‎src/firefly/java/edu/caltech/ipac/firefly/server/catquery/GatorDD.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ public FileInfo createDbFile(TableServerRequest treq) throws DataAccessException
6969
new DataType("tableflg", Integer.class),
7070
new DataType("sel", String.class)
7171
});
72-
EmbeddedDbUtil.createDDTbl(dbFile, dd, dbAdapter);
72+
EmbeddedDbUtil.createDDTbl(dbFile, dd, dbAdapter, "data");
7373
EmbeddedDbUtil.setDbMetaInfo(treq, DbAdapter.getAdapter(treq), dbFile);
7474
}
7575
} catch (IOException e) {
@@ -97,7 +97,7 @@ protected DataGroupPart getDataset(TableServerRequest treq, File dbFile) throws
9797
String sql = String.format("%s %s %s", dbAdapter.selectPart(treq), dbAdapter.fromPart(treq), dbAdapter.wherePart(treq));
9898
sql = dbAdapter.translateSql(sql);
9999

100-
DataGroup dg = EmbeddedDbUtil.runQuery(dbAdapter, dbFile, sql);
100+
DataGroup dg = EmbeddedDbUtil.runQuery(dbAdapter, dbFile, sql, "data");
101101
TableDef tm = new TableDef();
102102
tm.setStatus(DataGroupPart.State.COMPLETED);
103103
return new DataGroupPart(tm, dg, treq.getStartIndex(), dg.size());

‎src/firefly/java/edu/caltech/ipac/firefly/server/db/BaseDbAdapter.java

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,8 @@
2222
*/
2323
abstract public class BaseDbAdapter implements DbAdapter {
2424

25-
private static final String DD_INSERT_SQL = "insert into dd values (?,?,?,?,?,?,?,?,?,?)";
26-
private static final String DD_CREATE_SQL = "create table if not exists dd "+
25+
private static final String DD_INSERT_SQL = "insert into %s_dd values (?,?,?,?,?,?,?,?,?,?)";
26+
private static final String DD_CREATE_SQL = "create table if not exists %s_dd "+
2727
"(" +
2828
" cname varchar(1023)" +
2929
", label varchar(1023)" +
@@ -37,28 +37,28 @@ abstract public class BaseDbAdapter implements DbAdapter {
3737
", desc varchar(1023)" +
3838
")";
3939

40-
private static final String META_INSERT_SQL = "insert into meta values (?,?)";
41-
private static final String META_CREATE_SQL = "create table if not exists meta "+
40+
private static final String META_INSERT_SQL = "insert into %s_meta values (?,?)";
41+
private static final String META_CREATE_SQL = "create table if not exists %s_meta "+
4242
"(" +
4343
" key varchar(1023)" +
4444
", value varchar(2023)" +
4545
")";
4646

4747

48-
public String createMetaSql(DataType[] dataDefinitions) {
49-
return META_CREATE_SQL;
48+
public String createMetaSql(String forTable) {
49+
return String.format(META_CREATE_SQL, forTable);
5050
}
5151

52-
public String insertMetaSql(DataType[] dataDefinitions) {
53-
return META_INSERT_SQL;
52+
public String insertMetaSql(String forTable) {
53+
return String.format(META_INSERT_SQL, forTable);
5454
}
5555

56-
public String createDDSql(DataType[] dataDefinitions) {
57-
return DD_CREATE_SQL;
56+
public String createDDSql(String forTable) {
57+
return String.format(DD_CREATE_SQL, forTable);
5858
}
5959

60-
public String insertDDSql(DataType[] dataDefinitions) {
61-
return DD_INSERT_SQL;
60+
public String insertDDSql(String forTable) {
61+
return String.format(DD_INSERT_SQL, forTable);
6262
}
6363

6464
public String createDataSql(DataType[] dtTypes, String tblName) {
@@ -79,12 +79,12 @@ public String insertDataSql(DataType[] dtTypes, String tblName) {
7979
return String.format("insert into %s values(%s)", tblName, StringUtils.toString(var, ","));
8080
}
8181

82-
public String getMetaSql() {
83-
return "select * from meta";
82+
public String getMetaSql(String forTable) {
83+
return String.format("select * from %s_meta", forTable);
8484
}
8585

86-
public String getDDSql() {
87-
return "select * from dd";
86+
public String getDDSql(String forTable) {
87+
return String.format("select * from %s_dd", forTable);
8888
}
8989

9090
public String selectPart(TableServerRequest treq) {

‎src/firefly/java/edu/caltech/ipac/firefly/server/db/DbAdapter.java

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -45,14 +45,14 @@ public interface DbAdapter {
4545
String createDataSql(DataType[] dataDefinitions, String tblName);
4646
String insertDataSql(DataType[] dataDefinitions, String tblName);
4747

48-
String createMetaSql(DataType[] dataDefinitions);
49-
String insertMetaSql(DataType[] dataDefinitions);
48+
String createMetaSql(String forTable);
49+
String insertMetaSql(String forTable);
5050

51-
String createDDSql(DataType[] dataDefinitions);
52-
String insertDDSql(DataType[] dataDefinitions);
51+
String createDDSql(String forTable);
52+
String insertDDSql(String forTable);
5353

54-
String getDDSql();
55-
String getMetaSql();
54+
String getDDSql(String forTable);
55+
String getMetaSql(String forTable);
5656

5757
String selectPart(TableServerRequest treq);
5858
String fromPart(TableServerRequest treq);

‎src/firefly/java/edu/caltech/ipac/firefly/server/db/EmbeddedDbUtil.java

Lines changed: 69 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
import edu.caltech.ipac.firefly.server.ServerContext;
99
import edu.caltech.ipac.firefly.server.db.spring.JdbcFactory;
1010
import edu.caltech.ipac.firefly.server.db.spring.mapper.DataGroupUtil;
11+
import edu.caltech.ipac.firefly.server.query.SearchProcessor;
1112
import edu.caltech.ipac.firefly.server.util.Logger;
1213
import edu.caltech.ipac.firefly.server.util.ipactable.DataGroupPart;
1314
import edu.caltech.ipac.firefly.server.util.ipactable.TableDef;
@@ -32,6 +33,7 @@
3233
import java.util.Arrays;
3334
import java.util.List;
3435
import java.util.Map;
36+
import java.util.stream.Collectors;
3537

3638
import static edu.caltech.ipac.firefly.data.TableServerRequest.TBL_FILE_PATH;
3739
import static edu.caltech.ipac.firefly.data.TableServerRequest.TBL_FILE_TYPE;
@@ -54,11 +56,15 @@ public class EmbeddedDbUtil {
5456
* @return a FileInfo with sizeInBytes representing to the number of rows.
5557
*/
5658
public static FileInfo createDbFile(File dbFile, DataGroup dg, DbAdapter dbAdapter) {
59+
60+
// remove ROW_IDX or ROW_NUM if exists
61+
dg.removeDataDefinition(DataGroup.ROW_IDX);
62+
dg.removeDataDefinition(DataGroup.ROW_NUM);
63+
5764
int rowCount = createDataTbl(dbFile, dg, dbAdapter);
58-
createDDTbl(dbFile, dg, dbAdapter);
59-
createMetaTbl(dbFile, dg, dbAdapter);
65+
createDDTbl(dbFile, dg, dbAdapter, "data");
66+
createMetaTbl(dbFile, dg, dbAdapter, "data");
6067
FileInfo finfo = new FileInfo(dbFile);
61-
finfo.setSizeInBytes(rowCount);
6268
return finfo;
6369
}
6470

@@ -137,36 +143,69 @@ public static String setupDatasetTable(TableServerRequest treq) {
137143
JdbcFactory.getSimpleTemplate(dbInstance).queryForInt(String.format("select count(*) from %s", datasetID));
138144
} catch (Exception e) {
139145
// does not exists.. create table from orignal 'data' table
140-
List<String> cols = JdbcFactory.getSimpleTemplate(dbInstance).query("select cname from DD", (rs, i) -> rs.getString(1));
146+
List<String> cols = getColumnNames(dbInstance, "DATA");
141147
String wherePart = dbAdapter.wherePart(treq);
142148
String orderBy = dbAdapter.orderByPart(treq);
143149

150+
// copy data
144151
String datasetSql = String.format("select %s, %s from data %s %s", StringUtils.toString(cols), DataGroup.ROW_IDX, wherePart, orderBy);
145152
String datasetSqlWithIdx = String.format("select b.*, (ROWNUM-1) as %s from (%s) as b", DataGroup.ROW_NUM, datasetSql);
146153
String sql = dbAdapter.createTableFromSelect(getDatasetID(treq), datasetSqlWithIdx);
147154
JdbcFactory.getSimpleTemplate(dbInstance).update(sql);
155+
156+
// copy dd
157+
String ddSql = "select * from data_dd";
158+
ddSql = dbAdapter.createTableFromSelect(getDatasetID(treq) + "_dd", ddSql);
159+
JdbcFactory.getSimpleTemplate(dbInstance).update(ddSql);
160+
161+
// copy meta
162+
String metaSql = "select * from data_meta";
163+
metaSql = dbAdapter.createTableFromSelect(getDatasetID(treq) + "_meta", metaSql);
164+
JdbcFactory.getSimpleTemplate(dbInstance).update(metaSql);
165+
148166
}
149167

150168
return datasetID;
151169
}
152170

171+
public static List<String> getColumnNames(DbInstance dbInstance, String forTable) {
172+
List<String> cols = JdbcFactory.getSimpleTemplate(dbInstance).query(String.format("select cname from %s_DD", forTable), (rs, i) -> rs.getString(1));
173+
cols = cols.stream().map((s) -> "\"" + s.toUpperCase() + "\"").collect(Collectors.toList()); // internally.. columns are stored as uppercase. adding quotes to avoid db reserved words.
174+
return cols;
175+
}
176+
177+
/**
178+
* return the DB file this request mapped to.
179+
* @param treq
180+
* @return
181+
*/
153182
public static File getDbFile(TableServerRequest treq) {
154-
return ServerContext.convertToFile(treq.getMeta(TBL_FILE_PATH));
183+
String fname = String.format("%s_%s.%s", treq.getRequestId(), DigestUtils.md5Hex(getUniqueID(treq)), DbAdapter.getAdapter(treq).getName());
184+
return new File(ServerContext.getTempWorkDir(), fname);
155185
}
156186

157187
public static void setDbMetaInfo(TableServerRequest treq, DbAdapter dbAdapter, File dbFile) {
158188
treq.setMeta(TBL_FILE_PATH, ServerContext.replaceWithPrefix(dbFile));
159189
treq.setMeta(TBL_FILE_TYPE, dbAdapter.getName());
160190
}
161191

192+
public static String getTblFileType(TableServerRequest treq) {
193+
return treq.getMeta(TBL_FILE_TYPE);
194+
}
195+
162196
@NotNull
163197
public static String getDatasetID(TableServerRequest treq) {
164198
String id = StringUtils.toString(treq.getDataSetParam(), "|");
165-
return StringUtils.isEmpty(id) ? "" : "view_" + DigestUtils.md5Hex(id);
199+
return StringUtils.isEmpty(id) ? "" : "data_" + DigestUtils.md5Hex(id);
200+
}
201+
202+
@NotNull
203+
public static String getUniqueID(TableServerRequest treq) {
204+
return SearchProcessor.getUniqueIDDef(treq);
166205
}
167206

168-
public static DataGroupPart getResults(TableServerRequest treq, String sql) {
169-
DataGroup dg = runQuery(DbAdapter.getAdapter(treq), getDbFile(treq), sql);
207+
public static DataGroupPart getResults(TableServerRequest treq, String sql, String tblName) {
208+
DataGroup dg = runQuery(DbAdapter.getAdapter(treq), getDbFile(treq), sql, tblName);
170209
TableDef tm = new TableDef();
171210
tm.setStatus(DataGroupPart.State.COMPLETED);
172211
tm.setRowCount(dg.size());
@@ -208,7 +247,7 @@ public void doInTransactionWithoutResult(TransactionStatus status) {
208247
return totalRows;
209248
}
210249

211-
public static DataGroup runQuery(DbAdapter dbAdapter, File dbFile, String sql) {
250+
public static DataGroup runQuery(DbAdapter dbAdapter, File dbFile, String sql, String tblName) {
212251
DbInstance dbInstance = dbAdapter.getDbInstance(dbFile);
213252
sql = dbAdapter.translateSql(sql);
214253

@@ -218,20 +257,22 @@ public static DataGroup runQuery(DbAdapter dbAdapter, File dbFile, String sql) {
218257

219258
SimpleJdbcTemplate jdbc = JdbcFactory.getSimpleTemplate(dbAdapter.getDbInstance(dbFile));
220259

221-
// insert DD info into the results
222-
try {
223-
String ddSql = dbAdapter.getDDSql();
224-
jdbc.query(ddSql, (rs, i) -> EmbeddedDbUtil.insertDD(dg, rs));
225-
} catch (Exception e) {
226-
// ignore.. may not have DD table
227-
}
260+
if (tblName != null) {
261+
// insert DD info into the results
262+
try {
263+
String ddSql = dbAdapter.getDDSql(tblName);
264+
jdbc.query(ddSql, (rs, i) -> EmbeddedDbUtil.insertDD(dg, rs));
265+
} catch (Exception e) {
266+
// ignore.. may not have DD table
267+
}
228268

229-
// insert table meta info into the results
230-
try {
231-
String metaSql = dbAdapter.getMetaSql();
232-
jdbc.query(metaSql, (rs, i) -> EmbeddedDbUtil.insertMeta(dg, rs));
233-
} catch (Exception e) {
234-
// ignore.. may not have meta table
269+
// insert table meta info into the results
270+
try {
271+
String metaSql = dbAdapter.getMetaSql(tblName);
272+
jdbc.query(metaSql, (rs, i) -> EmbeddedDbUtil.insertMeta(dg, rs));
273+
} catch (Exception e) {
274+
// ignore.. may not have meta table
275+
}
235276
}
236277

237278
return dg;
@@ -259,10 +300,10 @@ public int getBatchSize() {
259300
});
260301
}
261302

262-
public static void createMetaTbl(File dbFile, DataGroup dg, DbAdapter dbAdapter) {
303+
public static void createMetaTbl(File dbFile, DataGroup dg, DbAdapter dbAdapter, String forTable) {
263304
if (dg.getAttributeKeys().size() == 0) return;
264305

265-
String createMetaSql = dbAdapter.createMetaSql(dg.getDataDefinitions());
306+
String createMetaSql = dbAdapter.createMetaSql(forTable);
266307
logger.debug("createMetaSql:" + createMetaSql);
267308
JdbcFactory.getSimpleTemplate(dbAdapter.getDbInstance(dbFile)).update(createMetaSql);
268309

@@ -272,16 +313,16 @@ public static void createMetaTbl(File dbFile, DataGroup dg, DbAdapter dbAdapter)
272313
String val = meta.get(key).getValue();
273314
data.add(new Object[]{key, val});
274315
}
275-
String insertDDSql = dbAdapter.insertMetaSql(dg.getDataDefinitions());
316+
String insertDDSql = dbAdapter.insertMetaSql(forTable);
276317
logger.debug("insertDDSql:" + insertDDSql);
277318
JdbcFactory.getSimpleTemplate(dbAdapter.getDbInstance(dbFile)).batchUpdate(insertDDSql, data);
278319
}
279320

280321

281-
public static void createDDTbl(File dbFile, DataGroup dg, DbAdapter dbAdapter) {
322+
public static void createDDTbl(File dbFile, DataGroup dg, DbAdapter dbAdapter, String forTable) {
282323

283324
makeDbCols(dg);
284-
String createDDSql = dbAdapter.createDDSql(dg.getDataDefinitions());
325+
String createDDSql = dbAdapter.createDDSql(forTable);
285326
logger.debug("createDDSql:" + createDDSql);
286327
JdbcFactory.getSimpleTemplate(dbAdapter.getDbInstance(dbFile)).update(createDDSql);
287328

@@ -310,7 +351,7 @@ public static void createDDTbl(File dbFile, DataGroup dg, DbAdapter dbAdapter) {
310351
}
311352
);
312353
}
313-
String insertDDSql = dbAdapter.insertDDSql(dg.getDataDefinitions());
354+
String insertDDSql = dbAdapter.insertDDSql(forTable);
314355
logger.debug("insertDDSql:" + insertDDSql);
315356
JdbcFactory.getSimpleTemplate(dbAdapter.getDbInstance(dbFile)).batchUpdate(insertDDSql, data);
316357
}

0 commit comments

Comments
 (0)