Skip to content

Commit 93c3f67

Browse files
loitlyloitly
committed
DM-8361: PDAC related issues
 - Table shows working icon even after when it's fully loaded. - fixed LSST pdac not reloadable. Need to sync encode/decode routines - fixed fits table read; removing control characters from table attributes.
1 parent 0ac9c66 commit 93c3f67

18 files changed

+648
-700
lines changed

‎src/firefly/java/edu/caltech/ipac/firefly/server/catquery/CatMasterTableQuery.java

Lines changed: 19 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -54,21 +54,6 @@ public class CatMasterTableQuery extends IpacTablePartProcessor {
5454
private static final String IRSA_HOST = AppProperties.getProperty("irsa.base.url", BaseGator.DEF_HOST);
5555
private static final String IRSA_BASE_URL = QueryUtil.makeUrlBase(IRSA_HOST);
5656

57-
58-
protected File loadDataFile(TableServerRequest request) throws IOException, DataAccessException {
59-
File retFile;
60-
61-
StringKey key = new StringKey(CatMasterTableQuery.class.getName(), getUniqueID(request));
62-
Cache cache = CacheManager.getCache(Cache.TYPE_TEMP_FILE);
63-
retFile = (File) cache.get(key);
64-
if (retFile == null) {
65-
retFile = getMasterCatalogFile(request);
66-
cache.put(key, retFile);
67-
}
68-
69-
return retFile;
70-
}
71-
7257
private static void placeOriginal(String catalogUrl, File originalFile) throws IOException {
7358
try {
7459
URL url = new URL(catalogUrl);
@@ -145,7 +130,7 @@ private static File getMasterCatalogFile(TableServerRequest request) throws IOEx
145130
outputDG= dgExtra;
146131
// DataGroupWriter.write(catOutFile, dgExtra, 0);
147132
}
148-
DataGroupWriter.write(catOutFile, outputDG, 0);
133+
DataGroupWriter.write(catOutFile, outputDG);
149134

150135
DataGroup data = DataGroupReader.read(catOutFile);
151136
// append hostname to relative path urls.
@@ -167,7 +152,7 @@ private static File getMasterCatalogFile(TableServerRequest request) throws IOEx
167152

168153
return retval;
169154
}
170-
155+
171156
private static String getValue(DataObject row, String colName) {
172157
try {
173158
return (String) row.getDataElement(colName);
@@ -176,8 +161,7 @@ private static String getValue(DataObject row, String colName) {
176161
}
177162
return null;
178163
}
179-
180-
164+
181165
private static void addSearchProcessorCols(DataGroup dg) {
182166

183167
DataType catDt= new DataType("catSearchProcessor", String.class);
@@ -192,11 +176,8 @@ private static void addSearchProcessorCols(DataGroup dg) {
192176
}
193177
}
194178

195-
196-
197-
198179
private static void makeIntoUrl(DataGroup dg, String colname, String linkDesc) {
199-
180+
200181
DataType col = dg.getDataDefintion(colname);
201182
for (int r = 0; r < dg.size(); r++) {
202183
DataObject row = dg.get(r);
@@ -217,14 +198,13 @@ private static void makeIntoUrl(DataGroup dg, String colname, String linkDesc) {
217198
}
218199
}
219200

220-
221201
private static void appendHostToUrls(DataGroup dg, String linkDesc, String... cols) {
222202

223203
if (cols == null || cols.length ==0) return;
224204
for(int c = 0; c < cols.length; c++) {
225205
DataType col = dg.getDataDefintion(cols[c]);
226206
if (col == null) continue;
227-
207+
228208
for (int r = 0; r < dg.size(); r++) {
229209
DataObject row = dg.get(r);
230210
String val = getValue(row, col.getKeyName());
@@ -276,5 +256,19 @@ public static DataGroup getBaseGatorData(String originalFilename) throws IOExcep
276256
return DataGroupReader.read(new File(ServerContext.getPermWorkDir(), originalFilename));
277257
}
278258

259+
protected File loadDataFile(TableServerRequest request) throws IOException, DataAccessException {
260+
File retFile;
261+
262+
StringKey key = new StringKey(CatMasterTableQuery.class.getName(), getUniqueID(request));
263+
Cache cache = CacheManager.getCache(Cache.TYPE_TEMP_FILE);
264+
retFile = (File) cache.get(key);
265+
if (retFile == null) {
266+
retFile = getMasterCatalogFile(request);
267+
cache.put(key, retFile);
268+
}
269+
270+
return retFile;
271+
}
272+
279273
}
280274

‎src/firefly/java/edu/caltech/ipac/firefly/server/persistence/QueryLSSTCatalog.java

Lines changed: 28 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
import edu.caltech.ipac.firefly.server.util.Logger;
1212
import edu.caltech.ipac.firefly.server.util.ipactable.DataGroupPart;
1313
import edu.caltech.ipac.firefly.server.util.ipactable.DataGroupWriter;
14+
import edu.caltech.ipac.firefly.server.util.ipactable.BgIpacTableHandler;
1415
import edu.caltech.ipac.firefly.visualize.VisUtil;
1516
import edu.caltech.ipac.util.*;
1617
import edu.caltech.ipac.util.download.FailedRequestException;
@@ -45,6 +46,32 @@ public class QueryLSSTCatalog extends IpacTablePartProcessor {
4546
private static String DATA_ACCESS_URI = AppProperties.getProperty("lsst.dataAccess.uri", "lsst.dataAccess.uri");
4647
private static String DATABASE = AppProperties.getProperty("lsst.dataAccess.db", "lsst.dataAccess.db");
4748

49+
public static void main(String[] args) {
50+
51+
//String sql="select * from DeepSource where qserv_areaspec_box(0.4, 1.05, 0.5, 1.15)";
52+
String sql="SELECT ra,decl,filterName FROM DC_W13_Stripe82.Science_Ccd_Exposure WHERE scisql_s2PtInBox(ra,decl,330,-0.1,335.1,-0.08)=1";
53+
54+
55+
try {
56+
long cTime = System.currentTimeMillis();
57+
String url = "http://localhost:8661/db/v0/query?sql="+ URLEncoder.encode(sql, "UTF-8");
58+
System.out.println("Executing SQL query: " + sql);
59+
try {
60+
URLConnection uc = URLDownload.makeConnection(new URL(url));
61+
uc.setRequestProperty("Accept", "text/plain");
62+
URLDownload.getDataToFile(uc, new File("/tmp/result.json"), null);
63+
} catch (FailedRequestException e) {
64+
throw new IOException("Request Failed", e);
65+
}
66+
67+
System.out.println("SELECT took " + (System.currentTimeMillis() - cTime) + "ms");
68+
69+
70+
} catch (Exception e) {
71+
System.out.println("Exception "+e.getMessage());
72+
e.printStackTrace();
73+
}
74+
}
4875

4976
@Override
5077
protected File loadDataFile(TableServerRequest request) throws IOException, DataAccessException {
@@ -60,8 +87,7 @@ protected File loadDataFile(TableServerRequest request) throws IOException, Data
6087

6188
DataGroup dg = JsonToDataGroup.parse(file);
6289
File inf = createFile(request, ".tbl");
63-
DataGroupWriter.write(inf, dg, 0); // for big files write will happen in background
64-
90+
DataGroupWriter.write(new BgIpacTableHandler(inf, dg, request)); // for big files write will happen in background
6591
return inf;
6692
}
6793

@@ -173,7 +199,6 @@ void doGetData(File oFile, List<Param> params, WorldPt wpt) throws DataAccessExc
173199
}
174200
}
175201

176-
177202
@Override
178203
public void prepareTableMeta(TableMeta meta, List<DataType> columns, ServerRequest request) {
179204
super.prepareTableMeta(meta, columns, request);
@@ -232,8 +257,6 @@ private String getSelectedColumnsUpdate(String selectedColumns, String raCol, St
232257

233258
}
234259

235-
236-
237260
protected void setColumnTips(TableMeta meta, ServerRequest request) {
238261

239262
TableServerRequest req = new TableServerRequest("LSSTCatalogDD");
@@ -273,32 +296,5 @@ protected void setColumnTips(TableMeta meta, ServerRequest request) {
273296
}
274297
}
275298

276-
public static void main(String[] args) {
277-
278-
//String sql="select * from DeepSource where qserv_areaspec_box(0.4, 1.05, 0.5, 1.15)";
279-
String sql="SELECT ra,decl,filterName FROM DC_W13_Stripe82.Science_Ccd_Exposure WHERE scisql_s2PtInBox(ra,decl,330,-0.1,335.1,-0.08)=1";
280-
281-
282-
try {
283-
long cTime = System.currentTimeMillis();
284-
String url = "http://localhost:8661/db/v0/query?sql="+ URLEncoder.encode(sql, "UTF-8");
285-
System.out.println("Executing SQL query: " + sql);
286-
try {
287-
URLConnection uc = URLDownload.makeConnection(new URL(url));
288-
uc.setRequestProperty("Accept", "text/plain");
289-
URLDownload.getDataToFile(uc, new File("/tmp/result.json"), null);
290-
} catch (FailedRequestException e) {
291-
throw new IOException("Request Failed", e);
292-
}
293-
294-
System.out.println("SELECT took " + (System.currentTimeMillis() - cTime) + "ms");
295-
296-
297-
} catch (Exception e) {
298-
System.out.println("Exception "+e.getMessage());
299-
e.printStackTrace();
300-
}
301-
}
302-
303299

304300
}

‎src/firefly/java/edu/caltech/ipac/firefly/server/persistence/QueryLSSTCatalogDD.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,7 @@ protected File loadDataFile(TableServerRequest request) throws IOException, Data
9595
}
9696
toReturn.shrinkToFitData();
9797
File inf = createFile(request, ".tbl");
98-
DataGroupWriter.write(inf, toReturn, 0);
98+
DataGroupWriter.write(inf, toReturn);
9999
return inf;
100100
} else {
101101
return null;

‎src/firefly/java/edu/caltech/ipac/firefly/server/persistence/QuerySearchHistory.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,7 @@ public int compare(SearchInfo o1, SearchInfo o2) {
7777
dg.add(row);
7878
}
7979
File f = File.createTempFile(getFilePrefix(request), ".tbl", ServerContext.getTempWorkDir());
80-
DataGroupWriter.write(f, dg, Integer.MAX_VALUE);
80+
DataGroupWriter.write(f, dg);
8181
return f;
8282
} else {
8383
return super.loadDataFile(request);

‎src/firefly/java/edu/caltech/ipac/firefly/server/query/HistogramProcessor.java

Lines changed: 34 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,39 @@ public class HistogramProcessor extends IpacTablePartProcessor {
6060
private double falsePostiveRate = 0.05;
6161
private boolean showEmptyBin=false;
6262

63+
private static String[] getInputFilePath(String inputFileName) {
64+
String[] dirs = inputFileName.split("/");
65+
String name = dirs[dirs.length - 1];
66+
String path = inputFileName.substring(0, inputFileName.length() - name.length());
67+
return new String[]{path, name};
68+
}
69+
70+
public static void main(String args[]) throws IOException, DataAccessException {
71+
72+
if (args.length > 0) {
73+
String path = getInputFilePath(args[0])[0];
74+
String inFileName = getInputFilePath(args[0])[1];
75+
if (args.length > 0) {
76+
try {
77+
File inFile = new File(args[0]);
78+
DataGroup dg = IpacTableReader.readIpacTable(inFile, null, false, "inputTable");
79+
80+
HistogramProcessor hp = new HistogramProcessor();
81+
hp.columnExpression = "f_y";
82+
83+
double[] columnData = hp.getColumnData(dg);
84+
85+
DataGroup outDg = hp.createHistogramTable(columnData);
86+
String outFileName = path + "output_" + inFileName;
87+
File outFile = new File(outFileName);
88+
IpacTableWriter.save(outFile, outDg);
89+
90+
} catch (IpacTableException e) {
91+
e.printStackTrace();
92+
}
93+
}
94+
}
95+
}
6396

6497
protected File loadDataFile(TableServerRequest request) throws IOException, DataAccessException {
6598

@@ -104,7 +137,7 @@ protected File loadDataFile(TableServerRequest request) throws IOException, Data
104137
addFormatInfoAtt(histogramDataGroup, columns[2]);
105138

106139
File histogramFile = createFile(request);
107-
DataGroupWriter.write(histogramFile, histogramDataGroup, 0);
140+
DataGroupWriter.write(histogramFile, histogramDataGroup);
108141
return histogramFile;
109142
}
110143

@@ -287,13 +320,6 @@ private double[] getColumnData(DataGroup dg) {
287320
return Arrays.stream(data).filter(d -> !Double.isNaN(d)).toArray();
288321
}
289322

290-
private static String[] getInputFilePath(String inputFileName) {
291-
String[] dirs = inputFileName.split("/");
292-
String name = dirs[dirs.length - 1];
293-
String path = inputFileName.substring(0, inputFileName.length() - name.length());
294-
return new String[]{path, name};
295-
}
296-
297323
/**
298324
* This method calculate the variable bins based on the paper:
299325
* http://iopscience.iop.org/article/10.1088/0004-637X/764/2/167/pdf;jsessionid=22827FAAA086B2A127E88C517E0E8DD3.c1.iopscience.cld.iop.org
@@ -498,7 +524,6 @@ private int getIndexOfTheMaxValue(double[] inArray) {
498524
return maxIdx;
499525
}
500526

501-
502527
/**
503528
* This method evaluate the fitness function for these possibilities
504529
*/
@@ -659,32 +684,5 @@ private double[] multiply(double[] array, double a) {
659684
return newArray;
660685
}
661686

662-
public static void main(String args[]) throws IOException, DataAccessException {
663-
664-
if (args.length > 0) {
665-
String path = getInputFilePath(args[0])[0];
666-
String inFileName = getInputFilePath(args[0])[1];
667-
if (args.length > 0) {
668-
try {
669-
File inFile = new File(args[0]);
670-
DataGroup dg = IpacTableReader.readIpacTable(inFile, null, false, "inputTable");
671-
672-
HistogramProcessor hp = new HistogramProcessor();
673-
hp.columnExpression = "f_y";
674-
675-
double[] columnData = hp.getColumnData(dg);
676-
677-
DataGroup outDg = hp.createHistogramTable(columnData);
678-
String outFileName = path + "output_" + inFileName;
679-
File outFile = new File(outFileName);
680-
IpacTableWriter.save(outFile, outDg);
681-
682-
} catch (IpacTableException e) {
683-
e.printStackTrace();
684-
}
685-
}
686-
}
687-
}
688-
689687

690688
}

‎src/firefly/java/edu/caltech/ipac/firefly/server/query/IpacTablePartProcessor.java

Lines changed: 5 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -22,18 +22,12 @@
2222
import edu.caltech.ipac.firefly.server.util.Logger;
2323
import edu.caltech.ipac.firefly.server.util.QueryUtil;
2424
import edu.caltech.ipac.firefly.server.util.StopWatch;
25-
import edu.caltech.ipac.firefly.server.util.ipactable.DataGroupFilter;
26-
import edu.caltech.ipac.firefly.server.util.ipactable.DataGroupPart;
27-
import edu.caltech.ipac.firefly.server.util.ipactable.DataGroupReader;
28-
import edu.caltech.ipac.firefly.server.util.ipactable.DataGroupWriter;
29-
import edu.caltech.ipac.firefly.server.util.ipactable.IpacTableParser;
30-
import edu.caltech.ipac.firefly.server.util.ipactable.TableDef;
25+
import edu.caltech.ipac.firefly.server.util.ipactable.*;
3126
import edu.caltech.ipac.util.AppProperties;
3227
import edu.caltech.ipac.util.CollectionUtil;
3328
import edu.caltech.ipac.util.DataGroup;
3429
import edu.caltech.ipac.util.DataObject;
3530
import edu.caltech.ipac.util.DataType;
36-
import edu.caltech.ipac.util.FileUtil;
3731
import edu.caltech.ipac.util.IpacTableUtil;
3832
import edu.caltech.ipac.util.StringUtils;
3933
import edu.caltech.ipac.util.cache.Cache;
@@ -120,7 +114,7 @@ protected static File convertToIpacTable(File tblFile, TableServerRequest reques
120114
// read in any format.. then write it back out as ipac table
121115
DataGroup dg = DataGroupReader.readAnyFormat(tblFile, tblIdx);
122116
File convertedFile = File.createTempFile(request.getRequestId(), ".tbl", ServerContext.getTempWorkDir());
123-
DataGroupWriter.write(convertedFile, dg, 0, request.getMeta());
117+
DataGroupWriter.write(new BgIpacTableHandler(convertedFile, dg, request));
124118
return convertedFile;
125119
} else {
126120
throw new DataAccessException("Source file has an unknown format:" + ServerContext.replaceWithPrefix(tblFile));
@@ -430,7 +424,7 @@ public File getDataFile(TableServerRequest request) throws IpacTableException, I
430424

431425
deciFile = File.createTempFile(getFilePrefix(request), ".tbl", ServerContext.getTempWorkDir());
432426
DataGroup retval = QueryUtil.doDecimation(dg, decimateInfo);
433-
DataGroupWriter.write(deciFile, retval, Integer.MAX_VALUE, request.getMeta());
427+
DataGroupWriter.write(deciFile, retval);
434428
cache.put(key, deciFile);
435429
}
436430
resultsFile = deciFile;
@@ -496,7 +490,7 @@ protected void doSort(File inFile, File outFile, SortInfo sortInfo, TableServerR
496490
QueryUtil.doSort(dg, sortInfo);
497491
timer.printLog("sort");
498492
timer.start("write");
499-
DataGroupWriter.write(outFile, dg, pageSize, request.getMeta());
493+
DataGroupWriter.write(new BgIpacTableHandler(outFile, dg, request));
500494
timer.printLog("write");
501495
}
502496

@@ -604,7 +598,7 @@ private boolean isInitLoad(TableServerRequest req) {
604598
protected void doFilter(File outFile, File source, CollectionUtil.Filter<DataObject>[] filters, TableServerRequest request) throws IOException {
605599
StopWatch timer = StopWatch.getInstance();
606600
timer.start("filter");
607-
DataGroupFilter.filter(outFile, source, filters, request.getPageSize(), request.getMeta());
601+
DataGroupWriter.write(new FilterHanlder(outFile, source, filters, request));
608602
timer.printLog("filter");
609603
}
610604

0 commit comments

Comments
 (0)