Skip to content

Commit 89b574b

Browse files
loitlyloitly
committed
FIREFLY-148: Refactor file analysis to make it more general
Server-side - refactor AnyFileUpload. - much easier to read - dramatically improve table upload due to bad logic from previous implementation - added ability to ask for different types of analysis report (Brief, Normal, Details) - added FileAnalysisCmd to get an analysis report from a file already on the server - define a model for the analysis report - every data product must returns the same model, this includes fits, votable, ipactable, csv, and tsv. - Remove error-prone guessing of file format based on file extension - tests added and updated where applicable. Client-side - refactor FileUploadViewPanel - convert it into a functional component - change logic so it will take the new analysis model - display is consistent for all data types - table is much more advanced with sorting and filtering - added display of Table Meta information into TablePanel - this will show meta, params, and links. - groups and/or data links can be added later. - Update FileUpload to handle new analysis model
1 parent f81bc3e commit 89b574b

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

43 files changed

+1559
-1270
lines changed

‎src/firefly/html/css/global.css

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -277,6 +277,20 @@ div.rootStyle>img, div.rootStyle > html {
277277
font-weight: bold;
278278
}
279279

280+
281+
.keyword-label {
282+
font-weight: bold;
283+
margin-right: 3px;
284+
}
285+
286+
.keyword-value {
287+
color: maroon;
288+
margin-right: 10px;
289+
text-overflow: ellipsis;
290+
}
291+
292+
293+
280294
/*-------------------< loading mask ---------------------*/
281295
.loading-mask {
282296
position: absolute;

‎src/firefly/java/edu/caltech/ipac/firefly/core/FileAnalysis.java

Lines changed: 166 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,166 @@
1+
/*
2+
* License information at https://github.com/Caltech-IPAC/firefly/blob/master/License.txt
3+
*/
4+
5+
package edu.caltech.ipac.firefly.core;
6+
7+
import edu.caltech.ipac.firefly.messaging.JsonHelper;
8+
import edu.caltech.ipac.table.DataGroup;
9+
import edu.caltech.ipac.table.JsonTableUtil;
10+
import edu.caltech.ipac.table.TableUtil;
11+
import edu.caltech.ipac.table.TableUtil.Format;
12+
import edu.caltech.ipac.table.io.DsvTableIO;
13+
import edu.caltech.ipac.table.io.IpacTableReader;
14+
import edu.caltech.ipac.table.io.VoTableReader;
15+
import edu.caltech.ipac.util.FitsHDUUtil;
16+
import edu.caltech.ipac.util.StringUtils;
17+
18+
import java.io.File;
19+
import java.util.ArrayList;
20+
import java.util.List;
21+
import java.util.stream.Collectors;
22+
23+
import static edu.caltech.ipac.util.StringUtils.isEmpty;
24+
25+
/**
26+
* Date: 2019-06-27
27+
*
28+
* @author loi
29+
* @version $Id: $
30+
*/
31+
public class FileAnalysis {
32+
public enum ReportType {Brief, // expect to only get a report with one part without details
33+
Normal, // a report with all parts populated, but not details
34+
Details} // a full report with details
35+
public enum Type {Image, Table, Spectrum, ImageNoData, Unknown}
36+
37+
38+
public static Report analyze(File infile, ReportType type) throws Exception {
39+
40+
Format format = TableUtil.guessFormat(infile);
41+
switch (format) {
42+
case VO_TABLE:
43+
return VoTableReader.analyze(infile, type);
44+
case FITS:
45+
return FitsHDUUtil.analyze(infile, type);
46+
case IPACTABLE:
47+
return IpacTableReader.analyze(infile, type);
48+
case CSV:
49+
case TSV:
50+
return DsvTableIO.analyze(infile, format.type, type);
51+
default:
52+
53+
}
54+
Report report = new Report(type, infile.length(), infile.getAbsolutePath());
55+
report.addPart(new Part(Type.Unknown, 0, "Unknown Type"));
56+
return report;
57+
};
58+
59+
public static String toJsonString(Report report) {
60+
JsonHelper helper = new JsonHelper();
61+
helper.setValue(report.filePath, "filePath");
62+
helper.setValue(report.fileName, "fileName");
63+
helper.setValue(report.fileSize, "fileSize");
64+
helper.setValue(report.type.name(), "type");
65+
helper.setValue(report.getDataType(), "dataTypes");
66+
for(int i = 0; i < report.getParts().size(); i++) {
67+
Part p = report.getParts().get(i);
68+
helper.setValue(p.index, "parts", i+"", "index");
69+
helper.setValue(p.type.name(), "parts", i+"", "type");
70+
helper.setValue(p.desc, "parts", i+"", "desc");
71+
if (!isEmpty(p.getDetails())) {
72+
helper.setValue(JsonTableUtil.toJsonDataGroup(p.getDetails()), "parts", i+"", "details");
73+
}
74+
}
75+
return helper.toJson();
76+
}
77+
78+
79+
//====================================================================
80+
//
81+
//====================================================================
82+
83+
public static class Report {
84+
private ReportType type;
85+
private long fileSize;
86+
private String filePath;
87+
private String fileName;
88+
private List<Part> parts;
89+
90+
public Report(ReportType type, long fileSize, String filePath) {
91+
this.type = type;
92+
this.fileSize = fileSize;
93+
this.filePath = filePath;
94+
}
95+
96+
public ReportType getType() {
97+
return type;
98+
}
99+
100+
public long getFileSize() {
101+
return fileSize;
102+
}
103+
104+
public String getFilePath() {
105+
return filePath;
106+
}
107+
108+
public List<Part> getParts() {
109+
return parts;
110+
}
111+
112+
public String getFileName() { return fileName; }
113+
114+
public void setFileName(String fileName) { this.fileName = fileName; }
115+
116+
public void addPart(Part part) {
117+
if (parts == null) parts = new ArrayList<>();
118+
parts.add(part);
119+
}
120+
121+
public String getDataType() {
122+
if (parts != null) {
123+
if (parts.size() == 1) {
124+
return parts.get(0).type.name();
125+
} else {
126+
List<String> types = parts.stream().map(part -> part.type.name()).distinct().collect(Collectors.toList());
127+
return StringUtils.toString(types);
128+
}
129+
}
130+
return "";
131+
}
132+
}
133+
134+
public static class Part {
135+
private Type type;
136+
private int index;
137+
private String desc;
138+
private DataGroup details;
139+
140+
public Part(Type type) {
141+
this.type = type;
142+
}
143+
144+
public Part(Type type, int index, String desc) {
145+
this.type = type;
146+
this.index = index;
147+
this.desc = desc;
148+
}
149+
150+
public Type getType() { return type; }
151+
public void setType(Type type) { this.type = type; }
152+
153+
public int getIndex() { return index; }
154+
public void setIndex(int index) { this.index = index; }
155+
156+
public String getDesc() { return desc;}
157+
public void setDesc(String desc) { this.desc = desc; }
158+
159+
public DataGroup getDetails() {
160+
return details;
161+
}
162+
public void setDetails(DataGroup details) {
163+
this.details = details;
164+
}
165+
}
166+
}

‎src/firefly/java/edu/caltech/ipac/firefly/data/ServerParams.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -187,5 +187,6 @@ public class ServerParams {
187187
public static final String WS_CREATE_FOLDER = "wsParent";
188188
public static final String SOURCE_FROM = "sourceFrom";
189189
public static final String IS_WS = "isWs";
190+
public static final String FILE_ANALYSIS = "fileAnalysis";
190191
}
191192

‎src/firefly/java/edu/caltech/ipac/firefly/messaging/JsonHelper.java

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -115,9 +115,12 @@ private Consumer getContainer(String[] paths) {
115115
}
116116

117117
private Object ensureCont(Object cont, String key) {
118-
if (cont != null) return cont;
119118
int idx = getInt(key, -1);
120-
return idx == -1 ? new HashMap() : ensureList(new ArrayList(), idx);
119+
if (cont == null) {
120+
cont = idx == -1 ? new HashMap() :new ArrayList();
121+
}
122+
if (idx >= 0 && cont instanceof List) cont = ensureList((List) cont, idx);
123+
return cont;
121124
}
122125

123126
private Object getCont(Object source, String key, String nkey) {
@@ -130,6 +133,8 @@ private Object getCont(Object source, String key, String nkey) {
130133
if (cont == null) {
131134
cont = ensureCont(cont, nkey);
132135
clist.set(idx, cont);
136+
} else {
137+
cont = ensureCont(cont, nkey);
133138
}
134139
return cont;
135140
} else if (source instanceof Map) {
@@ -138,6 +143,8 @@ private Object getCont(Object source, String key, String nkey) {
138143
if (cont == null) {
139144
cont = ensureCont(cont, nkey);
140145
cmap.put(key, cont);
146+
} else {
147+
cont = ensureCont(cont, nkey);
141148
}
142149
return cont;
143150
}

‎src/firefly/java/edu/caltech/ipac/firefly/server/ServerCommandAccess.java

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,7 @@ private static void initCommand() {
8989
_cmdMap.put(ServerParams.CREATE_DOWNLOAD_SCRIPT, new SearchServerCommands.CreateDownloadScript());
9090
_cmdMap.put(ServerParams.PACKAGE_REQUEST, new SearchServerCommands.PackageRequest());
9191
_cmdMap.put(ServerParams.TABLE_SEARCH, new SearchServerCommands.TableSearch());
92-
_cmdMap.put(ServerParams.QUERY_TABLE, new SearchServerCommands.QueryTable());
92+
_cmdMap.put(ServerParams.QUERY_TABLE, new SearchServerCommands.QueryTable());
9393
_cmdMap.put(ServerParams.SELECTED_VALUES, new SearchServerCommands.SelectedValues());
9494
_cmdMap.put(ServerParams.JSON_SEARCH, new SearchServerCommands.JsonSearch());
9595

@@ -98,6 +98,8 @@ private static void initCommand() {
9898

9999
_cmdMap.put(ServerParams.RESOLVE_NAME, new ResolveServerCommands.ResolveName());
100100
_cmdMap.put(ServerParams.RESOLVE_NAIFID, new ResolveServerCommands.ResolveNaifidName());
101+
_cmdMap.put(ServerParams.FILE_ANALYSIS, new ResolveServerCommands.FileAnalysisCmd());
102+
101103
_cmdMap.put(ServerParams.VIS_PUSH_ALIVE_CHECK, new PushCommands.PushAliveCheck());
102104
_cmdMap.put(ServerParams.VIS_PUSH_ALIVE_COUNT, new PushCommands.PushAliveCount());
103105
_cmdMap.put(ServerParams.VIS_PUSH_ACTION, new PushCommands.PushAction());

‎src/firefly/java/edu/caltech/ipac/firefly/server/SrvParam.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,10 @@ public static SrvParam makeSrvParamSimpleMap(Map<String, String> map) {
5252
return new SrvParam(targetMap);
5353
}
5454

55+
public void setParam(String key, String value) {
56+
paramMap.put(key, new String[] {value});
57+
}
58+
5559
public void addParams(Map<String, String> map) {
5660
for( Map.Entry<String,String> entry : map.entrySet()) {
5761
if (!paramMap.containsKey(entry.getKey())) {

‎src/firefly/java/edu/caltech/ipac/firefly/server/query/EmbeddedDbProcessor.java

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
import edu.caltech.ipac.table.DataType;
3434
import edu.caltech.ipac.util.StringUtils;
3535
import org.apache.commons.codec.digest.DigestUtils;
36+
import org.json.simple.JSONObject;
3637
import org.springframework.core.NestedRuntimeException;
3738
import org.springframework.jdbc.BadSqlGrammarException;
3839
import org.springframework.jdbc.core.simple.SimpleJdbcTemplate;
@@ -239,6 +240,8 @@ protected FileInfo ingestDataIntoDb(TableServerRequest req, File dbFile) throws
239240
DataGroup dg = fetchDataGroup(req);
240241
StopWatch.getInstance().stop("fetchDataGroup: " + req.getRequestId()).printLog("fetchDataGroup: " + req.getRequestId());
241242

243+
if (dg == null) throw new DataAccessException("Failed to retrieve data");
244+
242245
prepareTableMeta(dg.getTableMeta(), Arrays.asList(dg.getDataDefinitions()), req);
243246
TableUtil.consumeColumnMeta(dg, null); // META-INFO in the request should only be pass-along and not persist.
244247

@@ -563,8 +566,10 @@ private static void enumeratedValuesCheckBG(File dbFile, DataGroupPart results,
563566
enumeratedValuesCheck(dbFile, results, treq);
564567
DataGroup updates = new DataGroup(null, results.getData().getDataDefinitions());
565568
updates.getTableMeta().setTblId(results.getData().getTableMeta().getTblId());
569+
JSONObject changes = JsonTableUtil.toJsonDataGroup(updates);
570+
changes.remove("totalRows"); //changes contains only the columns with 0 rows.. we don't want to update totalRows
566571

567-
FluxAction action = new FluxAction(FluxAction.TBL_UPDATE, JsonTableUtil.toJsonDataGroup(updates));
572+
FluxAction action = new FluxAction(FluxAction.TBL_UPDATE, changes);
568573
ServerEventManager.fireAction(action, target);
569574
});
570575
}

‎src/firefly/java/edu/caltech/ipac/firefly/server/rpc/ResolveServerCommands.java

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,13 +6,16 @@
66

77
import edu.caltech.ipac.astro.net.HorizonsEphPairs;
88
import edu.caltech.ipac.astro.net.Resolver;
9+
import edu.caltech.ipac.firefly.core.FileAnalysis;
910
import edu.caltech.ipac.firefly.data.ServerParams;
1011
import edu.caltech.ipac.firefly.server.ServCommand;
1112
import edu.caltech.ipac.firefly.server.SrvParam;
13+
import edu.caltech.ipac.util.StringUtils;
1214
import edu.caltech.ipac.visualize.plot.ResolvedWorldPt;
1315
import org.json.simple.JSONArray;
1416
import org.json.simple.JSONObject;
1517

18+
import java.io.File;
1619
import java.util.HashMap;
1720
import java.util.Map;
1821

@@ -76,5 +79,25 @@ public String doCommand(SrvParam sp) throws Exception {
7679

7780
public boolean getCanCreateJson() { return true; }
7881
}
82+
83+
84+
public static class FileAnalysisCmd extends ServCommand {
85+
86+
87+
public String doCommand(SrvParam sp) throws Exception {
88+
89+
String infile = null;
90+
try {
91+
infile = sp.getRequired("filePath");
92+
String rtype = sp.getOptional("reportType");
93+
FileAnalysis.ReportType reportType = StringUtils.isEmpty(rtype) ? FileAnalysis.ReportType.Brief : FileAnalysis.ReportType.valueOf(rtype); // defaults to Brief
94+
FileAnalysis.Report report = FileAnalysis.analyze(new File(infile), reportType);
95+
return FileAnalysis.toJsonString(report);
96+
}catch (Exception e){
97+
throw new Exception("Fail to analyze file: "+ infile);
98+
}
99+
}
100+
}
101+
79102
}
80103

0 commit comments

Comments
 (0)