Skip to content

Commit f9f36a3

Browse files
author
Cindy Wang
committed
Merge branch 'DM-7725-FileUpload' into dev
2 parents b128527 + 1dc4f50 commit f9f36a3

20 files changed

+1688
-90
lines changed

‎src/firefly/html/firefly.html

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
{label:'Images', action:'ImageSelectDropDownCmd'},
1818
{label:'Catalogs', action:'IrsaCatalogDropDown'},
1919
{label:'Charts', action:'ChartSelectDropDownCmd'},
20+
{label:'Upload', action: 'FileUploadDropDownCmd'}
2021
],
2122
options : {
2223
MenuItemKeys: {maskOverlay:true},

‎src/firefly/java/edu/caltech/ipac/firefly/server/servlets/AnyFileUpload.java

Lines changed: 131 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -3,19 +3,23 @@
33
*/
44
package edu.caltech.ipac.firefly.server.servlets;
55

6+
import edu.caltech.ipac.firefly.data.TableServerRequest;
67
import edu.caltech.ipac.firefly.server.Counters;
78
import edu.caltech.ipac.firefly.server.ServerContext;
89
import edu.caltech.ipac.firefly.server.cache.UserCache;
910
import edu.caltech.ipac.firefly.server.util.Logger;
1011
import edu.caltech.ipac.firefly.server.util.StopWatch;
1112
import edu.caltech.ipac.firefly.server.util.ipactable.DataGroupReader;
1213
import edu.caltech.ipac.firefly.server.util.ipactable.DataGroupWriter;
14+
import edu.caltech.ipac.firefly.server.util.ipactable.JsonTableUtil;
1315
import edu.caltech.ipac.firefly.server.util.multipart.UploadFileInfo;
1416
import edu.caltech.ipac.util.DataGroup;
1517
import edu.caltech.ipac.util.FileUtil;
1618
import edu.caltech.ipac.util.IpacTableUtil;
1719
import edu.caltech.ipac.util.StringUtils;
1820
import edu.caltech.ipac.util.cache.StringKey;
21+
import edu.caltech.ipac.util.download.URLDownload;
22+
import edu.caltech.ipac.firefly.data.FileInfo;
1923
import org.apache.commons.fileupload.FileItemIterator;
2024
import org.apache.commons.fileupload.FileItemStream;
2125
import org.apache.commons.fileupload.servlet.ServletFileUpload;
@@ -24,6 +28,11 @@
2428
import javax.servlet.http.HttpServletResponse;
2529
import java.io.*;
2630
import java.util.HashMap;
31+
import org.json.simple.JSONObject;
32+
import java.util.Iterator;
33+
import java.util.Map.Entry;
34+
import java.util.Map;
35+
import java.net.URL;
2736

2837
/**
2938
* Date: Feb 16, 2011
@@ -37,7 +46,7 @@ public class AnyFileUpload extends BaseHttpServlet {
3746
public static final String PRELOAD_PARAM = "preload";
3847
public static final String FILE_TYPE = "type";
3948
public static final String CACHE_KEY = "cacheKey";
40-
private enum FileType {FITS, TABLE, REGION, UNKNOWN}
49+
private enum FileType {FITS, TABLE, REGION, XML, UNKNOWN}
4150

4251
protected void processRequest(HttpServletRequest req, HttpServletResponse res) throws Exception {
4352
doFileUpload(req, res);
@@ -53,6 +62,8 @@ public static void doFileUpload(HttpServletRequest req, HttpServletResponse res)
5362
ServletFileUpload upload = new ServletFileUpload();
5463
FileItemIterator iter = upload.getItemIterator(req);
5564
FileItemStream file = null;
65+
String url = null;
66+
5667
HashMap<String, String> params = new HashMap<>();
5768

5869
while (iter.hasNext()) {
@@ -65,45 +76,123 @@ public static void doFileUpload(HttpServletRequest req, HttpServletResponse res)
6576
String name = item.getFieldName();
6677
String value = FileUtil.readFile(item.openStream());
6778
params.put(name, value);
79+
if (name.equals("URL")) {
80+
url = new String(value);
81+
}
6882
}
6983
}
7084

7185
String dest = getParam(DEST_PARAM, params, req);
7286
String preload = getParam(PRELOAD_PARAM, params, req);
7387
String overrideCacheKey= getParam(CACHE_KEY, params, req);
7488
String fileType= getParam(FILE_TYPE, params, req);
89+
String fileAnalysis = getParam("fileAnalysis", params, req);
7590

76-
if (file != null) {
77-
String fileName = file.getName();
78-
InputStream inStream = new BufferedInputStream(file.openStream(), IpacTableUtil.FILE_IO_BUFFER_SIZE);
79-
String ext = resolveExt(fileName);
80-
FileType fType = resolveType(fileType, ext, file.getContentType());
81-
File destDir = resolveDestDir(dest, fType);
82-
// boolean doPreload = resolvePreload(preload, fType); // we are no longer pre-loading fits files. performance gain was not significant.
91+
if (file != null || url != null) {
92+
UploadFileInfo fi;
93+
String ext;
94+
File destDir;
95+
FileType fType;
96+
File uf;
97+
String fileName;
98+
FileInfo urlDownloadInfo = null;
99+
100+
if (file != null) {
101+
fileName = file.getName();
102+
} else {
103+
int idx = url.lastIndexOf('/');
104+
fileName = (idx >= 0) ? url.substring(idx + 1) : new String(url);
105+
}
106+
ext = resolveExt(fileName);
107+
fType = resolveType(fileType, ext, (file != null ? file.getContentType() : null));
108+
destDir = resolveDestDir(dest, fType);
109+
uf = File.createTempFile("upload_", ext, destDir); // other parts of system depend on file name starting with "upload_"
110+
if (file != null) {
111+
InputStream inStream = new BufferedInputStream(file.openStream(), IpacTableUtil.FILE_IO_BUFFER_SIZE);
112+
FileUtil.writeToFile(inStream, uf);
113+
} else {
114+
urlDownloadInfo = URLDownload.getDataToFile(new URL(url), uf);
115+
}
83116

84-
File uf = File.createTempFile("upload_", ext, destDir); // other parts of system depend on file name starting with "upload_"
85117
String rPathInfo = ServerContext.replaceWithPrefix(uf);
118+
fi = new UploadFileInfo(rPathInfo, uf, fileName, (file != null ? file.getContentType() : null));
119+
120+
JSONObject analysisResult = null;
121+
if (fileAnalysis != null && fileAnalysis.equalsIgnoreCase("true")) {
122+
if (url != null && urlDownloadInfo != null &&
123+
!(urlDownloadInfo.getResponseCodeMsg().equals("OK"))) {
124+
throw new Exception("invalid upload from URL: " + urlDownloadInfo.getResponseCodeMsg());
125+
}
126+
127+
analysisResult = createAnalysisResult(fi);
128+
}
86129

87-
UploadFileInfo fi= new UploadFileInfo(rPathInfo,uf,fileName,file.getContentType());
88-
FileUtil.writeToFile(inStream, uf);
89130
if (fType == FileType.TABLE) {
90131
uf = File.createTempFile("upload_", ".tbl", destDir); // cleaned ipac file.
91132
rPathInfo = ServerContext.replaceWithPrefix(uf);
92133
DataGroup dg = DataGroupReader.readAnyFormat(fi.getFile(), 0);
93134
DataGroupWriter.write(new DataGroupWriter.IpacTableHandler(uf, dg));
94-
fi= new UploadFileInfo(rPathInfo,uf,fileName,file.getContentType());
135+
fi = new UploadFileInfo(rPathInfo, uf, fileName, (file != null ? file.getContentType() : null));
95136
}
96-
String fileCacheKey= overrideCacheKey!=null ? overrideCacheKey : rPathInfo;
137+
String fileCacheKey = overrideCacheKey != null ? overrideCacheKey : rPathInfo;
97138
UserCache.getInstance().put(new StringKey(fileCacheKey), fi);
98139

140+
if (analysisResult != null) {
141+
String resultS = analysisResult.toJSONString();
142+
String fFormat = (String)analysisResult.get("fileFormat");
143+
144+
if (!StringUtils.isEmpty(resultS)) {
145+
fileCacheKey = fileCacheKey + "::" + fFormat + "::" + resultS;
146+
}
147+
}
148+
99149
sendReturnMsg(res, 200, null, fileCacheKey);
150+
100151
Counters.getInstance().increment(Counters.Category.Upload, fi.getContentType());
101152

102153
}
103154

104155
StopWatch.getInstance().printLog("Upload File");
105156
}
106157

158+
159+
private static JSONObject createAnalysisResult(UploadFileInfo fi) throws Exception {
160+
JSONObject analysisModel = null;
161+
DataGroupReader.Format fileFormat = null;
162+
DataGroup dgAnalysis = null;
163+
String analysisSummary = "";
164+
165+
JSONObject analysisResult = new JSONObject();
166+
File f = fi.getFile();
167+
long size = f.length();
168+
169+
fileFormat = DataGroupReader.guessFormat(f);
170+
dgAnalysis = DataGroupReader.readAnyFormatHeader(f, fileFormat);
171+
if (dgAnalysis != null) {
172+
analysisSummary = dgAnalysis.getTitle();
173+
if (!analysisSummary.contains("invalid")) {
174+
analysisModel = toJsonAnalysisTableModel(dgAnalysis, fileFormat, size);
175+
}
176+
} else {
177+
analysisSummary = "invalid " + fileFormat.toString() + " file";
178+
}
179+
180+
if (analysisSummary.startsWith("invalid")) {
181+
throw new Exception(analysisSummary);
182+
}
183+
184+
analysisResult.put("status", 200);
185+
analysisResult.put("message", "");
186+
analysisResult.put("fileCacheKey", fi.getPname());
187+
analysisResult.put("analysisSummary", analysisSummary);
188+
analysisResult.put("fileFormat", fileFormat.toString());
189+
if (analysisModel != null) {
190+
analysisResult.put("analysisModel", analysisModel);
191+
}
192+
193+
return analysisResult;
194+
}
195+
107196
private static String getParam(String key, HashMap<String, String> params, HttpServletRequest req) {
108197
if (key == null) return null;
109198
if (params.containsKey(key)) {
@@ -148,8 +237,10 @@ private static FileType resolveType(String fileType, String fileExtension, Strin
148237
ftype = FileType.FITS;
149238
} else if (fileExtension.matches("\\.tbl|\\.csv|\\.tsv")) {
150239
ftype = FileType.TABLE;
151-
} else if (fileExtension.matches(".reg")) {
240+
} else if (fileExtension.matches("\\.reg")) {
152241
ftype = FileType.REGION;
242+
} else if (fileExtension.matches("\\.xml|\\.vot")) {
243+
ftype = FileType.XML;
153244
}
154245
} else {
155246
// guess using contentType
@@ -171,6 +262,32 @@ private boolean resolvePreload(String preload, FileType fileType) {
171262
}
172263
}
173264

265+
private static JSONObject toJsonAnalysisTableModel(DataGroup dg, DataGroupReader.Format ff, long size ) {
266+
JSONObject tableModel = new JSONObject();
267+
JSONObject tableData = JsonTableUtil.toJsonTableData(dg, null);
268+
String tblId = "UPLOAD_ANALYSIS";
269+
270+
tableModel.put("tableData", tableData);
271+
tableModel.put("tbl_id", tblId);
272+
tableModel.put("title", dg.getTitle());
273+
tableModel.put("totalRows", dg.values().size());
274+
tableModel.put("fileFormat", ff.toString());
275+
tableModel.put("highlightedRow", 0);
276+
tableModel.put("size", size);
277+
278+
JSONObject tableMeta = new JSONObject();
279+
Iterator<Entry<String, DataGroup.Attribute>> attributes = dg.getAttributes().entrySet().iterator();
280+
281+
while( attributes.hasNext() ) {
282+
Map.Entry<String, DataGroup.Attribute> entry = attributes.next();
283+
DataGroup.Attribute att = entry.getValue();
284+
285+
tableMeta.put(att.getKey(), att.getValue());
286+
}
287+
288+
tableModel.put("tableMeta", tableMeta);
289+
return tableModel;
290+
}
174291

175292
}
176293

‎src/firefly/java/edu/caltech/ipac/firefly/server/util/ipactable/DataGroupReader.java

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,24 @@ public static DataGroup readAnyFormat(File inf, int tableIndex) throws IOExcepti
6363
}
6464
}
6565

66+
public static DataGroup readAnyFormatHeader(File inf, Format ff) throws IOException {
67+
Format format = ff == null ? guessFormat(inf) : ff;
68+
DataGroup dg = null;
69+
70+
if (format == Format.VO_TABLE) {
71+
dg = VoTableUtil.voHeaderToDataGroup(inf.getAbsolutePath());
72+
} else if (format == Format.FITS) {
73+
dg = FitsHDUUtil.fitsHeaderToDataGroup(inf.getAbsolutePath());
74+
} else if (format == Format.CSV || format == Format.TSV || format == Format.IPACTABLE || format == Format.JSON) {
75+
String A = (format == Format.IPACTABLE) ? "IPAC Table" : format.toString();
76+
String title = String.format("%s", A);
77+
dg = new DataGroup(title, new ArrayList<DataType>());
78+
} else {
79+
dg = new DataGroup("invalid file format", new ArrayList<DataType>());
80+
}
81+
return dg;
82+
}
83+
6684
public static DataGroup read(File inf, String... onlyColumns) throws IOException {
6785
return read(inf, false, onlyColumns);
6886

@@ -182,7 +200,10 @@ public static Format guessFormat(File inf) throws IOException {
182200
String line = reader.readLine();
183201
if (line.startsWith("{")) {
184202
return Format.JSON;
203+
} else if (line.startsWith("SIMPLE = ")) {
204+
return Format.FITS;
185205
}
206+
186207
int[][] counts = new int[readAhead][2];
187208
int csvIdx = 0, tsvIdx = 1;
188209
while (line != null && row < readAhead) {
@@ -195,6 +216,8 @@ public static Format guessFormat(File inf) throws IOException {
195216
//EQUINOX: xxx
196217
//NAME-RESOLVER: xxx
197218
return Format.FIXEDTARGETS;
219+
} else if (line.startsWith("<VOTABLE")) {
220+
return Format.VO_TABLE;
198221
}
199222

200223
counts[row][csvIdx] = CSVFormat.DEFAULT.parse(new StringReader(line)).iterator().next().size();
@@ -310,8 +333,6 @@ private static DataGroup doRead(BufferedReader bufferedReader, TableDef tableDef
310333

311334
}
312335

313-
314-
315336
//====================================================================
316337
//
317338
//====================================================================

‎src/firefly/java/edu/caltech/ipac/firefly/server/util/ipactable/JsonTableUtil.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,8 @@ private static TableDef mergeAttributes(TableDef tblDef, DataGroup data) {
122122
*/
123123
public static JSONObject toJsonTableData(DataGroup data, TableDef tableDef) {
124124

125+
tableDef = mergeAttributes(tableDef, data);
126+
125127
// set display format if exists. this modifies DataType directly because it assumes it will no longer be used.
126128
// if that is not the case, DataType will have to be cloned.
127129
// also set flag to recalculate the max width of column's data
@@ -183,6 +185,8 @@ public static JSONObject toJsonTableMeta(TableDef tableDef) {
183185

184186
private static List<JSONObject> toJsonTableColumn(DataGroup dataGroup, TableDef tableDef) {
185187

188+
tableDef = mergeAttributes(tableDef, dataGroup);
189+
186190
DataType[] dataTypes = tableDef.getCols().size() > 0 ? tableDef.getCols().toArray(new DataType[0]) : dataGroup.getDataDefinitions();
187191

188192
ArrayList<JSONObject> cols = new ArrayList<JSONObject>();

0 commit comments

Comments
 (0)