Skip to content

Commit 4e56f28

Browse files
robywwrobyww
authored
DM-12113: Merge PR #471 from DM-12113-image-svc
DM-12113: refactor image services code.
2 parents e04472b + ab75d29 commit 4e56f28

36 files changed

+465
-842
lines changed

‎src/firefly/java/edu/caltech/ipac/firefly/data/FileInfo.java

Lines changed: 3 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,9 @@ public void setCookies(Map<String, String> cookies) {
128128

129129
public void setSizeInBytes(long sizeInBytes) { putAttribute(SIZE_IN_BYTES,sizeInBytes+""); }
130130

131+
public void setDesc(String desc) { putAttribute(DESC,desc); }
132+
133+
131134
@Override
132135
public boolean equals(Object o) {
133136
if (o instanceof FileInfo) {
@@ -173,21 +176,6 @@ public FileInfo copy() {
173176
return fi;
174177
}
175178

176-
public FileInfo copyWith(String... valuePairs) {
177-
FileInfo fi= copy();
178-
if (valuePairs != null && valuePairs.length > 0) {
179-
for(int i= 0; (i<valuePairs.length); i+=2) {
180-
if (valuePairs[i]!=null && valuePairs.length<i+1) {
181-
fi.putAttribute(valuePairs[i], valuePairs[i+1]);
182-
}
183-
}
184-
}
185-
return fi;
186-
}
187-
public FileInfo copyWithDesc(String desc) {
188-
return copyWith(DESC, desc);
189-
}
190-
191179
public interface FileNameResolver {
192180
String getResolvedName(String input);
193181
}

‎src/firefly/java/edu/caltech/ipac/firefly/server/persistence/QueryLSSTCatalog.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ public static void main(String[] args) {
5959
try {
6060
URLConnection uc = URLDownload.makeConnection(new URL(url));
6161
uc.setRequestProperty("Accept", "text/plain");
62-
URLDownload.getDataToFile(uc, new File("/tmp/result.json"), null);
62+
URLDownload.getDataToFile(uc, new File("/tmp/result.json"));
6363
} catch (FailedRequestException e) {
6464
throw new IOException("Request Failed", e);
6565
}
@@ -187,7 +187,7 @@ void doGetData(File oFile, List<Param> params, WorldPt wpt) throws DataAccessExc
187187
try {
188188
URLConnection uc = URLDownload.makeConnection(new URL(url));
189189
uc.setRequestProperty("Accept", "text/plain");
190-
URLDownload.getDataToFile(uc, oFile, null);
190+
URLDownload.getDataToFile(uc, oFile);
191191
} catch (FailedRequestException e) {
192192
throw new IOException("Request Failed", e);
193193
}

‎src/firefly/java/edu/caltech/ipac/firefly/server/persistence/QueryLSSTCatalogDD.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ protected File loadDataFile(TableServerRequest request) throws IOException, Data
5959
URLConnection uc = URLDownload.makeConnection(new URL(url));
6060
uc.setRequestProperty("Accept", "text/plain");
6161
File file = createFile(request, ".json");
62-
URLDownload.getDataToFile(uc, file, null);
62+
URLDownload.getDataToFile(uc, file);
6363
_log.briefDebug("SHOW COLUMNS took " + (System.currentTimeMillis() - cTime) + "ms");
6464

6565
DataGroup dg = JsonToDataGroup.parse(file);

‎src/firefly/java/edu/caltech/ipac/firefly/server/query/URLFileInfoProcessor.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,7 @@ public static FileInfo retrieveViaURL(URL url,
7878
params.addCookie(key, identityCookies.get(key));
7979
}
8080
}
81-
retval= LockingVisNetwork.retrieve(params);
81+
retval= LockingVisNetwork.retrieveURL(params);
8282
if (retval.getResponseCode()>=500) {
8383
File f= new File(retval.getInternalFilename());
8484
if (f.length()<800) {

‎src/firefly/java/edu/caltech/ipac/firefly/server/query/wise/WiseFileRetrieve.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -355,7 +355,7 @@ public FileInfo getCutoutData(ServerRequest sr) throws DataAccessException {
355355
// fd.getSugestedExternalName(),
356356
// fd.getFile().length());
357357

358-
retval= LockingVisNetwork.retrieve(url);
358+
retval= LockingVisNetwork.retrieveURL(url);
359359

360360
// remove authenticator from http requests
361361
if (USE_HTTP_AUTHENTICATOR) {
@@ -423,7 +423,7 @@ public FileInfo getNonCutoutData(ServerRequest sr) throws DataAccessException {
423423
setAuthenticator();
424424
}
425425

426-
retval= LockingVisNetwork.retrieve(url);
426+
retval= LockingVisNetwork.retrieveURL(url);
427427
_logger.info("retrieving URL:" + url.toString());
428428

429429
// remove authenticator from http requests

‎src/firefly/java/edu/caltech/ipac/firefly/server/rpc/PlotServiceImpl.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ public class PlotServiceImpl extends BaseRemoteService implements PlotService {
4040

4141

4242
public WebPlotResult getFlux(PlotState state, ImagePt inIpt) {
43-
return VisServerOps.getFlux(state,inIpt);
43+
return VisServerOps.getFluxOld(state,inIpt);
4444
}
4545

4646

‎src/firefly/java/edu/caltech/ipac/firefly/server/rpc/VisServerCommands.java

Lines changed: 20 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -85,25 +85,26 @@ public String doCommand(SrvParam sp) throws IllegalArgumentException {
8585
PlotState stateAry[]= sp.getStateAry();
8686
PlotState state= stateAry[0];
8787
ImagePt pt = sp.getRequiredImagePt("pt");
88-
89-
FileAndHeaderInfo fahAry[];
90-
List<FileAndHeaderInfo> list = new ArrayList<FileAndHeaderInfo>(state.getBands().length);
91-
for(Band b : state.getBands()) {
92-
list.add(state.getFileAndHeaderInfo(b));
93-
}
94-
95-
if (stateAry.length>1) {
96-
for(int i=1; (i<stateAry.length);i++) {
97-
list.add(stateAry[i].getFileAndHeaderInfo(Band.NO_BAND) );
98-
}
99-
}
100-
101-
102-
103-
fahAry = list.toArray(new FileAndHeaderInfo[list.size()]);
104-
105-
106-
String[] res = VisServerOps.getFileFlux(fahAry, pt);
88+
//
89+
// FileAndHeaderInfo fahAry[];
90+
// List<FileAndHeaderInfo> list = new ArrayList<FileAndHeaderInfo>(state.getBands().length);
91+
// for(Band b : state.getBands()) {
92+
// list.add(state.getFileAndHeaderInfo(b));
93+
// }
94+
//
95+
// if (stateAry.length>1) {
96+
// for(int i=1; (i<stateAry.length);i++) {
97+
// list.add(stateAry[i].getFileAndHeaderInfo(Band.NO_BAND) );
98+
// }
99+
// }
100+
//
101+
//
102+
//
103+
// fahAry = list.toArray(new FileAndHeaderInfo[list.size()]);
104+
//
105+
//
106+
// String[] res = VisServerOps.getFileFlux(fahAry, pt);
107+
String[] res= VisServerOps.getFlux(stateAry,pt);
107108

108109

109110
JSONObject obj= new JSONObject();

‎src/firefly/java/edu/caltech/ipac/firefly/server/visualize/LockingVisNetwork.java

Lines changed: 37 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,11 @@
1111
import edu.caltech.ipac.util.download.DownloadEvent;
1212
import edu.caltech.ipac.util.download.DownloadListener;
1313
import edu.caltech.ipac.util.download.FailedRequestException;
14+
import edu.caltech.ipac.util.download.NetParams;
1415
import edu.caltech.ipac.util.download.ResponseMessage;
1516
import edu.caltech.ipac.util.download.URLDownload;
1617
import edu.caltech.ipac.visualize.net.AnyUrlParams;
17-
import edu.caltech.ipac.visualize.net.VisNetwork;
18+
import edu.caltech.ipac.visualize.net.ImageServiceParams;
1819

1920
import java.io.File;
2021
import java.io.IOException;
@@ -35,25 +36,28 @@ public class LockingVisNetwork {
3536
private static final Map<BaseNetParams, Object> _activeRequest =
3637
Collections.synchronizedMap(new HashMap<BaseNetParams, Object>());
3738

39+
public static FileInfo retrieveURL(AnyUrlParams params) throws FailedRequestException {
40+
return lockingRetrieve(params, false, null);
41+
}
3842

39-
public static FileInfo retrieve(BaseNetParams params) throws FailedRequestException {
40-
return lockingRetrieve(params, false);
43+
public static FileInfo retrieve(ImageServiceParams params, ServiceCaller svcCaller) throws FailedRequestException {
44+
return lockingRetrieve(params, false, svcCaller);
4145
}
4246

43-
public static FileInfo retrieve(URL url) throws FailedRequestException {
47+
public static FileInfo retrieveURL(URL url) throws FailedRequestException {
4448
AnyUrlParams p= new AnyUrlParams(url);
4549
p.setLocalFileExtensions(Collections.singletonList(FileUtil.FITS));
46-
return retrieve(p);
50+
return retrieveURL(p);
4751
}
4852

4953

5054
//======================================================================
5155
//----------------------- Private Methods ------------------------------
5256
//======================================================================
5357

54-
private static FileInfo lockingRetrieve(BaseNetParams params, boolean unzip)
55-
throws FailedRequestException, SecurityException {
58+
private static FileInfo lockingRetrieve(BaseNetParams params, boolean unzip, ServiceCaller svcCaller) throws FailedRequestException {
5659
Objects.requireNonNull(params);
60+
confirmParamsType(params);
5761
FileInfo retval;
5862
try {
5963
Object lockKey= _activeRequest.computeIfAbsent(params, k -> new Object());
@@ -63,39 +67,47 @@ private static FileInfo lockingRetrieve(BaseNetParams params, boolean unzip)
6367
// todo: it could be generalized by passing a DownloadListener
6468
dl = new DownloadProgress(params.getStatusKey(), params.getPlotid());
6569
}
66-
FileInfo fd;
67-
if (params instanceof AnyUrlParams) {
68-
fd = retrieveURL((AnyUrlParams)params, dl);
69-
}
70-
else {
71-
fd = VisNetwork.getImage(params, dl);
72-
}
70+
FileInfo fd= (params instanceof AnyUrlParams) ?
71+
retrieveURL((AnyUrlParams)params, dl) :
72+
retrieveService((ImageServiceParams) params, dl, svcCaller);
7373

7474
if (unzip) retval= new FileInfo(unzip(fd.getFile()),fd.getExternalName(),fd.getResponseCode(), fd.getResponseCodeMsg());
7575
else retval= fd;
7676

7777
}
78+
} catch (IOException | SecurityException e) {
79+
throw ResponseMessage.simplifyNetworkCallException(e);
7880
} finally {
7981
_activeRequest.remove(params);
8082
}
8183
return retval;
8284
}
8385

84-
private static File unzip(File f) throws FailedRequestException {
86+
private static void confirmParamsType(NetParams params) throws FailedRequestException {
87+
if (!(params instanceof ImageServiceParams) && !(params instanceof AnyUrlParams) ) {
88+
throw new FailedRequestException("Unrecognized Param Type");
89+
}
90+
}
91+
92+
93+
private static File unzip(File f) throws IOException, FailedRequestException {
8594
File retval = f;
8695
if (FileUtil.getExtension(f).equalsIgnoreCase(FileUtil.GZ)) {
87-
try {
88-
if (!FileUtil.computeUnzipFileName(f).canRead()) {
89-
retval = FileUtil.gUnzipFile(f);
90-
}
91-
} catch (IOException e) {
92-
throw new FailedRequestException("Could not unzip file", "Unzipping failed", e);
96+
if (!FileUtil.computeUnzipFileName(f).canRead()) {
97+
retval = FileUtil.gUnzipFile(f);
9398
}
9499
}
95100
return retval;
96101
}
97102

98-
103+
private static FileInfo retrieveService(ImageServiceParams params, DownloadListener dl, ServiceCaller svcCaller) throws IOException, FailedRequestException {
104+
File f= CacheHelper.getFile(params);
105+
if (f == null) {
106+
f= svcCaller.retrieve(params,CacheHelper.makeFitsFile(params));
107+
CacheHelper.putFile(params,f);
108+
}
109+
return new FileInfo(f);
110+
}
99111

100112
//======================================
101113
//======================================
@@ -124,6 +136,9 @@ private static FileInfo retrieveURL(AnyUrlParams params, DownloadListener dl) th
124136
}
125137
}
126138

139+
public interface ServiceCaller {
140+
File retrieve(ImageServiceParams p, File suggestedFile) throws IOException, FailedRequestException;
141+
}
127142

128143

129144
private static class DownloadProgress implements DownloadListener {

‎src/firefly/java/edu/caltech/ipac/firefly/server/visualize/VisServerOps.java

Lines changed: 69 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,7 @@
7878
import java.io.InputStreamReader;
7979
import java.io.RandomAccessFile;
8080
import java.util.ArrayList;
81+
import java.util.Arrays;
8182
import java.util.HashMap;
8283
import java.util.List;
8384
import java.util.Map;
@@ -241,6 +242,68 @@ public static double[] getBeta(PlotState state) {
241242
}
242243
}
243244

245+
246+
private static boolean isDirectFluxAccessAvailable(PlotState state) {
247+
//todo: make this test more sophisticated
248+
249+
for(Band b : state.getBands()) {
250+
if (state.getWorkingFitsFileStr(b).endsWith("gz")) {
251+
return false;
252+
}
253+
}
254+
return true;
255+
256+
257+
}
258+
259+
public static String[] getFlux(PlotState stateAry[], ImagePt ipt) {
260+
PlotState state= stateAry[0];
261+
FileAndHeaderInfo fahAry[];
262+
List<String> fluxList= new ArrayList<>();
263+
264+
265+
// 1. handle primary plot
266+
if (isDirectFluxAccessAvailable(state)) {
267+
List<FileAndHeaderInfo> list = new ArrayList<>();
268+
for(Band b : state.getBands()) {
269+
list.add(state.getFileAndHeaderInfo(b));
270+
}
271+
fahAry = list.toArray(new FileAndHeaderInfo[list.size()]);
272+
String[] res = VisServerOps.getFileFlux(fahAry, ipt);
273+
fluxList.addAll(Arrays.asList(res));
274+
}
275+
else {
276+
for(Band b : state.getBands()) {
277+
try {
278+
fluxList.add(getFluxValueInMemory(state, b, ipt)+"");
279+
} catch (IOException e) {
280+
fluxList.add("NaN");
281+
}
282+
}
283+
}
284+
285+
// 2. handle overlays
286+
if (stateAry.length>1) {
287+
for(int i=1; (i<stateAry.length);i++) {
288+
if (isDirectFluxAccessAvailable(stateAry[i])) {
289+
FileAndHeaderInfo fah[]= new FileAndHeaderInfo[] {stateAry[i].getFileAndHeaderInfo(Band.NO_BAND)};
290+
String[] res = VisServerOps.getFileFlux(fah, ipt);
291+
fluxList.add(res[0]);
292+
}
293+
else {
294+
try {
295+
fluxList.add(getFluxValueInMemory(stateAry[i],Band.NO_BAND, ipt)+"");
296+
} catch (IOException e) {
297+
fluxList.add("NaN");
298+
}
299+
}
300+
}
301+
}
302+
303+
// 3. return all the gathered fluxes
304+
return fluxList.toArray(new String[fluxList.size()]);
305+
}
306+
244307
public static String[] getFileFlux(FileAndHeaderInfo fileAndHeader[], ImagePt ipt) {
245308
try {
246309
String retval[] = new String[fileAndHeader.length];
@@ -256,9 +319,9 @@ public static String[] getFileFlux(FileAndHeaderInfo fileAndHeader[], ImagePt ip
256319
}
257320

258321

259-
private static double getFluxValue(PlotState state,
260-
Band band,
261-
ImagePt ipt) throws IOException {
322+
private static double getFluxValueInMemory(PlotState state,
323+
Band band,
324+
ImagePt ipt) throws IOException {
262325
if (state == null) throw new IllegalArgumentException("state must not be null");
263326
double retval;
264327
if (!CtxControl.isImagePlotAvailable(state.getContextString())) { // work directly on the file
@@ -280,12 +343,13 @@ private static double getFluxValue(PlotState state,
280343
return retval;
281344
}
282345

283-
public static WebPlotResult getFlux(PlotState state, ImagePt ipt) {
346+
@Deprecated
347+
public static WebPlotResult getFluxOld(PlotState state, ImagePt ipt) {
284348
try {
285349
Band bands[] = state.getBands();
286350
double fluxes[] = new double[bands.length];
287351
for (int i = 0; (i < bands.length); i++) {
288-
fluxes[i] = getFluxValue(state, bands[i], ipt);
352+
fluxes[i] = getFluxValueInMemory(state, bands[i], ipt);
289353
}
290354
WebPlotResult retval = new WebPlotResult(state.getContextString());
291355
retval.putResult(WebPlotResult.FLUX_VALUE, new DataEntry.DoubleArray(fluxes));

‎src/firefly/java/edu/caltech/ipac/firefly/server/visualize/WebPlotReader.java

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -162,7 +162,9 @@ public static FileReadInfo[] readOneFits(FileInfo fd, Band band, WebPlotRequest
162162
List<RelatedData> rd= null;
163163
if (relatedDataList!=null || fd.getRelatedData()!=null) {
164164
rd= new ArrayList<>();
165-
if (relatedDataList!=null) rd.addAll(relatedDataList.get(i));
165+
if (relatedDataList!=null && i<relatedDataList.size() && relatedDataList.get(i)!=null) {
166+
rd.addAll(relatedDataList.get(i));
167+
}
166168
if (fd.getRelatedData()!=null) rd.addAll(fd.getRelatedData());
167169
}
168170
retval[i]= new FileReadInfo(originalFile, frAry[i], band, i, fd.getDesc(), uploadedName, rd, null);

‎src/firefly/java/edu/caltech/ipac/firefly/server/visualize/imageretrieve/AllSkyRetriever.java

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -52,13 +52,14 @@ public FileInfo getFile(WebPlotRequest request) throws FailedRequestException, G
5252
URL url= this.getClass().getClassLoader().getResource(urlStr);
5353
AnyUrlParams p= new AnyUrlParams(url);
5454
p.setLocalFileExtensions(Collections.singletonList(FileUtil.FITS));
55-
fitsFileInfo= LockingVisNetwork.retrieve(p);
55+
fitsFileInfo= LockingVisNetwork.retrieveURL(p);
5656
} catch (FailedRequestException e) {
5757
throw e;
5858
} catch (Exception e) {
5959
throw new FailedRequestException("No data",null,e);
6060
}
61-
return fitsFileInfo.copyWithDesc(RESERVED_IMAGES[DEFAULT_ALLSKY].getDescription());
61+
fitsFileInfo.setDesc(RESERVED_IMAGES[DEFAULT_ALLSKY].getDescription());
62+
return fitsFileInfo;
6263
}
6364

6465
//===================================================================

0 commit comments

Comments
 (0)