Skip to content

DM-8288: Add support to show LSST SDSS images in triview #231

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Nov 18, 2016
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion src/firefly/html/lsst-api-triview.html
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,10 @@
MenuItemKeys: {maskOverlay:true},
imageTabs: [ 'fileUpload', 'url', '2mass', 'wise', 'sdss', 'msx', 'dss', 'iras' ],
irsaCatalogFilter: 'lsstFilter',
catalogSpacialOp: 'polygonWhenPlotExist'
catalogSpacialOp: 'polygonWhenPlotExist',
properties : {
daxUrl: 'http://lsst-qserv-dax01.ncsa.illinois.edu:5000'
}
}
}
};
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,11 +30,5 @@ public class MetaConst {


public static final String DEFAULT_COLOR="DEFAULT_COLOR";

//11/15/16 Added by LZ
public static final String LSST_SDSS_COADD="LSST_COADD_TABLE";
public static final String LSST_SDSS_SINGLE_EXPOSURE="LSST_SINGLE_EXPORSURE_TABLE";


}

Original file line number Diff line number Diff line change
Expand Up @@ -4,32 +4,40 @@
import edu.caltech.ipac.firefly.data.CatalogRequest;
import edu.caltech.ipac.firefly.data.ServerRequest;
import edu.caltech.ipac.firefly.data.TableServerRequest;
import edu.caltech.ipac.firefly.data.table.MetaConst;
import edu.caltech.ipac.firefly.data.table.TableMeta;
import edu.caltech.ipac.firefly.server.query.*;
import edu.caltech.ipac.firefly.server.query.DataAccessException;
import edu.caltech.ipac.firefly.server.query.IpacTablePartProcessor;
import edu.caltech.ipac.firefly.server.query.SearchManager;
import edu.caltech.ipac.firefly.server.query.SearchProcessorImpl;
import edu.caltech.ipac.firefly.server.util.Logger;
import edu.caltech.ipac.firefly.server.util.ipactable.DataGroupPart;
import edu.caltech.ipac.firefly.server.util.ipactable.DataGroupWriter;
import edu.caltech.ipac.firefly.util.DataSetParser;
import edu.caltech.ipac.firefly.visualize.VisUtil;
import edu.caltech.ipac.util.*;
import edu.caltech.ipac.util.AppProperties;
import edu.caltech.ipac.util.DataGroup;
import edu.caltech.ipac.util.DataObject;
import edu.caltech.ipac.util.DataType;
import edu.caltech.ipac.util.StringUtils;
import edu.caltech.ipac.util.download.FileData;
import edu.caltech.ipac.util.download.URLDownload;
import edu.caltech.ipac.visualize.plot.CoordinateSys;
import edu.caltech.ipac.visualize.plot.WorldPt;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;
import org.json.simple.parser.ParseException;
import java.io.*;

import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.net.URL;
import java.net.URLEncoder;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.lang.Exception;

import edu.caltech.ipac.visualize.plot.CoordinateSys;
import edu.caltech.ipac.firefly.data.table.MetaConst;


/**
Expand Down Expand Up @@ -552,24 +560,24 @@ public void prepareTableMeta(TableMeta meta, List<DataType> columns, ServerReque

super.prepareTableMeta(meta, columns, request);
String catTable = request.getParam("table_name");
String tUp=catTable.toUpperCase();

String RA = getRA(catTable);
String DEC = getDEC(catTable);
TableMeta.LonLatColumns llc = new TableMeta.LonLatColumns(RA, DEC, CoordinateSys.EQ_J2000);
meta.setCenterCoordColumns(llc);
meta.setAttribute(MetaConst.CATALOG_OVERLAY_TYPE, "LSST");
meta.setCenterCoordColumns(llc);
String tableName = request.getParam("table_name");
switch (tableName.toUpperCase()){
case "SCIENCE_CCD_EXPOSURE":
meta.setAttribute(MetaConst.LSST_SDSS_SINGLE_EXPOSURE, "LSST_CCD");
break;
case "DEEPCOADD":
meta.setAttribute(MetaConst.LSST_SDSS_COADD, "LSST_COADD");
break;
default:
meta.setAttribute(MetaConst.CATALOG_OVERLAY_TYPE, "LSST");
}

if (tUp.equals("SCIENCE_CCD_EXPOSURE") || tUp.equals("DEEPCOADD")) {
TableMeta.LonLatColumns c1= new TableMeta.LonLatColumns("corner1Ra", "corner1Decl", CoordinateSys.EQ_J2000);
TableMeta.LonLatColumns c2= new TableMeta.LonLatColumns("corner2Ra", "corner2Decl", CoordinateSys.EQ_J2000);
TableMeta.LonLatColumns c3= new TableMeta.LonLatColumns("corner3Ra", "corner3Decl", CoordinateSys.EQ_J2000);
TableMeta.LonLatColumns c4= new TableMeta.LonLatColumns("corner4Ra", "corner4Decl", CoordinateSys.EQ_J2000);
meta.setCorners(c1, c2, c3, c4);
meta.setAttribute(MetaConst.DATASET_CONVERTER, "lsst_sdss");
}
else {
meta.setAttribute(MetaConst.CATALOG_OVERLAY_TYPE, "LSST");
}
super.prepareTableMeta(meta, columns, request);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ public FileData getFile(WebPlotRequest request) throws FailedRequestException, G
params.setLoginName(ro.getUserInfo().getLoginName());
params.setSecurityCookie(ro.getRequestAgent().getAuthKey());
}
params.setCheckForNewer(true);
params.setCheckForNewer(request.getUrlCheckForNewer());
params.setLocalFileExtensions(Arrays.asList(FileUtil.FITS, FileUtil.GZ)); //assuming WebPlotRequest ONLY expect FITS or GZ file.
params.setMaxSizeToDownload(VisContext.FITS_MAX_SIZE);
if (request.getUserDesc() != null) params.setDesc(request.getUserDesc()); // set file description
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,7 @@ public enum GridOnStatus {FALSE,TRUE,TRUE_LABELS_FALSE}
public static final String HAS_MAX_ZOOM_LEVEL = "HasMaxZoomLevel";
public static final String THUMBNAIL_SIZE = "thumbnailSize";
public static final String PIPELINE_ORDER = "pipelineOrder"; // todo: convert, doc, add to allKeys
public static final String URL_CHECK_FOR_NEWER = "urlCheckForNewer"; // todo: convert, doc, add to allKeys

public static final String MASK_BITS= "MaskBits";
public static final String PLOT_AS_MASK= "PlotAsMask";
Expand Down Expand Up @@ -884,6 +885,10 @@ public String getURL() {
return getSafeParam(URL);
}

public void setUrlCheckForNewer(boolean check) { setSafeParam(URL_CHECK_FOR_NEWER,check+"");}

public boolean getUrlCheckForNewer() { return getBooleanParam(URL_CHECK_FOR_NEWER);}

public void setServiceType(ServiceType service) {
setParam(SERVICE, service.toString());
}
Expand Down
45 changes: 31 additions & 14 deletions src/firefly/java/edu/caltech/ipac/util/download/URLDownload.java
Original file line number Diff line number Diff line change
Expand Up @@ -364,13 +364,15 @@ public static FileData getDataToFile(URLConnection conn,
FileData outFileData;
FileData retval;
int responseCode= 200;
Map<String,List<String>> sendHeaders= null;
try {
if (conn instanceof HttpURLConnection) {
HttpURLConnection httpConn= (HttpURLConnection) conn;
if (postData!=null) {
pushPostData(httpConn,postData);
}
if (uncompress) httpConn.setRequestProperty("Accept-Encoding", "gzip, deflate");
sendHeaders= httpConn.getRequestProperties();
if (onlyIfModified) {
outFileData = checkAlreadyDownloaded(httpConn, outfile);
if (outFileData != null) return outFileData;
Expand All @@ -385,7 +387,7 @@ public static FileData getDataToFile(URLConnection conn,
//------
OutputStream out;
DataInputStream in;
logHeader(postData, conn);
logHeader(postData, conn, sendHeaders);
if (conn instanceof HttpURLConnection) responseCode= ((HttpURLConnection)conn).getResponseCode();
validFileSize(conn, maxFileSize);
File f = useSuggestedFilename ? makeFile(conn, outfile) : outfile;
Expand Down Expand Up @@ -658,18 +660,16 @@ public static void logError(URL url, String postData, Exception e) {
}


public static void logHeader(String postData, URLConnection conn) {
public static void logHeader(String postData, URLConnection conn, Map<String,List<String>> sendHeaders) {
StringBuffer workBuff;
try {
Set hSet= null;
String outStr[];
int extra = postData == null ? 3 : 4;
List<String> outStr= new ArrayList<>(40);
int extra = 30;
if (conn instanceof HttpURLConnection && ((HttpURLConnection)conn).getResponseCode()==-1) {
outStr = new String[extra+1];
}
else {
hSet = conn.getHeaderFields().entrySet();
outStr = new String[hSet.size() + extra];
}
Map.Entry entry;
List values;
Expand All @@ -678,18 +678,35 @@ public static void logHeader(String postData, URLConnection conn) {
String key;
Iterator k;
if (conn.getURL() != null) {
outStr[i++] = "----------Sending";
outStr[i++] = conn.getURL().toString();
outStr.add("----------Sending");
outStr.add( conn.getURL().toString());
if (sendHeaders!=null) {
for(Map.Entry<String,List<String>> se: sendHeaders.entrySet()) {
workBuff = new StringBuffer(100);
if (se.getKey() == null) key = "<none>";
else key= se.getKey();
workBuff.append(StringUtils.pad(20,key));
workBuff.append(": ");
if (key.equalsIgnoreCase("cookie")) {
workBuff.append("not shown");
}
else {
workBuff.append(se.getValue());
}
outStr.add(workBuff.toString());
}

}
}
if (postData != null) {
outStr[i++] = StringUtils.pad(20,"Post Data ") + ": " + postData;
outStr.add(StringUtils.pad(20,"Post Data ") + ": " + postData);
}
if (conn instanceof HttpURLConnection) {
int responseCode= ((HttpURLConnection)conn).getResponseCode();
outStr[i++] = "----------Received Headers, response status code: " + responseCode;
outStr.add("----------Received Headers, response status code: " + responseCode);
}
else {
outStr[i++] = "----------Received Headers";
outStr.add("----------Received Headers");
}
if (hSet!=null) {
for (Iterator j = hSet.iterator(); (j.hasNext()); ) {
Expand All @@ -704,11 +721,11 @@ public static void logHeader(String postData, URLConnection conn) {
if (m > 0) workBuff.append("; ");
workBuff.append(k.next().toString());
}
outStr[i++] = workBuff.toString();
outStr.add(workBuff.toString());
}
}
else {
outStr[i++] = "No headers or status received, invalid http response, using work around";
outStr.add("No headers or status received, invalid http response, using work around");
}
ClientLog.message(outStr);
} catch (Exception e) {
Expand All @@ -717,7 +734,7 @@ public static void logHeader(String postData, URLConnection conn) {
}

public static void logHeader(URLConnection conn) {
logHeader(null, conn);
logHeader(null, conn, null);
}

public static void logCompletedDownload(long size) {
Expand Down
2 changes: 1 addition & 1 deletion src/firefly/js/data/ServerRequest.js
Original file line number Diff line number Diff line change
Expand Up @@ -160,7 +160,7 @@ export class ServerRequest {
static parse(str,req) {
if (!str) return null;
words(str,PARAM_SEP).forEach((p) => {
var outParam= words(p,KW_VAL_SEP);
var outParam= p.split(/=(.+)?/,2);
if (outParam.length===2) {
var newParam= {name : outParam[0], value:outParam[1]};
if (!req.addPredefinedAttrib(newParam)) {
Expand Down
15 changes: 15 additions & 0 deletions src/firefly/js/metaConvert/ConverterFactory.js
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
import {get, has} from 'lodash';
import {makeWisePlotRequest} from './WiseRequestList.js';
import {make2MassPlotRequest} from './TwoMassRequestList.js';
import {makeLsstSdssPlotRequest} from './LsstSdssRequestList.js';
import {WebPlotRequest, TitleOptions} from '../visualize/WebPlotRequest.js';
import {ZoomType} from '../visualize/ZoomType.js';
import {Band} from '../visualize/Band';
Expand Down Expand Up @@ -51,6 +52,20 @@ export const converters = {
K : {color : Band.BLUE, title: 'K'}
}
},
'lsst_sdss' : {
threeColor : true,
hasRelatedBands : true,
canGrid : true,
maxPlots : 12,
makeRequest : makeLsstSdssPlotRequest,
threeColorBands : {
u : {color : null, title: 'u'},
g : {color : Band.RED, title: 'g'},
r : {color : Band.GREEN, title: 'r'},
i : {color : null, title: 'i'},
z : {color : Band.BLUE, title: 'z'}
}
},
'UNKNOWN' : {
threeColor : false,
hasRelatedBands : false,
Expand Down
90 changes: 90 additions & 0 deletions src/firefly/js/metaConvert/LsstSdssRequestList.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
/*
* License information at https://github.com/Caltech-IPAC/firefly/blob/master/License.txt
*/

/*
* License information at https://github.com/Caltech-IPAC/firefly/blob/master/License.txt
*/


import {padStart} from 'lodash';
import {getCellValue} from '../tables/TableUtil.js';
import {RangeValues,STRETCH_LINEAR,SIGMA} from '../visualize/RangeValues.js';
import {ZoomType} from '../visualize/ZoomType.js';
import {WebPlotRequest, TitleOptions} from '../visualize/WebPlotRequest.js';


const DAX_URL= 'http://lsst-qserv-dax01.ncsa.illinois.edu:5000';


function makeCcdReqBuilder(table, rowIdx) {
const rangeValues= RangeValues.makeRV({which:SIGMA, lowerValue:-2, upperValue:10, algorithm:STRETCH_LINEAR});
const run= getCellValue(table, rowIdx, 'run');
const field= padStart(getCellValue(table, rowIdx, 'field'), 4, '0');
const camcol= getCellValue(table, rowIdx, 'camcol');
const baseUrl= `${DAX_URL}/image/v0/calexp/ids`;

return (plotId, title, filterName) => {

// id is run + filterId + camcol + field
// const objId= getCellValue(table, rowIdx, 'scienceCcdExposureId');
// const objId= `${run}${filterId}${camcol}${field}`;
// const url= `http://lsst-qserv-dax01.ncsa.illinois.edu:5000/image/v0/calexp/id?id=${objId}`;
const url= `${baseUrl}?run=${run}&camcol=${camcol}&field=${field}&filter=${filterName}`;
const r= WebPlotRequest.makeURLPlotRequest(url);
r.setTitleOptions(TitleOptions.NONE);
r.setTitle(title);
r.setPlotId(plotId);
r.setMultiImageIdx(0);
r.setPreferenceColorKey('lsst-sdss-color-pref');
r.setZoomType(ZoomType.TO_WIDTH);
r.setInitialRangeValues(rangeValues);
return r;
};
}

/**
* make a list of plot request for wise. This function works with ConverterFactory.
* @param table
* @param row
* @param includeSingle
* @param includeStandard
* @param threeColorOps
* @return {{}}
*/
export function makeLsstSdssPlotRequest(table, row, includeSingle, includeStandard, threeColorOps) {

const bandMap= {u:'0', g:'1',r:'2',i:'3', z:'4'};
const retval= {};
var builder;
if (getCellValue(table, row, 'scienceCcdExposureId')) {
builder= makeCcdReqBuilder(table,row);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

please check if line 63 duplicates line 61?

}
else {
return {single:null, standard:[]};
}
const filterId= Number(getCellValue(table, row, 'filterId'));
const filterName= getCellValue(table, row, 'filterName');

if (includeSingle) {
retval.single= builder('lsst-sdss-'+filterName,filterName, filterName);
}

if (includeStandard) {
retval.standard= [
builder('lsst-sdss-u', 'u', 'u'),
builder('lsst-sdss-g', 'g', 'g'),
builder('lsst-sdss-r', 'r', 'r'),
builder('lsst-sdss-i', 'i', 'i'),
builder('lsst-sdss-z', 'z', 'z'),
];
if (retval.standard[filterId]) retval.highlightPlotId= retval.standard[filterId].getPlotId();
}

if (threeColorOps) {
retval.threeColor= threeColorOps.map( (b) => b && builder('lsst-sdss-threeC', 'SDSS 3 Color', b) );
}
return retval;
}


3 changes: 1 addition & 2 deletions src/firefly/js/metaConvert/TwoMassRequestList.js
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,7 @@ import {RangeValues,STRETCH_LINEAR,PERCENTAGE} from '../visualize/RangeValues.js
import {getCellValue} from '../tables/TableUtil.js';

const colToUse= [ 'filter', 'scanno', 'fname', 'ordate', 'hemisphere', 'in_ra', 'in_dec', 'image_set' ];
const rangeValues= RangeValues.make(PERCENTAGE, 1, PERCENTAGE, 99);
rangeValues.algorithm= STRETCH_LINEAR;
const rangeValues= RangeValues.makeRV({which:PERCENTAGE, lowerValue:1, upperValue:99, algorithm:STRETCH_LINEAR});

const bandIdx= {'J':0, 'K':1, 'H':2};

Expand Down
3 changes: 1 addition & 2 deletions src/firefly/js/metaConvert/WiseRequestList.js
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,7 @@ import {RangeValues,STRETCH_LINEAR,SIGMA} from '../visualize/RangeValues.js';
import {getCellValue} from '../tables/TableUtil.js';

const colToUse= ['scan_id', 'frame_num', 'coadd_id', 'in_ra', 'in_dec', 'image_set'];
const rangeValues= RangeValues.make(SIGMA, -2, SIGMA, 10);
rangeValues.algorithm= STRETCH_LINEAR;
const rangeValues= RangeValues.makeRV({which:SIGMA, lowerValue:-2, upperValue:10, algorithm:STRETCH_LINEAR});

const bandMap= {b1:'1', b2:'2',b3:'3',b4:'4'};

Expand Down
Loading