Skip to content

DM-8361: PDAC related issues #243

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Nov 30, 2016
Merged

DM-8361: PDAC related issues #243

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
93c3f67
DM-8361: PDAC related issues
loitly Nov 22, 2016
a2185fb
DM-8361: PDAC related issues
loitly Nov 30, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 19 additions & 25 deletions src/firefly/java/edu/caltech/ipac/firefly/server/catquery/CatMasterTableQuery.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,21 +54,6 @@ public class CatMasterTableQuery extends IpacTablePartProcessor {
private static final String IRSA_HOST = AppProperties.getProperty("irsa.base.url", BaseGator.DEF_HOST);
private static final String IRSA_BASE_URL = QueryUtil.makeUrlBase(IRSA_HOST);


protected File loadDataFile(TableServerRequest request) throws IOException, DataAccessException {
File retFile;

StringKey key = new StringKey(CatMasterTableQuery.class.getName(), getUniqueID(request));
Cache cache = CacheManager.getCache(Cache.TYPE_TEMP_FILE);
retFile = (File) cache.get(key);
if (retFile == null) {
retFile = getMasterCatalogFile(request);
cache.put(key, retFile);
}

return retFile;
}

private static void placeOriginal(String catalogUrl, File originalFile) throws IOException {
try {
URL url = new URL(catalogUrl);
Expand Down Expand Up @@ -145,7 +130,7 @@ private static File getMasterCatalogFile(TableServerRequest request) throws IOEx
outputDG= dgExtra;
// DataGroupWriter.write(catOutFile, dgExtra, 0);
}
DataGroupWriter.write(catOutFile, outputDG, 0);
DataGroupWriter.write(catOutFile, outputDG);

DataGroup data = DataGroupReader.read(catOutFile);
// append hostname to relative path urls.
Expand All @@ -167,7 +152,7 @@ private static File getMasterCatalogFile(TableServerRequest request) throws IOEx

return retval;
}

private static String getValue(DataObject row, String colName) {
try {
return (String) row.getDataElement(colName);
Expand All @@ -176,8 +161,7 @@ private static String getValue(DataObject row, String colName) {
}
return null;
}



private static void addSearchProcessorCols(DataGroup dg) {

DataType catDt= new DataType("catSearchProcessor", String.class);
Expand All @@ -192,11 +176,8 @@ private static void addSearchProcessorCols(DataGroup dg) {
}
}




private static void makeIntoUrl(DataGroup dg, String colname, String linkDesc) {

DataType col = dg.getDataDefintion(colname);
for (int r = 0; r < dg.size(); r++) {
DataObject row = dg.get(r);
Expand All @@ -217,14 +198,13 @@ private static void makeIntoUrl(DataGroup dg, String colname, String linkDesc) {
}
}


private static void appendHostToUrls(DataGroup dg, String linkDesc, String... cols) {

if (cols == null || cols.length ==0) return;
for(int c = 0; c < cols.length; c++) {
DataType col = dg.getDataDefintion(cols[c]);
if (col == null) continue;

for (int r = 0; r < dg.size(); r++) {
DataObject row = dg.get(r);
String val = getValue(row, col.getKeyName());
Expand Down Expand Up @@ -276,5 +256,19 @@ public static DataGroup getBaseGatorData(String originalFilename) throws IOExcep
return DataGroupReader.read(new File(ServerContext.getPermWorkDir(), originalFilename));
}

protected File loadDataFile(TableServerRequest request) throws IOException, DataAccessException {
File retFile;

StringKey key = new StringKey(CatMasterTableQuery.class.getName(), getUniqueID(request));
Cache cache = CacheManager.getCache(Cache.TYPE_TEMP_FILE);
retFile = (File) cache.get(key);
if (retFile == null) {
retFile = getMasterCatalogFile(request);
cache.put(key, retFile);
}

return retFile;
}

}

60 changes: 28 additions & 32 deletions src/firefly/java/edu/caltech/ipac/firefly/server/persistence/QueryLSSTCatalog.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
import edu.caltech.ipac.firefly.server.util.Logger;
import edu.caltech.ipac.firefly.server.util.ipactable.DataGroupPart;
import edu.caltech.ipac.firefly.server.util.ipactable.DataGroupWriter;
import edu.caltech.ipac.firefly.server.util.ipactable.BgIpacTableHandler;
import edu.caltech.ipac.firefly.visualize.VisUtil;
import edu.caltech.ipac.util.*;
import edu.caltech.ipac.util.download.FailedRequestException;
Expand Down Expand Up @@ -45,6 +46,32 @@ public class QueryLSSTCatalog extends IpacTablePartProcessor {
private static String DATA_ACCESS_URI = AppProperties.getProperty("lsst.dataAccess.uri", "lsst.dataAccess.uri");
private static String DATABASE = AppProperties.getProperty("lsst.dataAccess.db", "lsst.dataAccess.db");

public static void main(String[] args) {

//String sql="select * from DeepSource where qserv_areaspec_box(0.4, 1.05, 0.5, 1.15)";
String sql="SELECT ra,decl,filterName FROM DC_W13_Stripe82.Science_Ccd_Exposure WHERE scisql_s2PtInBox(ra,decl,330,-0.1,335.1,-0.08)=1";


try {
long cTime = System.currentTimeMillis();
String url = "http://localhost:8661/db/v0/query?sql="+ URLEncoder.encode(sql, "UTF-8");
System.out.println("Executing SQL query: " + sql);
try {
URLConnection uc = URLDownload.makeConnection(new URL(url));
uc.setRequestProperty("Accept", "text/plain");
URLDownload.getDataToFile(uc, new File("/tmp/result.json"), null);
} catch (FailedRequestException e) {
throw new IOException("Request Failed", e);
}

System.out.println("SELECT took " + (System.currentTimeMillis() - cTime) + "ms");


} catch (Exception e) {
System.out.println("Exception "+e.getMessage());
e.printStackTrace();
}
}

@Override
protected File loadDataFile(TableServerRequest request) throws IOException, DataAccessException {
Expand All @@ -60,8 +87,7 @@ protected File loadDataFile(TableServerRequest request) throws IOException, Data

DataGroup dg = JsonToDataGroup.parse(file);
File inf = createFile(request, ".tbl");
DataGroupWriter.write(inf, dg, 0); // for big files write will happen in background

DataGroupWriter.write(new BgIpacTableHandler(inf, dg, request)); // for big files write will happen in background
return inf;
}

Expand Down Expand Up @@ -173,7 +199,6 @@ void doGetData(File oFile, List<Param> params, WorldPt wpt) throws DataAccessExc
}
}


@Override
public void prepareTableMeta(TableMeta meta, List<DataType> columns, ServerRequest request) {
super.prepareTableMeta(meta, columns, request);
Expand Down Expand Up @@ -232,8 +257,6 @@ private String getSelectedColumnsUpdate(String selectedColumns, String raCol, St

}



protected void setColumnTips(TableMeta meta, ServerRequest request) {

TableServerRequest req = new TableServerRequest("LSSTCatalogDD");
Expand Down Expand Up @@ -273,32 +296,5 @@ protected void setColumnTips(TableMeta meta, ServerRequest request) {
}
}

public static void main(String[] args) {

//String sql="select * from DeepSource where qserv_areaspec_box(0.4, 1.05, 0.5, 1.15)";
String sql="SELECT ra,decl,filterName FROM DC_W13_Stripe82.Science_Ccd_Exposure WHERE scisql_s2PtInBox(ra,decl,330,-0.1,335.1,-0.08)=1";


try {
long cTime = System.currentTimeMillis();
String url = "http://localhost:8661/db/v0/query?sql="+ URLEncoder.encode(sql, "UTF-8");
System.out.println("Executing SQL query: " + sql);
try {
URLConnection uc = URLDownload.makeConnection(new URL(url));
uc.setRequestProperty("Accept", "text/plain");
URLDownload.getDataToFile(uc, new File("/tmp/result.json"), null);
} catch (FailedRequestException e) {
throw new IOException("Request Failed", e);
}

System.out.println("SELECT took " + (System.currentTimeMillis() - cTime) + "ms");


} catch (Exception e) {
System.out.println("Exception "+e.getMessage());
e.printStackTrace();
}
}


}
2 changes: 1 addition & 1 deletion src/firefly/java/edu/caltech/ipac/firefly/server/persistence/QueryLSSTCatalogDD.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ protected File loadDataFile(TableServerRequest request) throws IOException, Data
}
toReturn.shrinkToFitData();
File inf = createFile(request, ".tbl");
DataGroupWriter.write(inf, toReturn, 0);
DataGroupWriter.write(inf, toReturn);
return inf;
} else {
return null;
Expand Down
2 changes: 1 addition & 1 deletion src/firefly/java/edu/caltech/ipac/firefly/server/persistence/QuerySearchHistory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ public int compare(SearchInfo o1, SearchInfo o2) {
dg.add(row);
}
File f = File.createTempFile(getFilePrefix(request), ".tbl", ServerContext.getTempWorkDir());
DataGroupWriter.write(f, dg, Integer.MAX_VALUE);
DataGroupWriter.write(f, dg);
return f;
} else {
return super.loadDataFile(request);
Expand Down
70 changes: 34 additions & 36 deletions src/firefly/java/edu/caltech/ipac/firefly/server/query/HistogramProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,39 @@ public class HistogramProcessor extends IpacTablePartProcessor {
private double falsePostiveRate = 0.05;
private boolean showEmptyBin=false;

private static String[] getInputFilePath(String inputFileName) {
String[] dirs = inputFileName.split("/");
String name = dirs[dirs.length - 1];
String path = inputFileName.substring(0, inputFileName.length() - name.length());
return new String[]{path, name};
}

public static void main(String args[]) throws IOException, DataAccessException {

if (args.length > 0) {
String path = getInputFilePath(args[0])[0];
String inFileName = getInputFilePath(args[0])[1];
if (args.length > 0) {
try {
File inFile = new File(args[0]);
DataGroup dg = IpacTableReader.readIpacTable(inFile, null, false, "inputTable");

HistogramProcessor hp = new HistogramProcessor();
hp.columnExpression = "f_y";

double[] columnData = hp.getColumnData(dg);

DataGroup outDg = hp.createHistogramTable(columnData);
String outFileName = path + "output_" + inFileName;
File outFile = new File(outFileName);
IpacTableWriter.save(outFile, outDg);

} catch (IpacTableException e) {
e.printStackTrace();
}
}
}
}

protected File loadDataFile(TableServerRequest request) throws IOException, DataAccessException {

Expand Down Expand Up @@ -104,7 +137,7 @@ protected File loadDataFile(TableServerRequest request) throws IOException, Data
addFormatInfoAtt(histogramDataGroup, columns[2]);

File histogramFile = createFile(request);
DataGroupWriter.write(histogramFile, histogramDataGroup, 0);
DataGroupWriter.write(histogramFile, histogramDataGroup);
return histogramFile;
}

Expand Down Expand Up @@ -287,13 +320,6 @@ private double[] getColumnData(DataGroup dg) {
return Arrays.stream(data).filter(d -> !Double.isNaN(d)).toArray();
}

private static String[] getInputFilePath(String inputFileName) {
String[] dirs = inputFileName.split("/");
String name = dirs[dirs.length - 1];
String path = inputFileName.substring(0, inputFileName.length() - name.length());
return new String[]{path, name};
}

/**
* This method calculate the variable bins based on the paper:
* http://iopscience.iop.org/article/10.1088/0004-637X/764/2/167/pdf;jsessionid=22827FAAA086B2A127E88C517E0E8DD3.c1.iopscience.cld.iop.org
Expand Down Expand Up @@ -498,7 +524,6 @@ private int getIndexOfTheMaxValue(double[] inArray) {
return maxIdx;
}


/**
* This method evaluate the fitness function for these possibilities
*/
Expand Down Expand Up @@ -659,32 +684,5 @@ private double[] multiply(double[] array, double a) {
return newArray;
}

public static void main(String args[]) throws IOException, DataAccessException {

if (args.length > 0) {
String path = getInputFilePath(args[0])[0];
String inFileName = getInputFilePath(args[0])[1];
if (args.length > 0) {
try {
File inFile = new File(args[0]);
DataGroup dg = IpacTableReader.readIpacTable(inFile, null, false, "inputTable");

HistogramProcessor hp = new HistogramProcessor();
hp.columnExpression = "f_y";

double[] columnData = hp.getColumnData(dg);

DataGroup outDg = hp.createHistogramTable(columnData);
String outFileName = path + "output_" + inFileName;
File outFile = new File(outFileName);
IpacTableWriter.save(outFile, outDg);

} catch (IpacTableException e) {
e.printStackTrace();
}
}
}
}


}
16 changes: 5 additions & 11 deletions src/firefly/java/edu/caltech/ipac/firefly/server/query/IpacTablePartProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,18 +22,12 @@
import edu.caltech.ipac.firefly.server.util.Logger;
import edu.caltech.ipac.firefly.server.util.QueryUtil;
import edu.caltech.ipac.firefly.server.util.StopWatch;
import edu.caltech.ipac.firefly.server.util.ipactable.DataGroupFilter;
import edu.caltech.ipac.firefly.server.util.ipactable.DataGroupPart;
import edu.caltech.ipac.firefly.server.util.ipactable.DataGroupReader;
import edu.caltech.ipac.firefly.server.util.ipactable.DataGroupWriter;
import edu.caltech.ipac.firefly.server.util.ipactable.IpacTableParser;
import edu.caltech.ipac.firefly.server.util.ipactable.TableDef;
import edu.caltech.ipac.firefly.server.util.ipactable.*;
import edu.caltech.ipac.util.AppProperties;
import edu.caltech.ipac.util.CollectionUtil;
import edu.caltech.ipac.util.DataGroup;
import edu.caltech.ipac.util.DataObject;
import edu.caltech.ipac.util.DataType;
import edu.caltech.ipac.util.FileUtil;
import edu.caltech.ipac.util.IpacTableUtil;
import edu.caltech.ipac.util.StringUtils;
import edu.caltech.ipac.util.cache.Cache;
Expand Down Expand Up @@ -120,7 +114,7 @@ protected static File convertToIpacTable(File tblFile, TableServerRequest reques
// read in any format.. then write it back out as ipac table
DataGroup dg = DataGroupReader.readAnyFormat(tblFile, tblIdx);
File convertedFile = File.createTempFile(request.getRequestId(), ".tbl", ServerContext.getTempWorkDir());
DataGroupWriter.write(convertedFile, dg, 0, request.getMeta());
DataGroupWriter.write(new BgIpacTableHandler(convertedFile, dg, request));
return convertedFile;
} else {
throw new DataAccessException("Source file has an unknown format:" + ServerContext.replaceWithPrefix(tblFile));
Expand Down Expand Up @@ -430,7 +424,7 @@ public File getDataFile(TableServerRequest request) throws IpacTableException, I

deciFile = File.createTempFile(getFilePrefix(request), ".tbl", ServerContext.getTempWorkDir());
DataGroup retval = QueryUtil.doDecimation(dg, decimateInfo);
DataGroupWriter.write(deciFile, retval, Integer.MAX_VALUE, request.getMeta());
DataGroupWriter.write(deciFile, retval);
cache.put(key, deciFile);
}
resultsFile = deciFile;
Expand Down Expand Up @@ -496,7 +490,7 @@ protected void doSort(File inFile, File outFile, SortInfo sortInfo, TableServerR
QueryUtil.doSort(dg, sortInfo);
timer.printLog("sort");
timer.start("write");
DataGroupWriter.write(outFile, dg, pageSize, request.getMeta());
DataGroupWriter.write(new BgIpacTableHandler(outFile, dg, request));
timer.printLog("write");
}

Expand Down Expand Up @@ -604,7 +598,7 @@ private boolean isInitLoad(TableServerRequest req) {
protected void doFilter(File outFile, File source, CollectionUtil.Filter<DataObject>[] filters, TableServerRequest request) throws IOException {
StopWatch timer = StopWatch.getInstance();
timer.start("filter");
DataGroupFilter.filter(outFile, source, filters, request.getPageSize(), request.getMeta());
DataGroupWriter.write(new FilterHanlder(outFile, source, filters, request));
timer.printLog("filter");
}

Expand Down
Loading