Skip to content

DM-7630: Display null_string when value is null #185

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Sep 19, 2016
Merged

DM-7630: Display null_string when value is null #185

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
622 changes: 24 additions & 598 deletions src/firefly/java/edu/caltech/ipac/astro/IpacTableReader.java
View file
Open in desktop

Large diffs are not rendered by default.

108 changes: 0 additions & 108 deletions src/firefly/java/edu/caltech/ipac/astro/IpacTableWriter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,113 +110,5 @@ private static void save(PrintWriter out, DataGroup dataGroup)
// ----------------------------------- Private Methods ---------------------------------------
// ============================================================

/**
* extract data
*
* @param dataObject a fixed object
* @param dataType array of data types
* @return string
*/
private static String extractData(DataObject dataObject,
DataType dataType[]) {
StringBuffer extraData = new StringBuffer();
Object value;
for (DataType dt : dataType) {
value = dataObject.getDataElement(dt);
if (value == null && dt.getMayBeNull()) {
extraData.append(EMPTY_SEPARATOR).append(dt.getFormatInfo().formatData(Locale.US, value, NULL_STRING));
} else {
extraData.append(EMPTY_SEPARATOR).append(dt.getFormatInfo().formatData(Locale.US, value, NOTHING));
}
}
return extraData.append(" ").toString();
}

/**
* write out the header of the catalog
*
* @param out the writer
* @param dataGroup data group
*/
private static void writeHeader(PrintWriter out, DataGroup dataGroup) {
DataType[] dataType = dataGroup.getDataDefinitions();

IpacTableUtil.writeAttributes(out, dataGroup.getKeywords());
IpacTableUtil.writeHeader(out, Arrays.asList(dataType));
}

/**
* write out the header (data name) of the catalog
*
* @param out the writer
* @param dataType array of data types
*/
private static void writeName(PrintWriter out, DataType dataType[]) {
for (DataType dt : dataType) {
DataType.FormatInfo info = dt.getFormatInfo();
out.print(SEPARATOR + info.formatHeader(dt.getKeyName()));
}
out.print(SEPARATOR + LINE_SEP);
}

/**
* write out the header (data type) of the catalog
*
* @param out the writer
* @param dataType array of data types
*/
private static void writeDataType(PrintWriter out,
DataType dataType[]) {
for (DataType dt : dataType) {
DataType.FormatInfo info = dt.getFormatInfo();

// handle invalid type, throw runtime exeption for now
Assert.tst(IpacTableReader.isRecongnizedType(dt.getTypeDesc()),
"Invalid data Type:" + dt.getTypeDesc());

out.print(SEPARATOR + info.formatHeader(dt.getTypeDesc()));
}
out.print(SEPARATOR + LINE_SEP);
}

/**
* write out the header (data unit) of the catalog
*
* @param out the writer
* @param dataType array of data types
*
*/
private static void writeDataUnit(PrintWriter out,
DataType dataType[]) {
for (DataType dt : dataType) {
DataType.FormatInfo info = dt.getFormatInfo();
if (dt.getDataUnit() == null ) {
out.print(SEPARATOR + info.formatHeader(NOTHING));
} else {
out.print(SEPARATOR + info.formatHeader(dt.getDataUnit()));
}
}
out.print(SEPARATOR + LINE_SEP);
}

/**
* write out the header (may be null) of the catalog
*
* @param out the writer
* @param dataType array of data types
*/
private static void writeIsNullAllowed(PrintWriter out,
DataType dataType[]) {
for (DataType dt : dataType) {
DataType.FormatInfo info = dt.getFormatInfo();
if (dt.getMayBeNull()) {
out.print(SEPARATOR + info.formatHeader(NULL_STRING));
} else {
out.print(SEPARATOR + info.formatHeader(NOTHING));
}
}
out.print(SEPARATOR + LINE_SEP);
}

}

6 changes: 2 additions & 4 deletions src/firefly/java/edu/caltech/ipac/firefly/server/query/HistogramProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,10 +39,8 @@ public class HistogramProcessor extends IpacTablePartProcessor {
};
static {
// set default precision to 9 significant digits
DataType.FormatInfo fi = DataType.FormatInfo.createDefaultFormat(Double.class);
fi.setDataFormat("%.9g");
columns[1].setFormatInfo(fi);
columns[2].setFormatInfo(fi);
columns[1].getFormatInfo().setDataFormat("%.9g");
columns[2].getFormatInfo().setDataFormat("%.9g");
}
private final String FIXED_SIZE_ALGORITHM = "fixedSizeBins";
private final String NUMBER_BINS = "numBins";
Expand Down
170 changes: 91 additions & 79 deletions src/firefly/java/edu/caltech/ipac/firefly/server/util/ipactable/DataGroupReader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
package edu.caltech.ipac.firefly.server.util.ipactable;

import edu.caltech.ipac.astro.FITSTableReader;
import edu.caltech.ipac.firefly.data.table.TableMeta;
import edu.caltech.ipac.firefly.server.db.spring.mapper.DataGroupUtil;
import edu.caltech.ipac.firefly.server.query.TemplateGenerator;
import edu.caltech.ipac.firefly.server.util.DsvToDataGroup;
Expand All @@ -27,12 +28,6 @@ public class DataGroupReader {
public static final String LINE_SEP = System.getProperty("line.separator");
private static final Logger.LoggerImpl logger = Logger.getLogger();

public static enum Format { TSV(CSVFormat.TDF), CSV(CSVFormat.DEFAULT), IPACTABLE(), UNKNOWN(), FIXEDTARGETS(), FITS(), JSON();
CSVFormat type;
Format() {}
Format(CSVFormat type) {this.type = type;}
}

public static DataGroup readAnyFormat(File inf) throws IOException {
return readAnyFormat(inf, 0);
}
Expand Down Expand Up @@ -62,10 +57,10 @@ public static DataGroup readAnyFormat(File inf, int tableIndex) throws IOExcepti
throw new IOException("Unsupported format, file:" + inf);
}
}

public static DataGroup read(File inf, String... onlyColumns) throws IOException {
return read(inf, false, onlyColumns);

}

public static DataGroup read(File inf, boolean readAsString, String... onlyColumns) throws IOException {
Expand All @@ -77,78 +72,15 @@ public static DataGroup read(File inf, boolean isFixedLength, boolean readAsStri
}

public static DataGroup read(File inf, boolean isFixedLength, boolean readAsString, boolean saveFormattedData, String... onlyColumns) throws IOException {

TableDef tableDef = IpacTableUtil.getMetaInfo(inf);
List<DataGroup.Attribute> attributes = tableDef.getAllAttributes();
List<DataType> cols = tableDef.getCols();

if (readAsString) {
for (DataType dt : cols) {
dt.setDataType(String.class);
}
}

DataGroup headers = new DataGroup(null, cols);
DataGroup data = null;
boolean isSelectedColumns = onlyColumns != null && onlyColumns.length > 0;

if (isSelectedColumns) {
List<DataType> selCols = new ArrayList<DataType>();
for (String c : onlyColumns) {
DataType dt = headers.getDataDefintion(c);
if (dt != null) {
try {
selCols.add((DataType) dt.clone());
} catch (CloneNotSupportedException e) {} // shouldn't happen
}
}
data = new DataGroup(null, selCols);
} else {
data = headers;
}

data.setAttributes(attributes);

String line = null;
int lineNum = 0;

BufferedReader reader = new BufferedReader(new FileReader(inf), IpacTableUtil.FILE_IO_BUFFER_SIZE);
try {
line = reader.readLine();
lineNum++;
while (line != null) {
DataObject row = IpacTableUtil.parseRow(headers, line, isFixedLength, saveFormattedData);
if (row != null) {
if (isSelectedColumns) {
DataObject arow = new DataObject(data);
for (DataType dt : data.getDataDefinitions()) {
arow.setDataElement(dt, row.getDataElement(dt.getKeyName()));
if (dt.getFormatInfo().isDefault()) {
dt.getFormatInfo().setDataFormat(
headers.getDataDefintion(dt.getKeyName()).getFormatInfo().getDataFormatStr());
}
}
data.add(arow);
} else {
data.add(row);
}
}
line = reader.readLine();
lineNum++;
}
} catch(Exception e) {
String msg = e.getMessage()+"<br>on line "+lineNum+": " + line;
if (msg.length()>128) msg = msg.substring(0,128)+"...";
logger.error(e, "on line "+lineNum+": " + line);
throw new IOException(msg);
} finally {
reader.close();
}
BufferedReader bufferedReader = new BufferedReader(new FileReader(inf), IpacTableUtil.FILE_IO_BUFFER_SIZE);
return doRead(bufferedReader, tableDef, isFixedLength, readAsString, saveFormattedData, onlyColumns);
}

if (!saveFormattedData) {
data.shrinkToFitData();
}
return data;
public static DataGroup read(Reader reader, boolean isFixedLength, boolean readAsString, boolean saveFormattedData, String... onlyColumns) throws IOException {
BufferedReader bufferedReader = new BufferedReader(reader, IpacTableUtil.FILE_IO_BUFFER_SIZE);
TableDef tableDef = IpacTableUtil.getMetaInfo(bufferedReader);
return doRead(bufferedReader, tableDef, isFixedLength, readAsString, saveFormattedData, onlyColumns);
}

public static DataGroup getEnumValues(File inf, int cutoffPoint) throws IOException {
Expand Down Expand Up @@ -236,7 +168,7 @@ public static Format guessFormat(File inf) throws IOException {
}

int readAhead = 10;

int row = 0;
BufferedReader reader = new BufferedReader(new FileReader(inf), IpacTableUtil.FILE_IO_BUFFER_SIZE);
try {
Expand Down Expand Up @@ -297,11 +229,91 @@ public static Format guessFormat(File inf) throws IOException {

}

private static DataGroup doRead(BufferedReader bufferedReader, TableDef tableDef, boolean isFixedLength, boolean readAsString, boolean saveFormattedData, String... onlyColumns) throws IOException {

List<DataGroup.Attribute> attributes = tableDef.getAllAttributes();
List<DataType> cols = tableDef.getCols();

if (readAsString) {
for (DataType dt : cols) {
dt.setDataType(String.class);
}
}

DataGroup inData = new DataGroup(null, cols);
DataGroup outData = null;
boolean isSelectedColumns = onlyColumns != null && onlyColumns.length > 0;

if (isSelectedColumns) {
List<DataType> selCols = new ArrayList<DataType>();
for (String c : onlyColumns) {
DataType dt = inData.getDataDefintion(c);
if (dt != null) {
try {
selCols.add((DataType) dt.clone());
} catch (CloneNotSupportedException e) {} // shouldn't happen
}
}
outData = new DataGroup(null, selCols);
} else {
outData = inData;
}

outData.setAttributes(attributes);

String line = null;
int lineNum = tableDef.getExtras() == null ? 0 : tableDef.getExtras().getKey();

try {
line = tableDef.getExtras() == null ? bufferedReader.readLine() : tableDef.getExtras().getValue();
lineNum++;
while (line != null) {
DataObject row = IpacTableUtil.parseRow(inData, line, isFixedLength, saveFormattedData);
if (row != null) {
if (isSelectedColumns) {
DataObject arow = new DataObject(outData);
for (DataType dt : outData.getDataDefinitions()) {
arow.setDataElement(dt, row.getDataElement(dt.getKeyName()));
if (dt.getFormatInfo().isDefault()) {
dt.getFormatInfo().setDataFormat(
inData.getDataDefintion(dt.getKeyName()).getFormatInfo().getDataFormatStr());
}
}
outData.add(arow);
} else {
outData.add(row);
}
}
line = bufferedReader.readLine();
lineNum++;
}
} catch(Exception e) {
String msg = e.getMessage()+"<br>on line "+lineNum+": " + line;
if (msg.length()>128) msg = msg.substring(0,128)+"...";
logger.error(e, "on line "+lineNum+": " + line);
throw new IOException(msg);
} finally {
bufferedReader.close();
}

if (!saveFormattedData) {
outData.shrinkToFitData();
}
return outData;

}



//====================================================================
//
//====================================================================

public static enum Format { TSV(CSVFormat.TDF), CSV(CSVFormat.DEFAULT), IPACTABLE(), UNKNOWN(), FIXEDTARGETS(), FITS(), JSON();
CSVFormat type;
Format() {}
Format(CSVFormat type) {this.type = type;}
}

}

7 changes: 3 additions & 4 deletions src/firefly/java/edu/caltech/ipac/firefly/server/util/ipactable/JsonTableUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,10 +115,9 @@ public static JSONObject toJsonTableData(DataGroup data, TableDef tableDef) {
String fkey = DataSetParser.makeAttribKey(DataSetParser.FORMAT_DISP_TAG, dt.getKeyName());
if (tableDef.contains(fkey)) {
dt.getFormatInfo().setDataFormat(tableDef.getAttribute(fkey).getValue());
dt.getFormatInfo().setWidth(Arrays.stream(new int[]{
String.valueOf(dt.getKeyName()).length(),
String.valueOf(dt.getDataUnit()).length(),
String.valueOf(dt.getTypeDesc()).length()}).max().getAsInt());
String[] headers = new String[] {dt.getKeyName(), dt.getTypeDesc(), dt.getDataUnit(), dt.getNullString()};
int maxLength = Arrays.stream(headers).mapToInt(s -> s == null ? 0 : s.length()).max().getAsInt();
dt.getFormatInfo().setWidth(maxLength);
formatChanged = true;
}
}
Expand Down
Loading