Skip to content

Commit 24e6442

Browse files
loitlyloitly
committed
DM-7630: Display null_string when value is null
- delegate IpacTableReader to use DataGroupReader instead. - fix selectInfo not updating totalRows
1 parent e7f87a1 commit 24e6442

File tree

19 files changed

+282
-871
lines changed

19 files changed

+282
-871
lines changed

‎src/firefly/java/edu/caltech/ipac/astro/IpacTableReader.java

Lines changed: 24 additions & 598 deletions
Large diffs are not rendered by default.

‎src/firefly/java/edu/caltech/ipac/astro/IpacTableWriter.java

Lines changed: 0 additions & 108 deletions
Original file line numberDiff line numberDiff line change
@@ -110,113 +110,5 @@ private static void save(PrintWriter out, DataGroup dataGroup)
110110
// ----------------------------------- Private Methods ---------------------------------------
111111
// ============================================================
112112

113-
/**
114-
* extract data
115-
*
116-
* @param dataObject a fixed object
117-
* @param dataType array of data types
118-
* @return string
119-
*/
120-
private static String extractData(DataObject dataObject,
121-
DataType dataType[]) {
122-
StringBuffer extraData = new StringBuffer();
123-
Object value;
124-
for (DataType dt : dataType) {
125-
value = dataObject.getDataElement(dt);
126-
if (value == null && dt.getMayBeNull()) {
127-
extraData.append(EMPTY_SEPARATOR).append(dt.getFormatInfo().formatData(Locale.US, value, NULL_STRING));
128-
} else {
129-
extraData.append(EMPTY_SEPARATOR).append(dt.getFormatInfo().formatData(Locale.US, value, NOTHING));
130-
}
131-
}
132-
return extraData.append(" ").toString();
133-
}
134-
135-
/**
136-
* write out the header of the catalog
137-
*
138-
* @param out the writer
139-
* @param dataGroup data group
140-
*/
141-
private static void writeHeader(PrintWriter out, DataGroup dataGroup) {
142-
DataType[] dataType = dataGroup.getDataDefinitions();
143-
144-
IpacTableUtil.writeAttributes(out, dataGroup.getKeywords());
145-
IpacTableUtil.writeHeader(out, Arrays.asList(dataType));
146-
}
147-
148-
/**
149-
* write out the header (data name) of the catalog
150-
*
151-
* @param out the writer
152-
* @param dataType array of data types
153-
*/
154-
private static void writeName(PrintWriter out, DataType dataType[]) {
155-
for (DataType dt : dataType) {
156-
DataType.FormatInfo info = dt.getFormatInfo();
157-
out.print(SEPARATOR + info.formatHeader(dt.getKeyName()));
158-
}
159-
out.print(SEPARATOR + LINE_SEP);
160-
}
161-
162-
/**
163-
* write out the header (data type) of the catalog
164-
*
165-
* @param out the writer
166-
* @param dataType array of data types
167-
*/
168-
private static void writeDataType(PrintWriter out,
169-
DataType dataType[]) {
170-
for (DataType dt : dataType) {
171-
DataType.FormatInfo info = dt.getFormatInfo();
172-
173-
// handle invalid type, throw runtime exeption for now
174-
Assert.tst(IpacTableReader.isRecongnizedType(dt.getTypeDesc()),
175-
"Invalid data Type:" + dt.getTypeDesc());
176-
177-
out.print(SEPARATOR + info.formatHeader(dt.getTypeDesc()));
178-
}
179-
out.print(SEPARATOR + LINE_SEP);
180-
}
181-
182-
/**
183-
* write out the header (data unit) of the catalog
184-
*
185-
* @param out the writer
186-
* @param dataType array of data types
187-
*
188-
*/
189-
private static void writeDataUnit(PrintWriter out,
190-
DataType dataType[]) {
191-
for (DataType dt : dataType) {
192-
DataType.FormatInfo info = dt.getFormatInfo();
193-
if (dt.getDataUnit() == null ) {
194-
out.print(SEPARATOR + info.formatHeader(NOTHING));
195-
} else {
196-
out.print(SEPARATOR + info.formatHeader(dt.getDataUnit()));
197-
}
198-
}
199-
out.print(SEPARATOR + LINE_SEP);
200-
}
201-
202-
/**
203-
* write out the header (may be null) of the catalog
204-
*
205-
* @param out the writer
206-
* @param dataType array of data types
207-
*/
208-
private static void writeIsNullAllowed(PrintWriter out,
209-
DataType dataType[]) {
210-
for (DataType dt : dataType) {
211-
DataType.FormatInfo info = dt.getFormatInfo();
212-
if (dt.getMayBeNull()) {
213-
out.print(SEPARATOR + info.formatHeader(NULL_STRING));
214-
} else {
215-
out.print(SEPARATOR + info.formatHeader(NOTHING));
216-
}
217-
}
218-
out.print(SEPARATOR + LINE_SEP);
219-
}
220-
221113
}
222114

‎src/firefly/java/edu/caltech/ipac/firefly/server/query/HistogramProcessor.java

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -39,10 +39,8 @@ public class HistogramProcessor extends IpacTablePartProcessor {
3939
};
4040
static {
4141
// set default precision to 9 significant digits
42-
DataType.FormatInfo fi = DataType.FormatInfo.createDefaultFormat(Double.class);
43-
fi.setDataFormat("%.9g");
44-
columns[1].setFormatInfo(fi);
45-
columns[2].setFormatInfo(fi);
42+
columns[1].getFormatInfo().setDataFormat("%.9g");
43+
columns[2].getFormatInfo().setDataFormat("%.9g");
4644
}
4745
private final String FIXED_SIZE_ALGORITHM = "fixedSizeBins";
4846
private final String NUMBER_BINS = "numBins";

‎src/firefly/java/edu/caltech/ipac/firefly/server/util/ipactable/DataGroupReader.java

Lines changed: 91 additions & 79 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
package edu.caltech.ipac.firefly.server.util.ipactable;
55

66
import edu.caltech.ipac.astro.FITSTableReader;
7+
import edu.caltech.ipac.firefly.data.table.TableMeta;
78
import edu.caltech.ipac.firefly.server.db.spring.mapper.DataGroupUtil;
89
import edu.caltech.ipac.firefly.server.query.TemplateGenerator;
910
import edu.caltech.ipac.firefly.server.util.DsvToDataGroup;
@@ -27,12 +28,6 @@ public class DataGroupReader {
2728
public static final String LINE_SEP = System.getProperty("line.separator");
2829
private static final Logger.LoggerImpl logger = Logger.getLogger();
2930

30-
public static enum Format { TSV(CSVFormat.TDF), CSV(CSVFormat.DEFAULT), IPACTABLE(), UNKNOWN(), FIXEDTARGETS(), FITS(), JSON();
31-
CSVFormat type;
32-
Format() {}
33-
Format(CSVFormat type) {this.type = type;}
34-
}
35-
3631
public static DataGroup readAnyFormat(File inf) throws IOException {
3732
return readAnyFormat(inf, 0);
3833
}
@@ -62,10 +57,10 @@ public static DataGroup readAnyFormat(File inf, int tableIndex) throws IOExcepti
6257
throw new IOException("Unsupported format, file:" + inf);
6358
}
6459
}
65-
60+
6661
public static DataGroup read(File inf, String... onlyColumns) throws IOException {
6762
return read(inf, false, onlyColumns);
68-
63+
6964
}
7065

7166
public static DataGroup read(File inf, boolean readAsString, String... onlyColumns) throws IOException {
@@ -77,78 +72,15 @@ public static DataGroup read(File inf, boolean isFixedLength, boolean readAsStri
7772
}
7873

7974
public static DataGroup read(File inf, boolean isFixedLength, boolean readAsString, boolean saveFormattedData, String... onlyColumns) throws IOException {
80-
8175
TableDef tableDef = IpacTableUtil.getMetaInfo(inf);
82-
List<DataGroup.Attribute> attributes = tableDef.getAllAttributes();
83-
List<DataType> cols = tableDef.getCols();
84-
85-
if (readAsString) {
86-
for (DataType dt : cols) {
87-
dt.setDataType(String.class);
88-
}
89-
}
90-
91-
DataGroup headers = new DataGroup(null, cols);
92-
DataGroup data = null;
93-
boolean isSelectedColumns = onlyColumns != null && onlyColumns.length > 0;
94-
95-
if (isSelectedColumns) {
96-
List<DataType> selCols = new ArrayList<DataType>();
97-
for (String c : onlyColumns) {
98-
DataType dt = headers.getDataDefintion(c);
99-
if (dt != null) {
100-
try {
101-
selCols.add((DataType) dt.clone());
102-
} catch (CloneNotSupportedException e) {} // shouldn't happen
103-
}
104-
}
105-
data = new DataGroup(null, selCols);
106-
} else {
107-
data = headers;
108-
}
109-
110-
data.setAttributes(attributes);
111-
112-
String line = null;
113-
int lineNum = 0;
114-
115-
BufferedReader reader = new BufferedReader(new FileReader(inf), IpacTableUtil.FILE_IO_BUFFER_SIZE);
116-
try {
117-
line = reader.readLine();
118-
lineNum++;
119-
while (line != null) {
120-
DataObject row = IpacTableUtil.parseRow(headers, line, isFixedLength, saveFormattedData);
121-
if (row != null) {
122-
if (isSelectedColumns) {
123-
DataObject arow = new DataObject(data);
124-
for (DataType dt : data.getDataDefinitions()) {
125-
arow.setDataElement(dt, row.getDataElement(dt.getKeyName()));
126-
if (dt.getFormatInfo().isDefault()) {
127-
dt.getFormatInfo().setDataFormat(
128-
headers.getDataDefintion(dt.getKeyName()).getFormatInfo().getDataFormatStr());
129-
}
130-
}
131-
data.add(arow);
132-
} else {
133-
data.add(row);
134-
}
135-
}
136-
line = reader.readLine();
137-
lineNum++;
138-
}
139-
} catch(Exception e) {
140-
String msg = e.getMessage()+"<br>on line "+lineNum+": " + line;
141-
if (msg.length()>128) msg = msg.substring(0,128)+"...";
142-
logger.error(e, "on line "+lineNum+": " + line);
143-
throw new IOException(msg);
144-
} finally {
145-
reader.close();
146-
}
76+
BufferedReader bufferedReader = new BufferedReader(new FileReader(inf), IpacTableUtil.FILE_IO_BUFFER_SIZE);
77+
return doRead(bufferedReader, tableDef, isFixedLength, readAsString, saveFormattedData, onlyColumns);
78+
}
14779

148-
if (!saveFormattedData) {
149-
data.shrinkToFitData();
150-
}
151-
return data;
80+
public static DataGroup read(Reader reader, boolean isFixedLength, boolean readAsString, boolean saveFormattedData, String... onlyColumns) throws IOException {
81+
BufferedReader bufferedReader = new BufferedReader(reader, IpacTableUtil.FILE_IO_BUFFER_SIZE);
82+
TableDef tableDef = IpacTableUtil.getMetaInfo(bufferedReader);
83+
return doRead(bufferedReader, tableDef, isFixedLength, readAsString, saveFormattedData, onlyColumns);
15284
}
15385

15486
public static DataGroup getEnumValues(File inf, int cutoffPoint) throws IOException {
@@ -236,7 +168,7 @@ public static Format guessFormat(File inf) throws IOException {
236168
}
237169

238170
int readAhead = 10;
239-
171+
240172
int row = 0;
241173
BufferedReader reader = new BufferedReader(new FileReader(inf), IpacTableUtil.FILE_IO_BUFFER_SIZE);
242174
try {
@@ -297,11 +229,91 @@ public static Format guessFormat(File inf) throws IOException {
297229

298230
}
299231

232+
private static DataGroup doRead(BufferedReader bufferedReader, TableDef tableDef, boolean isFixedLength, boolean readAsString, boolean saveFormattedData, String... onlyColumns) throws IOException {
233+
234+
List<DataGroup.Attribute> attributes = tableDef.getAllAttributes();
235+
List<DataType> cols = tableDef.getCols();
236+
237+
if (readAsString) {
238+
for (DataType dt : cols) {
239+
dt.setDataType(String.class);
240+
}
241+
}
242+
243+
DataGroup inData = new DataGroup(null, cols);
244+
DataGroup outData = null;
245+
boolean isSelectedColumns = onlyColumns != null && onlyColumns.length > 0;
246+
247+
if (isSelectedColumns) {
248+
List<DataType> selCols = new ArrayList<DataType>();
249+
for (String c : onlyColumns) {
250+
DataType dt = inData.getDataDefintion(c);
251+
if (dt != null) {
252+
try {
253+
selCols.add((DataType) dt.clone());
254+
} catch (CloneNotSupportedException e) {} // shouldn't happen
255+
}
256+
}
257+
outData = new DataGroup(null, selCols);
258+
} else {
259+
outData = inData;
260+
}
261+
262+
outData.setAttributes(attributes);
263+
264+
String line = null;
265+
int lineNum = tableDef.getExtras() == null ? 0 : tableDef.getExtras().getKey();
266+
267+
try {
268+
line = tableDef.getExtras() == null ? bufferedReader.readLine() : tableDef.getExtras().getValue();
269+
lineNum++;
270+
while (line != null) {
271+
DataObject row = IpacTableUtil.parseRow(inData, line, isFixedLength, saveFormattedData);
272+
if (row != null) {
273+
if (isSelectedColumns) {
274+
DataObject arow = new DataObject(outData);
275+
for (DataType dt : outData.getDataDefinitions()) {
276+
arow.setDataElement(dt, row.getDataElement(dt.getKeyName()));
277+
if (dt.getFormatInfo().isDefault()) {
278+
dt.getFormatInfo().setDataFormat(
279+
inData.getDataDefintion(dt.getKeyName()).getFormatInfo().getDataFormatStr());
280+
}
281+
}
282+
outData.add(arow);
283+
} else {
284+
outData.add(row);
285+
}
286+
}
287+
line = bufferedReader.readLine();
288+
lineNum++;
289+
}
290+
} catch(Exception e) {
291+
String msg = e.getMessage()+"<br>on line "+lineNum+": " + line;
292+
if (msg.length()>128) msg = msg.substring(0,128)+"...";
293+
logger.error(e, "on line "+lineNum+": " + line);
294+
throw new IOException(msg);
295+
} finally {
296+
bufferedReader.close();
297+
}
298+
299+
if (!saveFormattedData) {
300+
outData.shrinkToFitData();
301+
}
302+
return outData;
303+
304+
}
305+
300306

301307

302308
//====================================================================
303309
//
304310
//====================================================================
305311

312+
public static enum Format { TSV(CSVFormat.TDF), CSV(CSVFormat.DEFAULT), IPACTABLE(), UNKNOWN(), FIXEDTARGETS(), FITS(), JSON();
313+
CSVFormat type;
314+
Format() {}
315+
Format(CSVFormat type) {this.type = type;}
316+
}
317+
306318
}
307319

‎src/firefly/java/edu/caltech/ipac/firefly/server/util/ipactable/JsonTableUtil.java

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -115,10 +115,9 @@ public static JSONObject toJsonTableData(DataGroup data, TableDef tableDef) {
115115
String fkey = DataSetParser.makeAttribKey(DataSetParser.FORMAT_DISP_TAG, dt.getKeyName());
116116
if (tableDef.contains(fkey)) {
117117
dt.getFormatInfo().setDataFormat(tableDef.getAttribute(fkey).getValue());
118-
dt.getFormatInfo().setWidth(Arrays.stream(new int[]{
119-
String.valueOf(dt.getKeyName()).length(),
120-
String.valueOf(dt.getDataUnit()).length(),
121-
String.valueOf(dt.getTypeDesc()).length()}).max().getAsInt());
118+
String[] headers = new String[] {dt.getKeyName(), dt.getTypeDesc(), dt.getDataUnit(), dt.getNullString()};
119+
int maxLength = Arrays.stream(headers).mapToInt(s -> s == null ? 0 : s.length()).max().getAsInt();
120+
dt.getFormatInfo().setWidth(maxLength);
122121
formatChanged = true;
123122
}
124123
}

0 commit comments

Comments
 (0)