diff --git a/src/firefly/java/edu/caltech/ipac/astro/IpacTableReader.java b/src/firefly/java/edu/caltech/ipac/astro/IpacTableReader.java index 57fe523097..54e3605010 100644 --- a/src/firefly/java/edu/caltech/ipac/astro/IpacTableReader.java +++ b/src/firefly/java/edu/caltech/ipac/astro/IpacTableReader.java @@ -3,13 +3,9 @@ */ package edu.caltech.ipac.astro; -import edu.caltech.ipac.util.AppProperties; -import edu.caltech.ipac.util.Assert; -import edu.caltech.ipac.util.DataGroup; -import edu.caltech.ipac.util.DataObject; -import edu.caltech.ipac.util.DataType; -import edu.caltech.ipac.util.FileUtil; -import edu.caltech.ipac.util.ServerStringUtil; +import edu.caltech.ipac.firefly.server.util.ipactable.DataGroupReader; +import edu.caltech.ipac.firefly.server.util.ipactable.TableDef; +import edu.caltech.ipac.util.*; import edu.caltech.ipac.util.action.ActionConst; import edu.caltech.ipac.util.action.ClassProperties; @@ -37,85 +33,8 @@ public final class IpacTableReader { private static ClassProperties _prop= new ClassProperties( IpacTableReader.class); - private static final String WRONG_FORMAT = _prop.getError("wrongFormat"); private static final String NO_DATA = _prop.getError("noData"); - private static final String DOUBLE_TYPE[]= { "doub.*", "d", "real", - "r", "float", "f"}; - private static final String INT_TYPE[]= { "int.*", "i"} ; - private static final String LONG_TYPE[]= { "long", "l"} ; - private static final String BOOL_TYPE[]= { "bool", "b"} ; - private static final String STRING_TYPE[]= {"cha.*", "str.*", "s", "c"}; - - private String _line; // to keep the line read from file - private DataGroup _dataGroup; - private String _onlyColumns[]; - private final boolean _useFloatsForDoubles; - private boolean _isHeadersOnlyAllow; - - - private IpacTableReader(Reader fr, - String catName, - String onlyColumns[], - boolean useFloatsForDoubles, - long estFileSize, - boolean isHeadersOnlyAllow ) - throws IpacTableException { - _useFloatsForDoubles= useFloatsForDoubles; - int rowsRead = 0; - _onlyColumns= onlyColumns; - _isHeadersOnlyAllow = isHeadersOnlyAllow; - List columnsDesc; - DataType [] extraData; - List attributes = null; - - //String line; - //System.out.println("readIpacTable........: "); - - try { - LineNumberReader reader= new LineNumberReader(fr); - attributes = getAttributes(reader); - columnsDesc = getHeaderInfo(reader); - - //cols = columnsDesc.size(); - //extraData= new DataType[cols]; - extraData= setupExtraData(columnsDesc); - _dataGroup= new DataGroup(catName, extraData); - - _dataGroup.setAttributes(attributes); - - if ( columnsDesc.size() > 0 && _line != null) { - if (estFileSize>0 && estFileSize>(_line.length()+1) ) { - int estLines= (int)(estFileSize / (_line.length()+1)); - _dataGroup.ensureCapacity(estLines); - } - - while (_line != null) { - addToGroup(_line, columnsDesc, _dataGroup, extraData); - rowsRead++; - _line= reader.readLine(); - } - guessTypesIfNecessary(_dataGroup, columnsDesc); - } - } catch (IOException e) { - System.out.println("IO Exception"); - } - if (rowsRead == 0) { - if (_isHeadersOnlyAllow) { - //there are headers info, therefore it is not an exception - } else { - String name = AppProperties.getProperty("CatalogDialog.cats." - +catName+ ".ShortName"); - if (name == null) - name = catName; - throw new IpacTableException(NO_DATA + ": " + name); - } - } - //System.out.println("right before return "); - - } - - /** * Parse the file in IPAC table format and put the data in a * DataObjectGroup. If there is no data in the file, throw @@ -159,12 +78,13 @@ public static DataGroup readIpacTable(Reader fr, long estFileSize, boolean isHeadersOnlyAllow) throws IpacTableException { - IpacTableReader tableReader= new IpacTableReader(fr,catName, - onlyColumns, - useFloatsForDoubles, - estFileSize, - isHeadersOnlyAllow); - return tableReader._dataGroup; + try { + DataGroup retval = DataGroupReader.read(fr, true, false, false, onlyColumns); + ensureIpac(retval, catName, isHeadersOnlyAllow); + return retval; + } catch (IOException e) { + throw new IpacTableException(e.getMessage(), e); + } } @@ -193,525 +113,32 @@ public static DataGroup readIpacTable(File f, String catName, boolean isHeadersOnlyAllow ) throws IpacTableException { - DataGroup retval= null; - BufferedReader br= null; try { - br= new BufferedReader(new FileReader(f),8192) ; - retval= readIpacTable(br, - catName, onlyColumns, useFloatsForDoubles, - f.length(), isHeadersOnlyAllow); + DataGroup retval = DataGroupReader.read(f,true, false, false, onlyColumns); + ensureIpac(retval, catName, isHeadersOnlyAllow); + return retval; } catch (FileNotFoundException fnfe) { System.out.println("File not found Exception"); throw new IpacTableException("File or object not found"); - } finally { - FileUtil.silentClose(br); - } - return retval; - } - - private List getAttributes(LineNumberReader reader) - throws IOException { - - List attributes = - new ArrayList(); - reader.mark(1024); - _line = reader.readLine(); - while (_line != null) { - if ( _line.startsWith("\\") ) { // line starting with '\' is a keyword line. - DataGroup.Attribute attrib = DataGroup.Attribute.parse(_line); - if (attrib != null) { - attributes.add(attrib); - } - } else if ( _line.indexOf("|") >= 0) { - reader.reset(); // return the reader to the previous line.. allow getHeaderInfo code to run without changes. - return attributes; - } - reader.mark(1024); - _line = reader.readLine(); - } - return attributes; - } - - private List getHeaderInfo(LineNumberReader reader) - throws IpacTableException, - IOException { - List columnsDesc; - IpacTableColumn itc; - int cols; - String mayBeNull; - - _line = reader.readLine(); - while (_line != null) { - if (_line.indexOf("|") == -1) - _line= reader.readLine(); - else - break; - } - - if ( _line == null ) { - if (_isHeadersOnlyAllow) { - return new ArrayList(0); - } else { - throw new IpacTableException("This table does not have column headers information."); - } - } - - // got the first header line here - - try { - StringTokenizer tokens = new StringTokenizer(_line, "|"); - cols = tokens.countTokens(); - int lastEndIdx= 0; - //System.out.println("number of columns: " + cols); - columnsDesc = new ArrayList(cols); - for (int i=0; i 0) { - itc = columnsDesc.get(0); - if (itc._name.trim().equals("")) - { - columnsDesc.remove(0); //remove columns if no name - cols--; - } - } - - - - } catch (NullPointerException npex) { - throw new IpacTableException(WRONG_FORMAT); - } - return columnsDesc; - } - - public static boolean isSpaces(String s) { - int length = s.length(); - boolean retval = true; - - for (int i=0; i columnsDesc, - DataGroup dataGroup, - DataType extraData[]) { - DataObject dataObj = new DataObject(dataGroup); - int cols = columnsDesc.size(); - String tStr; - boolean wrongData = false; - IpacTableColumn itc; - IpacTableColumn lastItc; - int workingEndIdx; - - if (isSpaces(line)) return; - - // -check to see if this line is shorter then expected - int realIdx = _line.length(); - lastItc = columnsDesc.get(columnsDesc.size() - 1); - if (realIdx <= lastItc._startIndex) { - System.out.println("The following line is " + - (lastItc._endIndex - realIdx) + - " characters shorter then expected. " + - "The Catalog may be corrupt"); - System.out.println(_line); - } - - for (int i = 0; i < cols; i++) { - itc = columnsDesc.get(i); - workingEndIdx = (realIdx < itc._endIndex) ? realIdx : itc._endIndex; - if (workingEndIdx < itc._startIndex) { - break; - } - String str = line.substring(itc._startIndex, workingEndIdx); - tStr = str.trim(); - int idx = findExtraDataIdx(extraData, itc._name); - if (itc._useColumn) { - wrongData = !setExtraDataItem(tStr, itc, dataObj, extraData[idx]); - } - - if (idx >= 0) { - // guess what the format of the column should be - setFormatInfoWhenNecessary(extraData[idx], itc, str); - } - - } // end loop - if (!wrongData) - dataGroup.add(dataObj); - } - - private void setFormatInfoWhenNecessary(DataType dataType, IpacTableColumn colDesc, String value) { - - if (!dataType.hasFormatInfo()) { - int width = colDesc._endIndex - colDesc._startIndex - 1; - String formatStr = guessFormatStr(dataType, value.trim()); - DataType.FormatInfo.Align align = value.endsWith(" ") ? DataType.FormatInfo.Align.LEFT - : DataType.FormatInfo.Align.RIGHT; - if (formatStr != null) { - DataType.FormatInfo fi = new DataType.FormatInfo(width); - fi.setDataFormat(formatStr); - fi.setDataAlign(align); - dataType.setFormatInfo(fi); - } - } - } - - - public boolean setExtraDataItem(String tStr, - IpacTableColumn itc, - DataObject dataObj, - DataType extraData) { - boolean success= true; - tStr= tStr.trim(); - try { - if (tStr.length()==0 || tStr.equals("null") || - (itc._badValueString!=null && tStr.equals(itc._badValueString))) { - if (itc._mayBeNull) { - dataObj.setDataElement(extraData, null); - } - else { - success= false; - } - } - else if (itc._foundType==null) { - dataObj.setDataElement(extraData, tStr); - } - else if (itc._foundType==Float.class) { - tStr= tStr.replaceAll(",", ""); - Float v = tStr.equalsIgnoreCase(Float.toString(Float.NaN)) ? Float.NaN : Float.valueOf(tStr); - dataObj.setDataElement(extraData, v); - } - else if (itc._foundType==Double.class) { - tStr= tStr.replaceAll(",", ""); - Double v = tStr.equalsIgnoreCase(Double.toString(Float.NaN)) ? Double.NaN : Double.valueOf(tStr); - dataObj.setDataElement(extraData, v); - - } - else if (itc._foundType==Integer.class) { - if (tStr.equals("null")) { - success= false; - } - else { - dataObj.setDataElement(extraData, Integer.valueOf(tStr)); - } - } - else if (itc._foundType==Long.class) { - if (tStr.equals("null")) { - success= false; - } - else { - dataObj.setDataElement(extraData, Long.valueOf(tStr)); - } - } - else if (itc._foundType==String.class) { - dataObj.setDataElement(extraData, tStr); - } - else { - dataObj.setDataElement(extraData, tStr); - } - } catch (NumberFormatException e) { - dataObj.setDataElement(extraData, tStr); - itc._type=null; - itc._type=null; - } - return success; - } - - /* - public static void main(String args[]) { - File f= new File(args[0]); - DataObjectGroup fixedGroup= readDummyData(f); - System.out.println("min ra= "+ fixedGroup.minRa() ); - System.out.println("max ra= "+ fixedGroup.maxRa() ); - System.out.println("min dec= "+ fixedGroup.minDec() ); - System.out.println("max dec= "+ fixedGroup.maxDec() ); - } - */ - - - private boolean findStartEndIndex(String line, - IpacTableColumn itc, - int startingPoint) { - int index = line.indexOf(itc._name, startingPoint); - int indexStartBar = line.lastIndexOf("|", index); - int indexEndBar = line.indexOf("|", index); - - itc._startIndex = indexStartBar; // start bar included - itc._endIndex = indexEndBar; // end bar not, subString() does this - return (indexEndBar > indexStartBar); - } - - private DataType [] setupExtraData(List columnsDesc) { - int cols = columnsDesc.size(); - String colName, colKey; - IpacTableColumn itc; - List extraData= new ArrayList(columnsDesc.size()); - DataType dataType= null; - - for (int i=0; i columnsDesc) { - int eIdx; - DataType extraData[]= dataGroup.getDataDefinitions(); - String colKey; - - for(IpacTableColumn itc: columnsDesc) { - if (!isRecognizedType(itc)) { // try to guess type - colKey = itc._name; - eIdx = findExtraDataIdx(extraData, colKey); - if (eIdx>-1) { - boolean guessed= guessType(dataGroup, extraData[eIdx]); - if (guessed) { - setAllInColumnToType(dataGroup, extraData[eIdx]); - } - } - } - } - } - - - private void setAllInColumnToType(DataGroup dataGroup, DataType dataType) { - DataObject dataObj; - Object data; - Assert.tst( dataType.isKnownType()); - for(Iterator i= dataGroup.iterator(); (i.hasNext()); ) { - dataObj= (DataObject)i.next(); - data= dataObj.getDataElement(dataType); - dataObj.setDataElement( dataType, - dataType.convertStringToData(data.toString()) ); - } - } - - private int findExtraDataIdx(DataType extraData[], String name) { - - boolean found= false; - int retval= -1; - if (extraData!=null) { - for(int i=0; (i= 0 ? "E" : "e"; - String[] ary = val.split("e|E"); - if (ary.length == 2) { - int prec = ary[0].length() - ary[0].indexOf(".") - 1; - return "%." + prec + convStr; - } - } else if (val.indexOf(".") >= 0) { - // decimal format - int idx = val.indexOf("."); - int prec = val.length() - idx - 1; - return "%." + prec + "f"; - } else { - boolean isFloat= (cls==Float.class || cls==Double.class); - formatStr = isFloat ? "%.0f" : "%d"; - } - } - } catch (NumberFormatException e) { - formatStr = "%s"; - } - return formatStr; - } - - private boolean isRecognizedType(IpacTableColumn itc) { - return isRecongnizedType(itc._type); - } - -//=================================================================== -// public static methods -//=================================================================== - public static boolean isRecongnizedType(String type) { - if ( type == null ) { - return false; - } - return ServerStringUtil.matchesRegExpList(type, DOUBLE_TYPE, true) || - ServerStringUtil.matchesRegExpList(type, INT_TYPE, true) || - ServerStringUtil.matchesRegExpList(type, LONG_TYPE, true) || - ServerStringUtil.matchesRegExpList(type, BOOL_TYPE, true) || - ServerStringUtil.matchesRegExpList(type, STRING_TYPE, true); - } - - public static Class resolveClass(String type) { - if ( ServerStringUtil.matchesRegExpList(type, DOUBLE_TYPE, true) ) { - return Double.class; - } else if ( ServerStringUtil.matchesRegExpList(type, INT_TYPE, true) ) { - return Integer.class; - } else if ( ServerStringUtil.matchesRegExpList(type, LONG_TYPE, true) ) { - return Long.class; - } else if ( ServerStringUtil.matchesRegExpList(type, BOOL_TYPE, true) ) { - return Boolean.class; - } - return String.class; - } - - public static Object parseString(String type, String s) { - if ( Double.class.isAssignableFrom(resolveClass(type)) ) { - Double v = s.equalsIgnoreCase(Double.toString(Double.NaN)) ? Double.NaN : Double.valueOf(s); - return v; - } else if ( Integer.class.isAssignableFrom(resolveClass(type)) ) { - return Integer.valueOf(s); - } else if ( Long.class.isAssignableFrom(resolveClass(type)) ) { - return Long.valueOf(s); - } - return s; - } -//=================================================================== - - private static class IpacTableColumn { - public int _startIndex; // inclusive - public int _endIndex; // exclusive - public String _name; - public String _type; // can be: char, double, int - public Class _foundType= null; // can be: String, Double, Float, Int - public String _unit; - public boolean _useColumn= false; // true: use this column, false: ignore - public boolean _mayBeNull; - public String _badValueString = null; - } - public static void main(String args[]) { @@ -742,5 +169,4 @@ public static void main(String args[]) { } } - } diff --git a/src/firefly/java/edu/caltech/ipac/astro/IpacTableWriter.java b/src/firefly/java/edu/caltech/ipac/astro/IpacTableWriter.java index 47ae45381c..9e4c65e258 100644 --- a/src/firefly/java/edu/caltech/ipac/astro/IpacTableWriter.java +++ b/src/firefly/java/edu/caltech/ipac/astro/IpacTableWriter.java @@ -110,113 +110,5 @@ private static void save(PrintWriter out, DataGroup dataGroup) // ----------------------------------- Private Methods --------------------------------------- // ============================================================ - /** - * extract data - * - * @param dataObject a fixed object - * @param dataType array of data types - * @return string - */ - private static String extractData(DataObject dataObject, - DataType dataType[]) { - StringBuffer extraData = new StringBuffer(); - Object value; - for (DataType dt : dataType) { - value = dataObject.getDataElement(dt); - if (value == null && dt.getMayBeNull()) { - extraData.append(EMPTY_SEPARATOR).append(dt.getFormatInfo().formatData(Locale.US, value, NULL_STRING)); - } else { - extraData.append(EMPTY_SEPARATOR).append(dt.getFormatInfo().formatData(Locale.US, value, NOTHING)); - } - } - return extraData.append(" ").toString(); - } - - /** - * write out the header of the catalog - * - * @param out the writer - * @param dataGroup data group - */ - private static void writeHeader(PrintWriter out, DataGroup dataGroup) { - DataType[] dataType = dataGroup.getDataDefinitions(); - - IpacTableUtil.writeAttributes(out, dataGroup.getKeywords()); - IpacTableUtil.writeHeader(out, Arrays.asList(dataType)); - } - - /** - * write out the header (data name) of the catalog - * - * @param out the writer - * @param dataType array of data types - */ - private static void writeName(PrintWriter out, DataType dataType[]) { - for (DataType dt : dataType) { - DataType.FormatInfo info = dt.getFormatInfo(); - out.print(SEPARATOR + info.formatHeader(dt.getKeyName())); - } - out.print(SEPARATOR + LINE_SEP); - } - - /** - * write out the header (data type) of the catalog - * - * @param out the writer - * @param dataType array of data types - */ - private static void writeDataType(PrintWriter out, - DataType dataType[]) { - for (DataType dt : dataType) { - DataType.FormatInfo info = dt.getFormatInfo(); - - // handle invalid type, throw runtime exeption for now - Assert.tst(IpacTableReader.isRecongnizedType(dt.getTypeDesc()), - "Invalid data Type:" + dt.getTypeDesc()); - - out.print(SEPARATOR + info.formatHeader(dt.getTypeDesc())); - } - out.print(SEPARATOR + LINE_SEP); - } - - /** - * write out the header (data unit) of the catalog - * - * @param out the writer - * @param dataType array of data types - * - */ - private static void writeDataUnit(PrintWriter out, - DataType dataType[]) { - for (DataType dt : dataType) { - DataType.FormatInfo info = dt.getFormatInfo(); - if (dt.getDataUnit() == null ) { - out.print(SEPARATOR + info.formatHeader(NOTHING)); - } else { - out.print(SEPARATOR + info.formatHeader(dt.getDataUnit())); - } - } - out.print(SEPARATOR + LINE_SEP); - } - - /** - * write out the header (may be null) of the catalog - * - * @param out the writer - * @param dataType array of data types - */ - private static void writeIsNullAllowed(PrintWriter out, - DataType dataType[]) { - for (DataType dt : dataType) { - DataType.FormatInfo info = dt.getFormatInfo(); - if (dt.getMayBeNull()) { - out.print(SEPARATOR + info.formatHeader(NULL_STRING)); - } else { - out.print(SEPARATOR + info.formatHeader(NOTHING)); - } - } - out.print(SEPARATOR + LINE_SEP); - } - } diff --git a/src/firefly/java/edu/caltech/ipac/firefly/server/query/HistogramProcessor.java b/src/firefly/java/edu/caltech/ipac/firefly/server/query/HistogramProcessor.java index 5347291b56..59d099fce5 100644 --- a/src/firefly/java/edu/caltech/ipac/firefly/server/query/HistogramProcessor.java +++ b/src/firefly/java/edu/caltech/ipac/firefly/server/query/HistogramProcessor.java @@ -39,10 +39,8 @@ public class HistogramProcessor extends IpacTablePartProcessor { }; static { // set default precision to 9 significant digits - DataType.FormatInfo fi = DataType.FormatInfo.createDefaultFormat(Double.class); - fi.setDataFormat("%.9g"); - columns[1].setFormatInfo(fi); - columns[2].setFormatInfo(fi); + columns[1].getFormatInfo().setDataFormat("%.9g"); + columns[2].getFormatInfo().setDataFormat("%.9g"); } private final String FIXED_SIZE_ALGORITHM = "fixedSizeBins"; private final String NUMBER_BINS = "numBins"; diff --git a/src/firefly/java/edu/caltech/ipac/firefly/server/util/ipactable/DataGroupReader.java b/src/firefly/java/edu/caltech/ipac/firefly/server/util/ipactable/DataGroupReader.java index 4fa6bfcd83..3fc5a7bc89 100644 --- a/src/firefly/java/edu/caltech/ipac/firefly/server/util/ipactable/DataGroupReader.java +++ b/src/firefly/java/edu/caltech/ipac/firefly/server/util/ipactable/DataGroupReader.java @@ -4,6 +4,7 @@ package edu.caltech.ipac.firefly.server.util.ipactable; import edu.caltech.ipac.astro.FITSTableReader; +import edu.caltech.ipac.firefly.data.table.TableMeta; import edu.caltech.ipac.firefly.server.db.spring.mapper.DataGroupUtil; import edu.caltech.ipac.firefly.server.query.TemplateGenerator; import edu.caltech.ipac.firefly.server.util.DsvToDataGroup; @@ -27,12 +28,6 @@ public class DataGroupReader { public static final String LINE_SEP = System.getProperty("line.separator"); private static final Logger.LoggerImpl logger = Logger.getLogger(); - public static enum Format { TSV(CSVFormat.TDF), CSV(CSVFormat.DEFAULT), IPACTABLE(), UNKNOWN(), FIXEDTARGETS(), FITS(), JSON(); - CSVFormat type; - Format() {} - Format(CSVFormat type) {this.type = type;} - } - public static DataGroup readAnyFormat(File inf) throws IOException { return readAnyFormat(inf, 0); } @@ -62,10 +57,10 @@ public static DataGroup readAnyFormat(File inf, int tableIndex) throws IOExcepti throw new IOException("Unsupported format, file:" + inf); } } - + public static DataGroup read(File inf, String... onlyColumns) throws IOException { return read(inf, false, onlyColumns); - + } public static DataGroup read(File inf, boolean readAsString, String... onlyColumns) throws IOException { @@ -77,78 +72,15 @@ public static DataGroup read(File inf, boolean isFixedLength, boolean readAsStri } public static DataGroup read(File inf, boolean isFixedLength, boolean readAsString, boolean saveFormattedData, String... onlyColumns) throws IOException { - TableDef tableDef = IpacTableUtil.getMetaInfo(inf); - List attributes = tableDef.getAllAttributes(); - List cols = tableDef.getCols(); - - if (readAsString) { - for (DataType dt : cols) { - dt.setDataType(String.class); - } - } - - DataGroup headers = new DataGroup(null, cols); - DataGroup data = null; - boolean isSelectedColumns = onlyColumns != null && onlyColumns.length > 0; - - if (isSelectedColumns) { - List selCols = new ArrayList(); - for (String c : onlyColumns) { - DataType dt = headers.getDataDefintion(c); - if (dt != null) { - try { - selCols.add((DataType) dt.clone()); - } catch (CloneNotSupportedException e) {} // shouldn't happen - } - } - data = new DataGroup(null, selCols); - } else { - data = headers; - } - - data.setAttributes(attributes); - - String line = null; - int lineNum = 0; - - BufferedReader reader = new BufferedReader(new FileReader(inf), IpacTableUtil.FILE_IO_BUFFER_SIZE); - try { - line = reader.readLine(); - lineNum++; - while (line != null) { - DataObject row = IpacTableUtil.parseRow(headers, line, isFixedLength, saveFormattedData); - if (row != null) { - if (isSelectedColumns) { - DataObject arow = new DataObject(data); - for (DataType dt : data.getDataDefinitions()) { - arow.setDataElement(dt, row.getDataElement(dt.getKeyName())); - if (dt.getFormatInfo().isDefault()) { - dt.getFormatInfo().setDataFormat( - headers.getDataDefintion(dt.getKeyName()).getFormatInfo().getDataFormatStr()); - } - } - data.add(arow); - } else { - data.add(row); - } - } - line = reader.readLine(); - lineNum++; - } - } catch(Exception e) { - String msg = e.getMessage()+"
on line "+lineNum+": " + line; - if (msg.length()>128) msg = msg.substring(0,128)+"..."; - logger.error(e, "on line "+lineNum+": " + line); - throw new IOException(msg); - } finally { - reader.close(); - } + BufferedReader bufferedReader = new BufferedReader(new FileReader(inf), IpacTableUtil.FILE_IO_BUFFER_SIZE); + return doRead(bufferedReader, tableDef, isFixedLength, readAsString, saveFormattedData, onlyColumns); + } - if (!saveFormattedData) { - data.shrinkToFitData(); - } - return data; + public static DataGroup read(Reader reader, boolean isFixedLength, boolean readAsString, boolean saveFormattedData, String... onlyColumns) throws IOException { + BufferedReader bufferedReader = new BufferedReader(reader, IpacTableUtil.FILE_IO_BUFFER_SIZE); + TableDef tableDef = IpacTableUtil.getMetaInfo(bufferedReader); + return doRead(bufferedReader, tableDef, isFixedLength, readAsString, saveFormattedData, onlyColumns); } public static DataGroup getEnumValues(File inf, int cutoffPoint) throws IOException { @@ -236,7 +168,7 @@ public static Format guessFormat(File inf) throws IOException { } int readAhead = 10; - + int row = 0; BufferedReader reader = new BufferedReader(new FileReader(inf), IpacTableUtil.FILE_IO_BUFFER_SIZE); try { @@ -297,11 +229,91 @@ public static Format guessFormat(File inf) throws IOException { } + private static DataGroup doRead(BufferedReader bufferedReader, TableDef tableDef, boolean isFixedLength, boolean readAsString, boolean saveFormattedData, String... onlyColumns) throws IOException { + + List attributes = tableDef.getAllAttributes(); + List cols = tableDef.getCols(); + + if (readAsString) { + for (DataType dt : cols) { + dt.setDataType(String.class); + } + } + + DataGroup inData = new DataGroup(null, cols); + DataGroup outData = null; + boolean isSelectedColumns = onlyColumns != null && onlyColumns.length > 0; + + if (isSelectedColumns) { + List selCols = new ArrayList(); + for (String c : onlyColumns) { + DataType dt = inData.getDataDefintion(c); + if (dt != null) { + try { + selCols.add((DataType) dt.clone()); + } catch (CloneNotSupportedException e) {} // shouldn't happen + } + } + outData = new DataGroup(null, selCols); + } else { + outData = inData; + } + + outData.setAttributes(attributes); + + String line = null; + int lineNum = tableDef.getExtras() == null ? 0 : tableDef.getExtras().getKey(); + + try { + line = tableDef.getExtras() == null ? bufferedReader.readLine() : tableDef.getExtras().getValue(); + lineNum++; + while (line != null) { + DataObject row = IpacTableUtil.parseRow(inData, line, isFixedLength, saveFormattedData); + if (row != null) { + if (isSelectedColumns) { + DataObject arow = new DataObject(outData); + for (DataType dt : outData.getDataDefinitions()) { + arow.setDataElement(dt, row.getDataElement(dt.getKeyName())); + if (dt.getFormatInfo().isDefault()) { + dt.getFormatInfo().setDataFormat( + inData.getDataDefintion(dt.getKeyName()).getFormatInfo().getDataFormatStr()); + } + } + outData.add(arow); + } else { + outData.add(row); + } + } + line = bufferedReader.readLine(); + lineNum++; + } + } catch(Exception e) { + String msg = e.getMessage()+"
on line "+lineNum+": " + line; + if (msg.length()>128) msg = msg.substring(0,128)+"..."; + logger.error(e, "on line "+lineNum+": " + line); + throw new IOException(msg); + } finally { + bufferedReader.close(); + } + + if (!saveFormattedData) { + outData.shrinkToFitData(); + } + return outData; + + } + //==================================================================== // //==================================================================== + public static enum Format { TSV(CSVFormat.TDF), CSV(CSVFormat.DEFAULT), IPACTABLE(), UNKNOWN(), FIXEDTARGETS(), FITS(), JSON(); + CSVFormat type; + Format() {} + Format(CSVFormat type) {this.type = type;} + } + } diff --git a/src/firefly/java/edu/caltech/ipac/firefly/server/util/ipactable/JsonTableUtil.java b/src/firefly/java/edu/caltech/ipac/firefly/server/util/ipactable/JsonTableUtil.java index 2c98019688..405f747cd6 100644 --- a/src/firefly/java/edu/caltech/ipac/firefly/server/util/ipactable/JsonTableUtil.java +++ b/src/firefly/java/edu/caltech/ipac/firefly/server/util/ipactable/JsonTableUtil.java @@ -115,10 +115,9 @@ public static JSONObject toJsonTableData(DataGroup data, TableDef tableDef) { String fkey = DataSetParser.makeAttribKey(DataSetParser.FORMAT_DISP_TAG, dt.getKeyName()); if (tableDef.contains(fkey)) { dt.getFormatInfo().setDataFormat(tableDef.getAttribute(fkey).getValue()); - dt.getFormatInfo().setWidth(Arrays.stream(new int[]{ - String.valueOf(dt.getKeyName()).length(), - String.valueOf(dt.getDataUnit()).length(), - String.valueOf(dt.getTypeDesc()).length()}).max().getAsInt()); + String[] headers = new String[] {dt.getKeyName(), dt.getTypeDesc(), dt.getDataUnit(), dt.getNullString()}; + int maxLength = Arrays.stream(headers).mapToInt(s -> s == null ? 0 : s.length()).max().getAsInt(); + dt.getFormatInfo().setWidth(maxLength); formatChanged = true; } } diff --git a/src/firefly/java/edu/caltech/ipac/firefly/server/util/ipactable/TableDef.java b/src/firefly/java/edu/caltech/ipac/firefly/server/util/ipactable/TableDef.java index 48bf721137..236a096023 100644 --- a/src/firefly/java/edu/caltech/ipac/firefly/server/util/ipactable/TableDef.java +++ b/src/firefly/java/edu/caltech/ipac/firefly/server/util/ipactable/TableDef.java @@ -4,6 +4,7 @@ import edu.caltech.ipac.util.DataGroup; import edu.caltech.ipac.util.DataType; import edu.caltech.ipac.util.StringUtils; +import javafx.util.Pair; import java.util.ArrayList; import java.util.Collections; @@ -24,6 +25,7 @@ public class TableDef { private int rowCount; private int rowStartOffset; private int lineSepLength; + private transient Pair extras; // used by IpacTableUtil to store extras data while parsing an ipac table via input stream public void addAttributes(DataGroup.Attribute... attributes) { if (attributes != null) { @@ -39,12 +41,20 @@ public void addAttributes(DataGroup.Attribute... attributes) { } } - public void setCols(List cols) { this.cols = cols; } + public Pair getExtras() { + return extras; + } + + public void setExtras(Integer numHeaderLines, String unreadLine) { + this.extras = new Pair<>(numHeaderLines, unreadLine); + } public List getCols() { return cols; } + public void setCols(List cols) { this.cols = cols; } + /** * returns all of the attributes including comments. This function returns the attributes * in the order it was added. @@ -71,10 +81,6 @@ public void removeAttribute(String name) { } } - public void setStatus(DataGroupPart.State status) { - addAttributes(new DataGroup.Attribute(DataGroupPart.LOADING_STATUS, status.name())); - } - DataGroup.Attribute getAttribute(String key) { return attributes.get(key); } @@ -92,6 +98,10 @@ public DataGroupPart.State getStatus() { return DataGroupPart.State.valueOf(String.valueOf(a.getValue())); } + public void setStatus(DataGroupPart.State status) { + addAttributes(new DataGroup.Attribute(DataGroupPart.LOADING_STATUS, status.name())); + } + public int getLineSepLength() { return lineSepLength; } diff --git a/src/firefly/java/edu/caltech/ipac/util/DataGroup.java b/src/firefly/java/edu/caltech/ipac/util/DataGroup.java index a0eee023f6..efee2daff0 100644 --- a/src/firefly/java/edu/caltech/ipac/util/DataGroup.java +++ b/src/firefly/java/edu/caltech/ipac/util/DataGroup.java @@ -129,6 +129,8 @@ public void shrinkToFitData(boolean force) { for (DataType dt : getDataDefinitions()) { int maxDataWidth = dt.getMaxDataWidth(); if (force || maxDataWidth == 0) { + String[] headers = new String[] {dt.getKeyName(), dt.getTypeDesc(), dt.getDataUnit(), dt.getNullString()}; + maxDataWidth = Arrays.stream(headers).mapToInt(s -> s == null ? 0 : s.length()).max().getAsInt(); for (DataObject row : this) { int vlength = row.getDataWidth(dt); maxDataWidth = Math.max(maxDataWidth, vlength); diff --git a/src/firefly/java/edu/caltech/ipac/util/DataObject.java b/src/firefly/java/edu/caltech/ipac/util/DataObject.java index bd0b612881..33efb3efa7 100644 --- a/src/firefly/java/edu/caltech/ipac/util/DataObject.java +++ b/src/firefly/java/edu/caltech/ipac/util/DataObject.java @@ -115,8 +115,7 @@ public String getFormatedData(DataType dt) { val = _formattedData[idx]; } else { Object v = getDataElement(dt); - String strForNull = StringUtils.isEmpty(dt.getNullString()) ? "" : dt.getNullString(); - val = dt.getFormatInfo().formatData(v, strForNull); + val = dt.getFormatInfo().formatData(v); } return val; } diff --git a/src/firefly/java/edu/caltech/ipac/util/DataType.java b/src/firefly/java/edu/caltech/ipac/util/DataType.java index 1a62c71349..ee6628d252 100644 --- a/src/firefly/java/edu/caltech/ipac/util/DataType.java +++ b/src/firefly/java/edu/caltech/ipac/util/DataType.java @@ -24,6 +24,7 @@ public enum Importance { HIGH, MEDIUM, LOW, IGNORE} private static final String S_LONG = "l"; private static final String S_CHAR = "c"; private static final String S_BOOL = "b"; + private static final String DEF_NULL_STR = ""; private Class _type; private String _units; @@ -129,6 +130,7 @@ public String getNullString() { public void setNullString(String nullString) { _nullString = nullString; + getFormatInfo().setNullString(_nullString); } public void setDataType(Class type) { _type= type; } @@ -269,12 +271,14 @@ public boolean hasFormatInfo() { public FormatInfo getFormatInfo() { if ( _formatInfo == null ) { _formatInfo = FormatInfo.createDefaultFormat(getDataType()); + _formatInfo.setNullString(getNullString()); } return _formatInfo; } public void setFormatInfo(FormatInfo info) { _formatInfo = info; + _formatInfo.setNullString(_nullString); } private String importanceAsString() { @@ -326,12 +330,12 @@ public enum Align {LEFT, RIGHT} private static final int DEF_WIDTH = 30; - private static final String NULL_STR = ""; private Align _headerAlign; private Align _dataAlign; private int _width; private String _dataFormat; private boolean _isDefault; + private String _nullString; /** * Creates a new FormatInfo good for formatting String. @@ -366,6 +370,10 @@ public FormatInfo(int width, Align headerAlign, Align dataAlign) { } + public void setNullString(String nullString) { + _nullString = nullString == null ? DEF_NULL_STR : nullString; + } + void setIsDefault(boolean aDefault) { _isDefault = aDefault; } @@ -423,7 +431,7 @@ public String formatDataOnly(Object value) { * @param value */ public String formatDataOnly(Locale locale, Object value) { - if (value == null) return NULL_STR; + if (value == null) return _nullString; return getDataFormatStr() == null ? String.valueOf(value) : String.format(locale, getDataFormatStr(), value); } @@ -438,23 +446,14 @@ public String formatData(Object value) { return formatData(Locale.getDefault(), value); } - public String formatData(Object value, String strForNull) { - return formatData(Locale.getDefault(), value, strForNull); - } - /** * Overloaded to format the data using a given locale. * @param locale * @param value */ public String formatData(Locale locale, Object value) { - return formatData(locale, value, ""); - } - - public String formatData(Locale locale, Object value, String strForNull) { String fmtStr = (getDataAlign() == Align.LEFT ? "%-" : "%") + getWidth() + "." + getWidth()+ "s"; - return String.format(locale, fmtStr, (value == null) ? strForNull : formatDataOnly(locale, value)); - + return String.format(locale, fmtStr, formatDataOnly(locale, value)); } /** @@ -463,7 +462,7 @@ public String formatData(Locale locale, Object value, String strForNull) { * @return a string */ public String formatHeader(String value) { - String v = value == null ? "" : value; + String v = value == null ? _nullString : value; String fmtStr = (getHeaderAlign() == Align.LEFT ? "%-" : "%") + getWidth() + "." + getWidth() + "s"; return String.format(fmtStr, v); } diff --git a/src/firefly/java/edu/caltech/ipac/util/IpacTableUtil.java b/src/firefly/java/edu/caltech/ipac/util/IpacTableUtil.java index 01a4e3f407..ad3a15eda8 100644 --- a/src/firefly/java/edu/caltech/ipac/util/IpacTableUtil.java +++ b/src/firefly/java/edu/caltech/ipac/util/IpacTableUtil.java @@ -28,6 +28,7 @@ public class IpacTableUtil { public static final int FILE_IO_BUFFER_SIZE = FileUtil.BUFFER_SIZE; + private static final String STRING_TYPE[]= {"cha.*", "str.*", "s", "c"}; public static void writeAttributes(PrintWriter writer, Collection attribs, String... ignoreList) { if (attribs == null) return; @@ -169,13 +170,14 @@ public static void setDataNullStr(List cols, String line) { if (line != null && line.startsWith("|")) { String[] nullables = parseHeadings(line.trim()); for(int i = 0; i < nullables.length; i++) { - cols.get(i).setNullString(nullables[i].trim()); + String nullString = nullables[i].trim(); + cols.get(i).setNullString(nullString); } } } public static void guessFormatInfo(DataType dataType, String value) { - String formatStr = IpacTableReader.guessFormatStr(dataType, value.trim()); + String formatStr = guessFormatStr(dataType, value.trim()); if (formatStr != null) { DataType.FormatInfo.Align align = value.startsWith(" ") ? DataType.FormatInfo.Align.RIGHT : DataType.FormatInfo.Align.LEFT; @@ -310,18 +312,20 @@ public static Map asMap(DataObject row) { } public static TableDef getMetaInfo(File inf) throws IOException { - FileReader infReader = null; + BufferedReader reader = null; try { - infReader = new FileReader(inf); - return getMetaInfo(new BufferedReader(infReader,FILE_IO_BUFFER_SIZE), inf); - + reader = new BufferedReader(new FileReader(inf), FILE_IO_BUFFER_SIZE); + return doGetMetaInfo(reader, inf); } finally { - FileUtil.silentClose(infReader); + FileUtil.silentClose(reader); } + } + public static TableDef getMetaInfo(BufferedReader reader) throws IOException { + return doGetMetaInfo(reader, null); } - public static TableDef getMetaInfo(BufferedReader reader, File src) throws IOException { + private static TableDef doGetMetaInfo(BufferedReader reader, File src) throws IOException { TableDef meta = new TableDef(); int nlchar = findLineSepLength(reader); meta.setLineSepLength(nlchar); @@ -330,9 +334,12 @@ public static TableDef getMetaInfo(BufferedReader reader, File src) throws IOExc List cols = null; int dataStartOffset = 0; + int lineNum = 0; + String line = reader.readLine(); // skip to column desc while (line != null) { + lineNum++; if (line.length() == 0) { // skip empty line dataStartOffset += line.length() + nlchar; @@ -373,6 +380,10 @@ public static TableDef getMetaInfo(BufferedReader reader, File src) throws IOExc } else { // data row begins. meta.setLineWidth(line.length() + nlchar); + if (src == null) { + // reading from stream, store the line read for later processing. + meta.setExtras(lineNum-1, line); + } break; } } @@ -413,6 +424,49 @@ public static void sendLoadStatusEvents(Map meta, File outf, int ServerEventManager.fireAction(action); } + public static String guessFormatStr(DataType type, String val) { + if (type.getTypeDesc() != null && + ServerStringUtil.matchesRegExpList(type.getTypeDesc(), STRING_TYPE, true)) { + return "%s"; + } else { + return guessFormatStr(val, type.getDataType()); + } + } + + private static String guessFormatStr(String val, Class cls) { + + String formatStr = null; + try { + //first check to see if it's numeric + double numval = Double.parseDouble(val); + + if (Double.isNaN(numval)) { + return null; + } else { + if (val.matches(".+[e|E].+")) { + // scientific notation + String convStr = val.indexOf("E") >= 0 ? "E" : "e"; + String[] ary = val.split("e|E"); + if (ary.length == 2) { + int prec = ary[0].length() - ary[0].indexOf(".") - 1; + return "%." + prec + convStr; + } + } else if (val.indexOf(".") >= 0) { + // decimal format + int idx = val.indexOf("."); + int prec = val.length() - idx - 1; + return "%." + prec + "f"; + } else { + boolean isFloat= (cls==Float.class || cls==Double.class); + formatStr = isFloat ? "%.0f" : "%d"; + } + } + } catch (NumberFormatException e) { + formatStr = "%s"; + } + return formatStr; + } + //==================================================================== // //==================================================================== diff --git a/src/firefly/js/tables/TablesCntlr.js b/src/firefly/js/tables/TablesCntlr.js index 7d779d79a9..ef5ae1ddfb 100644 --- a/src/firefly/js/tables/TablesCntlr.js +++ b/src/firefly/js/tables/TablesCntlr.js @@ -152,7 +152,7 @@ export function tableFetch(action) { const {tbl_id} = request; dispatchAddSaga( doOnTblLoaded, {tbl_id, callback:() => dispatchTableLoaded( Object.assign(TblUtil.getTblInfoById(tbl_id), {invokedBy: action.type}) )}); - dispatch( updateMerge(action, 'payload', {tbl_id, isFetching: true}) ); + dispatch( updateMerge(action, 'payload', {tbl_id}) ); request.startIdx = 0; TblUtil.doFetchTable(request, hlRowIdx).then ( (tableModel) => { dispatch( {type: TABLE_UPDATE, payload: tableModel} ); diff --git a/src/firefly/js/tables/reducer/TableDataReducer.js b/src/firefly/js/tables/reducer/TableDataReducer.js index 5e4cd2da19..74ed10090d 100644 --- a/src/firefly/js/tables/reducer/TableDataReducer.js +++ b/src/firefly/js/tables/reducer/TableDataReducer.js @@ -32,12 +32,23 @@ export function dataReducer(state={data:{}}, action={}) { case (Cntlr.TABLE_HIGHLIGHT) : case (Cntlr.TABLE_UPDATE) : { - const {tbl_id} = action.payload; - return TblUtil.smartMerge(root, {[tbl_id] : {isFetching:false, ...action.payload}}); + var {tbl_id, totalRows} = action.payload; + const updates = {[tbl_id] : {isFetching:false, ...action.payload}}; + if (totalRows) { // update selectInfo.rowCount + var selectInfo = get(TblUtil.getTblById(tbl_id), 'selectInfo'); + selectInfo = selectInfo ? {...selectInfo, rowCount: totalRows} : SelectInfo.newInstance({rowCount: totalRows}).data; + updates[tbl_id].selectInfo = selectInfo; + } + return TblUtil.smartMerge(root, updates); } case (Cntlr.TABLE_FETCH) : case (Cntlr.TABLE_FILTER) : case (Cntlr.TABLE_SORT) : + { + const {tbl_id} = action.payload || {}; + const nTable = Object.assign({isFetching:true, selectInfo: SelectInfo.newInstance({rowCount:0}).data}, action.payload); + return updateSet(root, [tbl_id], nTable); + } case (Cntlr.TABLE_REPLACE) : { const {tbl_id} = action.payload || {}; diff --git a/src/firefly/js/tables/ui/BasicTableView.jsx b/src/firefly/js/tables/ui/BasicTableView.jsx index 710490dc6d..52e20d6a41 100644 --- a/src/firefly/js/tables/ui/BasicTableView.jsx +++ b/src/firefly/js/tables/ui/BasicTableView.jsx @@ -174,7 +174,7 @@ export class BasicTableView extends React.Component { } {showMask &&
} - {!showMask && isEmpty(data) &&
No Data Found
} + {!showMask && isEmpty(data) &&
No Data Found
} ); } diff --git a/src/firefly/js/tables/ui/TablePanel.css b/src/firefly/js/tables/ui/TablePanel.css index 687df4442b..5a86a47f53 100644 --- a/src/firefly/js/tables/ui/TablePanel.css +++ b/src/firefly/js/tables/ui/TablePanel.css @@ -1,15 +1,18 @@ -.tablePanel__checkbox { +.TablePanel { display: flex; height: 100%; - align-items: center; - justify-content: center; + flex-direction: column; + overflow: hidden; } -.tablePanel__checkbox--cell { - background-color: inherit; +.TablePanel__checkbox { + display: flex; + height: 100%; + align-items: center; + justify-content: center; } -.tablePanel_NoData { +.TablePanel_NoData { position: absolute; top: 50%; left: 42%; @@ -18,10 +21,6 @@ font-size: large; } -.fixedDataTableRowLayout_rowWrapper .tablePanel__Row_highlighted { - background-color: #d3fad1 -} - .TablePanel__wrapper { position: relative; background-color: #c8c8c8; @@ -97,51 +96,55 @@ background-image: linear-gradient(#fff, #dcdcdc); } -button.paging_bar { +.PagingBar__button { height: 16px; width: 16px; margin-top: 1px; } -button.tablepanel { - height: 24px; - width: 24px; - margin: 0 2px; +.PagingBar__button:hover { + cursor: pointer; } -button.paging_bar.first { +.PagingBar__button.first { background-image: url("/images/icons-2014/16x16_BackwardToEnd.png"); } -button.paging_bar.last { +.PagingBar__button.last { background-image: url("/images/icons-2014/16x16_ForwardToEnd.png"); } -button.paging_bar.next { +.PagingBar__button.next { background-image: url("/images/icons-2014/16x16_Forward.png"); } -button.paging_bar.previous { +.PagingBar__button.previous { background-image: url("/images/icons-2014/16x16_Backward.png"); } -button.tablepanel.options { +.TablePanel__button { + height: 24px; + width: 24px; + margin: 0 2px; +} + +.TablePanel__button.options { background-image: url("/images/icons-2014/24x24_GearsNEW.png"); } -button.tablepanel.textView { +.TablePanel__button.textView { background-image: url("/images/icons-2014/24x24_TextView.png"); } -button.tablepanel.tableView { +.TablePanel__button.tableView { background-image: url("/images/icons-2014/24x24_TableView.png"); } -button.tablepanel.save { +.TablePanel__button.save { background-image: url("/images/icons-2014/24x24_Save.png"); } -button.tablepanel.clearFilters { +.TablePanel__button.clearFilters { background-image: url("/images/icons-2014/24x24_FilterOff_Circle.png"); } @@ -545,4 +548,9 @@ body[dir="rtl"] .fixedDataTableColumnResizerLineLayout_main, .fixedDataTableColu .fixedDataTableRowLayout_rowWrapper { position: absolute; top: 0 -} \ No newline at end of file +} + +.fixedDataTableRowLayout_rowWrapper .tablePanel__Row_highlighted { + background-color: #d3fad1 +} + diff --git a/src/firefly/js/tables/ui/TablePanel.jsx b/src/firefly/js/tables/ui/TablePanel.jsx index 91bec795d6..7c9ba99708 100644 --- a/src/firefly/js/tables/ui/TablePanel.jsx +++ b/src/firefly/js/tables/ui/TablePanel.jsx @@ -123,23 +123,23 @@ export class TablePanel extends Component { if (isEmpty(columns)) return ; const selectInfoCls = SelectInfo.newInstance(selectInfo, startIdx); - const viewIcoStyle = 'tablepanel ' + (textView ? 'tableView' : 'textView'); + const viewIcoStyle = 'TablePanel__button ' + (textView ? 'tableView' : 'textView'); const tableTopPos = showToolbar ? 29 : 0; const TT_VIEW = textView ? TT_TABLE_VIEW : TT_TEXT_VIEW; return (
-
+
{showToolbar &&
- {showTitle ? :
} + {showTitle ? :
} {showPaging && }
{showFilterButton && filterCount > 0 &&