diff --git a/src/firefly/java/edu/caltech/ipac/firefly/server/db/BaseDbAdapter.java b/src/firefly/java/edu/caltech/ipac/firefly/server/db/BaseDbAdapter.java
index d12fddd8df..1193d88ad5 100644
--- a/src/firefly/java/edu/caltech/ipac/firefly/server/db/BaseDbAdapter.java
+++ b/src/firefly/java/edu/caltech/ipac/firefly/server/db/BaseDbAdapter.java
@@ -9,7 +9,6 @@
import edu.caltech.ipac.firefly.server.util.Logger;
import edu.caltech.ipac.table.DataType;
import edu.caltech.ipac.util.StringUtils;
-import org.apache.xpath.operations.Bool;
import java.io.File;
import java.util.ArrayList;
@@ -19,6 +18,7 @@
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.regex.Pattern;
import java.util.stream.Collectors;
import static edu.caltech.ipac.firefly.data.TableServerRequest.INCL_COLUMNS;
@@ -134,10 +134,14 @@ public String wherePart(TableServerRequest treq) {
if (treq.getFilters() != null && treq.getFilters().size() > 0) {
where = "";
for (String cond :treq.getFilters()) {
- if (cond.matches("(?i).* LIKE .*(\\\\_|\\\\%|\\\\\\\\).*")) { // search for LIKE w/ \_, \%, or \\ in the condition.
+ if (cond.matches("(?i).* LIKE .*(\\\\_|\\\\%|\\\\\\\\).*")) { // search for LIKE with \_, \%, or \\ in the condition.
// for LIKE, to search for '%', '\' or '_' itself, an escape character must also be specified using the ESCAPE clause
cond += " ESCAPE '\\'";
}
+ String[] parts = StringUtils.groupMatch("(.+) IN (.+)", cond, Pattern.CASE_INSENSITIVE);
+ if (parts != null && cond.contains(NULL_TOKEN)) {
+ cond = String.format("%s OR %s IS NULL", cond.replace(NULL_TOKEN, NULL_TOKEN.substring(1)), parts[0]);
+ }
if (where.length() > 0) {
where += " and ";
}
diff --git a/src/firefly/java/edu/caltech/ipac/firefly/server/db/DbAdapter.java b/src/firefly/java/edu/caltech/ipac/firefly/server/db/DbAdapter.java
index 00a81e69e6..dff21f302a 100644
--- a/src/firefly/java/edu/caltech/ipac/firefly/server/db/DbAdapter.java
+++ b/src/firefly/java/edu/caltech/ipac/firefly/server/db/DbAdapter.java
@@ -24,6 +24,7 @@ public interface DbAdapter {
String HSQL = "hsql";
String MAIN_DB_TBL = "DATA";
+ String NULL_TOKEN = "%NULL";
/*
CLEAN UP POLICY:
diff --git a/src/firefly/java/edu/caltech/ipac/firefly/server/db/EmbeddedDbUtil.java b/src/firefly/java/edu/caltech/ipac/firefly/server/db/EmbeddedDbUtil.java
index 42e878258f..a2f23f0d23 100644
--- a/src/firefly/java/edu/caltech/ipac/firefly/server/db/EmbeddedDbUtil.java
+++ b/src/firefly/java/edu/caltech/ipac/firefly/server/db/EmbeddedDbUtil.java
@@ -433,7 +433,7 @@ private static int dbToDD(DataGroup dg, ResultSet rs) {
applyIfNotEmpty(rs.getString("label"), dtype::setLabel);
applyIfNotEmpty(rs.getString("units"), dtype::setUnits);
- applyIfNotEmpty(rs.getString("null_str"), dtype::setNullString);
+ dtype.setNullString(rs.getString("null_str"));
applyIfNotEmpty(rs.getString("format"), dtype::setFormat);
applyIfNotEmpty(rs.getString("fmtDisp"), dtype::setFmtDisp);
applyIfNotEmpty(rs.getInt("width"), dtype::setWidth);
diff --git a/src/firefly/java/edu/caltech/ipac/firefly/server/query/AsyncSearchProcessor.java b/src/firefly/java/edu/caltech/ipac/firefly/server/query/AsyncSearchProcessor.java
index 3206414c94..21cc6e7614 100644
--- a/src/firefly/java/edu/caltech/ipac/firefly/server/query/AsyncSearchProcessor.java
+++ b/src/firefly/java/edu/caltech/ipac/firefly/server/query/AsyncSearchProcessor.java
@@ -33,6 +33,7 @@ public DataGroup fetchDataGroup(TableServerRequest req) throws DataAccessExcepti
case COMPLETED:
return asyncJob.getDataGroup();
case ERROR:
+ case UNKNOWN:
throw new DataAccessException(asyncJob.getErrorMsg());
case ABORTED:
throw new DataAccessException("Query aborted");
diff --git a/src/firefly/java/edu/caltech/ipac/firefly/server/query/EmbeddedDbProcessor.java b/src/firefly/java/edu/caltech/ipac/firefly/server/query/EmbeddedDbProcessor.java
index 2bb9822287..a6c1709fa5 100644
--- a/src/firefly/java/edu/caltech/ipac/firefly/server/query/EmbeddedDbProcessor.java
+++ b/src/firefly/java/edu/caltech/ipac/firefly/server/query/EmbeddedDbProcessor.java
@@ -55,6 +55,7 @@
import static edu.caltech.ipac.firefly.data.table.MetaConst.HIGHLIGHTED_ROW_BY_ROWIDX;
import static edu.caltech.ipac.firefly.server.ServerContext.SHORT_TASK_EXEC;
import static edu.caltech.ipac.firefly.server.db.DbAdapter.MAIN_DB_TBL;
+import static edu.caltech.ipac.firefly.server.db.DbAdapter.NULL_TOKEN;
import static edu.caltech.ipac.firefly.server.db.EmbeddedDbUtil.execRequestQuery;
import static edu.caltech.ipac.util.StringUtils.isEmpty;
@@ -586,9 +587,8 @@ private static void enumeratedValuesCheck(File dbFile, DataGroupPart results, Ta
.queryForList(String.format("SELECT distinct \"%s\" FROM data order by 1", cname));
if (vals.size() <= MAX_COL_ENUM_COUNT) {
- DataType dt = results.getData().getDataDefintion(cname);
- String enumVals = vals.stream().map(m -> String.valueOf(m.get(cname))) // list of map to list of string(colname)
- .filter(s -> !isEmpty(s) && !StringUtils.areEqual(dt.getNullString(), s)) // remove null or blank values because it's hard to handle at the column filter level
+ String enumVals = vals.stream()
+ .map(m -> m.get(cname) == null ? NULL_TOKEN : m.get(cname).toString()) // list of map to list of string(colname)
.collect(Collectors.joining(",")); // combine the names into comma separated string.
results.getData().getDataDefintion(cname).setEnumVals(enumVals);
// update dd table
diff --git a/src/firefly/java/edu/caltech/ipac/table/DataType.java b/src/firefly/java/edu/caltech/ipac/table/DataType.java
index 1874881ec3..95150dacf2 100644
--- a/src/firefly/java/edu/caltech/ipac/table/DataType.java
+++ b/src/firefly/java/edu/caltech/ipac/table/DataType.java
@@ -52,7 +52,7 @@ public enum Visibility {show, hide, hidden};
private String typeDesc;
private Class type;
private String units;
- private String nullString = "";
+ private String nullString;
private String desc;
private int width;
private int prefWidth;
@@ -142,7 +142,7 @@ public void setUnits(String units) {
}
public String getNullString() {
- return nullString;
+ return nullString == null ? "" : nullString;
}
public void setNullString(String nullString) {
@@ -305,9 +305,8 @@ public String getRef() {
* @return
*/
public String format(Object value, boolean replaceCtrl) {
- if (value == null) {
- return getNullString() == null ? "" : getNullString();
- }
+ if (value == null) return getNullString();
+
// do escaping if requested
if (replaceCtrl && type == String.class) {
value = replaceCtrl((String)value);
@@ -419,8 +418,7 @@ public boolean isKnownType() {
* @return an object
*/
public Object convertStringToData(String s) {
- if (s == null || s.length() == 0 || s.equalsIgnoreCase("null")) return null;
- if (nullString != null && nullString.equals(s)) return null;
+ if (s == null || getNullString().equals(s)) return null;
Object retval= s;
try {
diff --git a/src/firefly/java/edu/caltech/ipac/table/IpacTableUtil.java b/src/firefly/java/edu/caltech/ipac/table/IpacTableUtil.java
index fdf6d1475c..c85afb3fd3 100644
--- a/src/firefly/java/edu/caltech/ipac/table/IpacTableUtil.java
+++ b/src/firefly/java/edu/caltech/ipac/table/IpacTableUtil.java
@@ -246,6 +246,7 @@ public static IpacTableDef createColumnDefs(String line) {
for (int idx = 0; idx < names.length; idx++) {
cname = names[idx];
DataType dt = new DataType(cname.trim(), null);
+ dt.setNullString("null"); // defaults to 'null'
cols.add(dt);
tableDef.setColOffsets(idx, cursor);
cursor += cname.length() + 1;
diff --git a/src/firefly/java/edu/caltech/ipac/util/StringUtils.java b/src/firefly/java/edu/caltech/ipac/util/StringUtils.java
index b8362a763a..b6bd5b71ab 100644
--- a/src/firefly/java/edu/caltech/ipac/util/StringUtils.java
+++ b/src/firefly/java/edu/caltech/ipac/util/StringUtils.java
@@ -74,7 +74,8 @@ public static String[] groupMatch(String regex, String val) {
* an array of strings, otherwise null.
* @param regex pattern to match
* @param val string value to match with
- * @return
+ * @param flags Match flags, a bit mask
+ * @return return all of the matching groups as an array of strings, otherwise null.
*/
public static String[] groupMatch(String regex, String val, int flags) {
return groupMatch(Pattern.compile(regex, flags), val);
diff --git a/src/firefly/js/tables/FilterInfo.js b/src/firefly/js/tables/FilterInfo.js
index ba3e28324c..de276f1c53 100644
--- a/src/firefly/js/tables/FilterInfo.js
+++ b/src/firefly/js/tables/FilterInfo.js
@@ -4,7 +4,7 @@
import {getColumnIdx, getColumn, isNumericType, getTblById, stripColumnNameQuotes} from './TableUtil.js';
import {Expression} from '../util/expr/Expression.js';
-import {isUndefined, get, isArray, isEmpty} from 'lodash';
+import {isNil, get, isArray, isEmpty} from 'lodash';
import {showInfoPopup} from '../ui/PopupUtil.jsx';
const operators = /(!=|>=|<=|<|>|=| like | in | is not | is )/i;
@@ -23,6 +23,8 @@ export const FILTER_TTIPS =
Examples: "ra" > 12345; "color" != 'blue'; "band" IN (1,2,3)
`;
+export const NULL_TOKEN = '%NULL'; // need to match DbAdapter.NULL_TOKEN
+
/**
* return [column_name, operator, value] triplet.
* @param {string} input
@@ -191,7 +193,7 @@ export class FilterInfo {
// remove the double quote or the single quote around cname and val (which is added in auto-correction)
const removeQuoteAroundString = (str, quote = "'") => {
- if (str.startsWith(quote)) {
+ if (str && str.startsWith(quote)) {
const reg = new RegExp('^' + quote + '(.*)' + quote + '$');
return str.replace(reg, '$1');
} else {
@@ -223,7 +225,7 @@ export class FilterInfo {
if (val.match(/^\(.*\)$/)) {
val = val.substring(1, val.length-1);
}
- val = val.split(',').map((s) => s.trim());
+ val = val.split(',').map((s) => removeQuoteAroundString(s.trim()) === NULL_TOKEN.toLowerCase() ? null : s.trim());
}
// remove single quote enclosing the string for the value of operater 'like' or char type column
@@ -236,12 +238,12 @@ export class FilterInfo {
return (row, idx) => {
if (!row) return false;
let compareTo = noROWID ? idx : row[cidx];
- if (isUndefined(compareTo)) return false;
+ compareTo = (compareTo === get(getColumn(tableModel, cname), 'nullString', '')) ? null : compareTo; // resolve nullString
if (op !== 'like' && colType.match(/^[dfil]/)) { // int, float, double, long .. or their short form.
- compareTo = Number(compareTo);
+ compareTo = compareTo ? Number(compareTo) : compareTo;
} else {
- compareTo = compareTo.toLowerCase();
+ compareTo = compareTo ? compareTo.toLowerCase() : compareTo;
}
switch (op) {
@@ -262,6 +264,10 @@ export class FilterInfo {
return compareTo <= val;
case 'in' :
return val.includes(compareTo);
+ case 'is' :
+ return val === 'null' && isNil(compareTo);
+ case 'is not' :
+ return val === 'null' && !isNil(compareTo);
default :
return false;
}
@@ -440,7 +446,7 @@ function autoCorrectCondition(v, isNumeric=false) {
switch (op) {
case 'like':
if (!val.match(/^'.*'$/)) {
- val = val.replace(/([_|%|\\])/g, '\\$1');
+ val = val.replace(/([_|%\\])/g, '\\$1');
val = encloseByQuote(encloseByQuote(val, '%'));
}
diff --git a/src/firefly/js/tables/TableUtil.js b/src/firefly/js/tables/TableUtil.js
index 4d105ec4ed..64de25beb3 100644
--- a/src/firefly/js/tables/TableUtil.js
+++ b/src/firefly/js/tables/TableUtil.js
@@ -533,12 +533,16 @@ export function sortTableData(tableData, columns, sortInfoStr) {
var comparator;
if (!col.type || ['char', 'c'].includes(col.type) ) {
comparator = (r1, r2) => {
- const [s1, s2] = [r1[colIdx], r2[colIdx]];
+ let [s1, s2] = [r1[colIdx], r2[colIdx]];
+ s1 = s1 === '' ? '\u0002' : s1 === null ? '\u0001' : isUndefined(s1) ? '\u0000' : s1;
+ s2 = s2 === '' ? '\u0002' : s2 === null ? '\u0001' : isUndefined(s2) ? '\u0000' : s2;
return multiplier * (s1 > s2 ? 1 : -1);
};
} else {
comparator = (r1, r2) => {
- const [v1, v2] = [r1[colIdx], r2[colIdx]];
+ let [v1, v2] = [r1[colIdx], r2[colIdx]];
+ v1 = v1 === null ? -Number.MAX_VALUE : isUndefined(v1) ? Number.NEGATIVE_INFINITY : Number(v1);
+ v2 = v2 === null ? -Number.MAX_VALUE : isUndefined(v2) ? Number.NEGATIVE_INFINITY : Number(v2);
return multiplier * (Number(v1) - Number(v2));
};
}
@@ -895,8 +899,13 @@ export function tableDetailsView(tbl_id, highlightedRow, details_tbl_id) {
*/
export function calcColumnWidths(columns, dataAry) {
return columns.map( (cv, idx) => {
+
+ let width = cv.prefWidth || cv.width;
+ if (width) {
+ return width;
+ }
const cname = cv.label || cv.name;
- var width = Math.max(cname.length, get(cv, 'units.length', 0), get(cv, 'type.length', 0));
+ width = Math.max(cname.length, get(cv, 'units.length', 0), get(cv, 'type.length', 0));
width = dataAry.reduce( (maxWidth, row) => {
return Math.max(maxWidth, get(row, [idx, 'length'], 0));
}, width); // max width of data
diff --git a/src/firefly/js/tables/__tests__/FilterInfo-test.js b/src/firefly/js/tables/__tests__/FilterInfo-test.js
new file mode 100644
index 0000000000..1043f58712
--- /dev/null
+++ b/src/firefly/js/tables/__tests__/FilterInfo-test.js
@@ -0,0 +1,47 @@
+
+import * as TblUtil from '../TableUtil.js'; // used for named import
+import {FilterInfo} from '../FilterInfo.js';
+
+describe('FilterInfo', () => {
+
+ test('autoCorrectConditions: string columns', () => {
+
+ /*
+ * This test is a little trickier. conditionValidator() takes a tbl_id that uses TblUtil.getTblById to resolve a tableModel.
+ * This involves a fully functional redux store which is not available to our JS test environment at the moment.
+ * So, we will 'fake' the return value of getTblById by 'mocking' that function
+ */
+ const aStringColumn = {
+ tableData: {
+ columns: [ {name: 'desc', type: 'char'}],
+ }
+ };
+
+ TblUtil.getTblById = jest.fn().mockReturnValue(aStringColumn); // mock getTblById to return the aStringColumn table
+
+ let actual = FilterInfo.conditionValidator('=abc', 'a_fake_tbl_id', 'desc');
+ expect(actual.valid).toBe(true);
+ expect(actual.value).toBe("= 'abc'"); // the validator correctly insert space and quote around a value of a string column.
+
+ actual = FilterInfo.conditionValidator('abc', 'a_fake_tbl_id', 'desc');
+ expect(actual.valid).toBe(true);
+ expect(actual.value).toBe("like '%abc%'"); // the validator correctly convert it into a LIKE operator
+ });
+
+ test('autoCorrectConditions: numeric columns', () => {
+
+ const aNumericColumn = {
+ tableData: {
+ columns: [ {name: 'ra', type: 'double'}],
+ }
+ };
+
+ TblUtil.getTblById = jest.fn().mockReturnValue(aNumericColumn); // once again mock getTblById to return the different (numeric) table
+
+ const {valid, value} = FilterInfo.conditionValidator('>1.23', 'a_fake_tbl_id', 'ra');
+ expect(valid).toBe(true);
+ expect(value).toBe('> 1.23'); // the validator correctly insert space and no quotes on numeric columns.
+ });
+
+});
+
diff --git a/src/firefly/js/tables/__tests__/TableUtil-test.js b/src/firefly/js/tables/__tests__/TableUtil-test.js
index 3e9a38ce71..1162332a1e 100644
--- a/src/firefly/js/tables/__tests__/TableUtil-test.js
+++ b/src/firefly/js/tables/__tests__/TableUtil-test.js
@@ -2,15 +2,14 @@ import {get} from 'lodash';
import * as TblUtil from '../TableUtil.js'; // used for named import
import TableUtil from '../TableUtil.js'; // using default import
-import {FilterInfo} from '../FilterInfo.js';
import {SelectInfo} from '../SelectInfo';
import {MetaConst} from '../../data/MetaConst';
-import {hasRowAccess} from '../TableUtil';
import {dataReducer} from '../reducer/TableDataReducer.js';
import {TABLE_LOADED} from '../TablesCntlr.js';
+import {NULL_TOKEN} from '../FilterInfo.js';
-describe('TableUtil:', () => {
+describe('TableUtil: ', () => {
test('isTableLoaded', () => {
// a simple test to ensure when a table is loaded.
@@ -96,10 +95,15 @@ describe('TableUtil:', () => {
});
});
+});
+
+
+describe('TableUtil: datarights', () => {
+
test('DATARIGHTS_COL', () => {
const table = {
tableMeta: {[MetaConst.DATARIGHTS_COL]: 'a'},
- totalRows: 6,
+ totalRows: 7,
tableData: {
columns: [ {name: 'a'}, {name: 'b'}, {name: 'c'}],
data: [
@@ -114,13 +118,13 @@ describe('TableUtil:', () => {
}
};
- expect(hasRowAccess(table, 0)).toBe(true);
- expect(hasRowAccess(table, 1)).toBe(false);
- expect(hasRowAccess(table, 2)).toBe(true);
- expect(hasRowAccess(table, 3)).toBe(true);
- expect(hasRowAccess(table, 4)).toBe(true);
- expect(hasRowAccess(table, 5)).toBe(false);
- expect(hasRowAccess(table, 6)).toBe(false);
+ expect(TblUtil.hasRowAccess(table, 0)).toBe(true);
+ expect(TblUtil.hasRowAccess(table, 1)).toBe(false);
+ expect(TblUtil.hasRowAccess(table, 2)).toBe(true);
+ expect(TblUtil.hasRowAccess(table, 3)).toBe(true);
+ expect(TblUtil.hasRowAccess(table, 4)).toBe(true);
+ expect(TblUtil.hasRowAccess(table, 5)).toBe(false);
+ expect(TblUtil.hasRowAccess(table, 6)).toBe(false);
});
test('RELEASE_DATE_COL', () => {
@@ -137,9 +141,9 @@ describe('TableUtil:', () => {
}
};
- expect(hasRowAccess(table, 0)).toBe(true);
- expect(hasRowAccess(table, 1)).toBe(false);
- expect(hasRowAccess(table, 2)).toBe(false);
+ expect(TblUtil.hasRowAccess(table, 0)).toBe(true);
+ expect(TblUtil.hasRowAccess(table, 1)).toBe(false);
+ expect(TblUtil.hasRowAccess(table, 2)).toBe(false);
});
test('both RELEASE_DATE_COL and DATARIGHTS_COL', () => {
@@ -159,9 +163,9 @@ describe('TableUtil:', () => {
}
};
- expect(hasRowAccess(table, 0)).toBe(true);
- expect(hasRowAccess(table, 1)).toBe(true);
- expect(hasRowAccess(table, 2)).toBe(false);
+ expect(TblUtil.hasRowAccess(table, 0)).toBe(true);
+ expect(TblUtil.hasRowAccess(table, 1)).toBe(true);
+ expect(TblUtil.hasRowAccess(table, 2)).toBe(false);
});
test('Proprietary data by ObsCore cnames', () => {
@@ -182,9 +186,9 @@ describe('TableUtil:', () => {
const dataRoot = dataReducer({data:{id123: table}}, {type: TABLE_LOADED, payload: table});
const otable = get(dataRoot, 'id123');
- expect(hasRowAccess(otable, 0)).toBe(true);
- expect(hasRowAccess(otable, 1)).toBe(true);
- expect(hasRowAccess(otable, 2)).toBe(false);
+ expect(TblUtil.hasRowAccess(otable, 0)).toBe(true);
+ expect(TblUtil.hasRowAccess(otable, 1)).toBe(true);
+ expect(TblUtil.hasRowAccess(otable, 2)).toBe(false);
});
test('Proprietary data by utype', () => {
@@ -205,9 +209,9 @@ describe('TableUtil:', () => {
const dataRoot = dataReducer({data:{id123: table}}, {type: TABLE_LOADED, payload: table});
const otable = get(dataRoot, 'id123');
- expect(hasRowAccess(otable, 0)).toBe(true);
- expect(hasRowAccess(otable, 1)).toBe(true);
- expect(hasRowAccess(otable, 2)).toBe(false);
+ expect(TblUtil.hasRowAccess(otable, 0)).toBe(true);
+ expect(TblUtil.hasRowAccess(otable, 1)).toBe(true);
+ expect(TblUtil.hasRowAccess(otable, 2)).toBe(false);
});
test('Proprietary data by UCD', () => {
@@ -229,55 +233,81 @@ describe('TableUtil:', () => {
const otable = get(dataRoot, 'id123');
// only release date matters
- expect(hasRowAccess(otable, 0)).toBe(true);
- expect(hasRowAccess(otable, 1)).toBe(false);
- expect(hasRowAccess(otable, 2)).toBe(false);
+ expect(TblUtil.hasRowAccess(otable, 0)).toBe(true);
+ expect(TblUtil.hasRowAccess(otable, 1)).toBe(false);
+ expect(TblUtil.hasRowAccess(otable, 2)).toBe(false);
});
});
-describe('FilterInfo', () => {
+describe('TableUtil: client_table', () => {
+
+ const table = {
+ totalRows: 6,
+ tableData: {
+ columns: [ {name: 'c1', type: 'char', nullString: 'null'},
+ {name: 'c2', type: 'double'},
+ {name: 'c3', type: 'int'}
+ ],
+ data: [
+ ['abc' , 0.123 , 100 ],
+ [undefined , -2.34 , -1 ],
+ ['123' , 0.0 , null ],
+ ['' , null , 50 ],
+ [null , 0.131 , -20 ],
+ ['ABC' , undefined , undefined]
+ ],
+ }
+ };
+
+ test('Sort', () => {
+ // undefined is smallest, then null, then natural order
+ let res = TblUtil.processRequest(table, {sortInfo: 'ASC,c1'});
+ expect(TblUtil.getColumnValues(res, 'c1')).toEqual([undefined, null, '', '123', 'ABC', 'abc']);
+
+ res = TblUtil.processRequest(table, {sortInfo: 'DESC,c1'});
+ expect(TblUtil.getColumnValues(res, 'c1')).toEqual(['abc', 'ABC', '123', '', null, undefined]);
+
+ res = TblUtil.processRequest(table, {sortInfo: 'ASC,c2'});
+ expect(TblUtil.getColumnValues(res, 'c2')).toEqual([undefined, null, -2.34, 0.0, 0.123, 0.131]);
+
+ res = TblUtil.processRequest(table, {sortInfo: 'DESC,c2'});
+ expect(TblUtil.getColumnValues(res, 'c2')).toEqual([0.131, 0.123, 0.0, -2.34, null, undefined]);
+
+ res = TblUtil.processRequest(table, {sortInfo: 'ASC,c3'});
+ expect(TblUtil.getColumnValues(res, 'c3')).toEqual([undefined, null, -20, -1, 50, 100]);
+
+ res = TblUtil.processRequest(table, {sortInfo: 'DESC,c3'});
+ expect(TblUtil.getColumnValues(res, 'c3')).toEqual([100, 50, -1, -20, null, undefined]);
+ });
- test('autoCorrectConditions: string columns', () => {
+ test('filter', () => {
- /*
- * This test is a little trickier. conditionValidator() takes a tbl_id that uses TblUtil.getTblById to resolve a tableModel.
- * This involves a fully functional redux store which is not available to our JS test environment at the moment.
- * So, we will 'fake' the return value of getTblById by 'mocking' that function
- */
- const aStringColumn = {
- tableData: {
- columns: [ {name: 'desc', type: 'char'}],
- }
- };
+ let res = TblUtil.processRequest(table, {filters: "c1 IN ('abc','')"});
+ expect(TblUtil.getColumnValues(res, 'c2')).toEqual([0.123, null, undefined]); // filtering is not case sensitive. abc === ABC
- TblUtil.getTblById = jest.fn().mockReturnValue(aStringColumn); // mock getTblById to return the aStringColumn table
+ res = TblUtil.processRequest(table, {filters: `c1 IN ('abc', ${NULL_TOKEN})`});
+ expect(TblUtil.getColumnValues(res, 'c2')).toEqual([0.123, 0.131, undefined]); // testing special NULL_TOKEN
- let actual = FilterInfo.conditionValidator('=abc', 'a_fake_tbl_id', 'desc');
- expect(actual.valid).toBe(true);
- expect(actual.value).toBe("= 'abc'"); // the validator correctly insert space and quote around a value of a string column.
+ res = TblUtil.processRequest(table, {filters: 'c2 > 0'});
+ expect(TblUtil.getColumnValues(res, 'c2')).toEqual([0.123, 0.131]); // testing greater than (>)
- actual = FilterInfo.conditionValidator('abc', 'a_fake_tbl_id', 'desc');
- expect(actual.valid).toBe(true);
- expect(actual.value).toBe("like '%abc%'"); // the validator correctly convert it into a LIKE operator
- });
+ res = TblUtil.processRequest(table, {filters: 'c2 < 0'});
+ expect(TblUtil.getColumnValues(res, 'c2')).toEqual([-2.34]); // equality test, null and undefined are ignored
- test('autoCorrectConditions: numeric columns', () => {
+ res = TblUtil.processRequest(table, {filters: 'c1 < a'});
+ expect(TblUtil.getColumnValues(res, 'c1')).toEqual(['123', '']); // testing less than (<) on string
- const aNumericColumn = {
- tableData: {
- columns: [ {name: 'ra', type: 'double'}],
- }
- };
+ res = TblUtil.processRequest(table, {filters: 'c1 IS NULL'});
+ expect(TblUtil.getColumnValues(res, 'c2')).toEqual([-2.34, 0.131]); // testing IS NULL on a string column
- TblUtil.getTblById = jest.fn().mockReturnValue(aNumericColumn); // once again mock getTblById to return the different (numeric) table
+ res = TblUtil.processRequest(table, {filters: 'c2 IS NULL'});
+ expect(TblUtil.getColumnValues(res, 'c1')).toEqual(['', 'ABC']); // testing IS NULL on numeric column
- const {valid, value} = FilterInfo.conditionValidator('>1.23', 'a_fake_tbl_id', 'ra');
- expect(valid).toBe(true);
- expect(value).toBe('> 1.23'); // the validator correctly insert space and no quotes on numeric columns.
+ res = TblUtil.processRequest(table, {filters: 'c2 IS NOT NULL'});
+ expect(TblUtil.getColumnValues(res, 'c1')).toEqual(['abc', undefined, '123', null]); // testing IS NOT NULL
});
-
});
diff --git a/src/firefly/js/tables/ui/BasicTableView.jsx b/src/firefly/js/tables/ui/BasicTableView.jsx
index ab4ea96c4d..e12c14f155 100644
--- a/src/firefly/js/tables/ui/BasicTableView.jsx
+++ b/src/firefly/js/tables/ui/BasicTableView.jsx
@@ -8,7 +8,7 @@ import FixedDataTable from 'fixed-data-table-2';
import {wrapResizer} from '../../ui/SizeMeConfig.js';
import {get, isEmpty} from 'lodash';
-import {tableTextView, getTableUiById, getProprietaryInfo, getTblById, hasRowAccess, uniqueTblUiId} from '../TableUtil.js';
+import {tableTextView, getTableUiById, getProprietaryInfo, getTblById, hasRowAccess, calcColumnWidths, uniqueTblUiId} from '../TableUtil.js';
import {SelectInfo} from '../SelectInfo.js';
import {FilterInfo} from '../FilterInfo.js';
import {SortInfo} from '../SortInfo.js';
@@ -46,11 +46,6 @@ const BasicTableViewInternal = React.memo((props) => {
const onFilter = useCallback( doFilter.bind({callbacks, filterInfo}), [callbacks, filterInfo]);
const onFilterSelected = useCallback( doFilterSelected.bind({callbacks, selectInfoCls}), [callbacks, selectInfoCls]);
- useEffect( () => {
- if (!columnWidths) {
- dispatchTableUiUpdate({tbl_ui_id, columnWidths: makeColWidth(columns, data)});
- }
- });
const headerHeight = 22 + (showUnits && 8) + (showTypes && 8) + (showFilters && 22);
let totalColWidths = 0;
if (!isEmpty(columns) && !isEmpty(columnWidths)) {
@@ -65,7 +60,7 @@ const BasicTableViewInternal = React.memo((props) => {
useEffect( () => {
const changes = {};
- if (!columnWidths) changes.columnWidths = makeColWidth(columns, data);
+ if (!columnWidths) changes.columnWidths = columnWidthsInPixel(columns, data);
if (adjScrollTop !== scrollTop) changes.scrollTop = adjScrollTop;
if (adjScrollLeft !== scrollLeft) changes.scrollLeft = adjScrollLeft;
@@ -296,30 +291,9 @@ function correctScrollLeftIfNeeded(totalColWidths, scrollLeft, width, triggeredB
return scrollLeft;
}
-function calcMaxWidth(idx, col, data) {
- let nchar = col.prefWidth || col.width;
- if (!nchar) {
- const label = col.label || col.name;
- const hWidth = Math.max(
- get(label, 'length', 0) + 2,
- get(col, 'units.length', 0) + 2,
- get(col, 'type.length', 0) + 2
- );
- nchar = hWidth;
- for (const r in data) {
- const w = get(data, [r, idx, 'length'], 0);
- if (w > nchar) nchar = w;
- }
- }
- return nchar * 7;
-}
-
-function makeColWidth(columns, data) {
-
- return !columns ? {} : columns.reduce((widths, col, idx) => {
- widths[idx] = calcMaxWidth(idx, col, data);
- return widths;
- }, {});
+function columnWidthsInPixel(columns, data) {
+ return calcColumnWidths(columns, data)
+ .map( (w) => (w + 2) * 7);
}
function rowClassNameGetter(tbl_id, hlRowIdx, startIdx) {
diff --git a/src/firefly/js/tables/ui/TablePanel.jsx b/src/firefly/js/tables/ui/TablePanel.jsx
index 63a1f4906f..bca9f43d5e 100644
--- a/src/firefly/js/tables/ui/TablePanel.jsx
+++ b/src/firefly/js/tables/ui/TablePanel.jsx
@@ -9,7 +9,7 @@ import shallowequal from 'shallowequal';
import {flux} from '../../Firefly.js';
import * as TblUtil from '../TableUtil.js';
-import {dispatchTableRemove, dispatchTblExpanded, dispatchTableSearch} from '../TablesCntlr.js';
+import {dispatchTableRemove, dispatchTblExpanded, dispatchTableFetch} from '../TablesCntlr.js';
import {TablePanelOptions} from './TablePanelOptions.jsx';
import {BasicTableView} from './BasicTableView.jsx';
import {TableInfo} from './TableInfo.jsx';
@@ -22,7 +22,6 @@ import {HelpIcon} from '../../ui/HelpIcon.jsx';
import {showTableDownloadDialog} from './TableSave.jsx';
import {showOptionsPopup} from '../../ui/PopupUtil.jsx';
import {BgMaskPanel} from '../../core/background/BgMaskPanel.jsx';
-import {getAppOptions} from '../../core/AppDataCntlr.js';
//import INFO from 'html/images/icons-2014/24x24_Info.png';
import FILTER from 'html/images/icons-2014/24x24_Filter.png';
@@ -346,7 +345,7 @@ function Loading({showTitle, tbl_id, title, removable, backgroundable}) {
function TableError({tbl_id, message}) {
const prevReq = TblUtil.getResultSetRequest(tbl_id);
const reloadTable = () => {
- dispatchTableSearch(JSON.parse(prevReq));
+ dispatchTableFetch(JSON.parse(prevReq));
};
return (
diff --git a/src/firefly/js/tables/ui/TableRenderer.js b/src/firefly/js/tables/ui/TableRenderer.js
index 8938e4ba67..edacef226a 100644
--- a/src/firefly/js/tables/ui/TableRenderer.js
+++ b/src/firefly/js/tables/ui/TableRenderer.js
@@ -6,7 +6,7 @@ import React, {Component, PureComponent, useRef, useCallback} from 'react';
import FixedDataTable from 'fixed-data-table-2';
import {set, get, isEqual, pick} from 'lodash';
-import {FilterInfo, FILTER_CONDITION_TTIPS} from '../FilterInfo.js';
+import {FilterInfo, FILTER_CONDITION_TTIPS, NULL_TOKEN} from '../FilterInfo.js';
import {isNumericType, tblDropDownId, getTblById, getColumn} from '../TableUtil.js';
import {SortInfo} from '../SortInfo.js';
import {InputField} from '../../ui/InputField.jsx';
@@ -20,6 +20,7 @@ import {CheckboxGroupInputField} from '../../ui/CheckboxGroupInputField';
import {showDropDown, hideDropDown} from '../../ui/DialogRootContainer.jsx';
import {FieldGroup} from '../../ui/FieldGroup';
import {getFieldVal} from '../../fieldGroup/FieldGroupUtils.js';
+import {dispatchValueChange} from '../../fieldGroup/FieldGroupCntlr.js';
import {useStoreConnector} from './../../ui/SimpleComponent.jsx';
import {resolveHRefVal} from '../../util/VOAnalyzer.js';
@@ -117,26 +118,34 @@ function Filter({cname, onFilter, filterInfo, tbl_id}) {
);
}
-function EnumSelect({col, tbl_id, filterInfo, filterInfoCls, onFilter}) {
+function EnumSelect({col, tbl_id, filterInfoCls, onFilter}) {
const {name, enumVals} = col || {};
const groupKey = 'TableRenderer_enum';
const fieldKey = tbl_id + '-' + name;
- const options = col.enumVals.split(',')
- .map( (s) => s.trim())
- .map( (s) => ({label: s, value: s}) );
- let value = '';
- const filterBy = (filterInfoCls.getFilter(name) || '').match(/IN \((.+)\)/);
+ const options = enumVals.split(',')
+ .map( (s) => {
+ const value = s === '' ? '%EMPTY' : s; // because CheckboxGroupInputField does not support '' as an option, use '%EMPTY' as substitute
+ const label = value === NULL_TOKEN ? '
' : value === '%EMPTY' ? '' : value;
+ return {label, value};
+ } );
+ let value;
+ const filterBy = (filterInfoCls.getFilter(name) || '').match(/IN \((.+)\)/i);
if (filterBy) {
// IN condition is used, set value accordingly. remove enclosed quote if exists
value = filterBy[1].split(',')
- .map( (s) => s.trim().replace(/^'(.+)'$/, '$1'))
+ .map( (s) => s.trim().replace(/^'(.*)'$/, '$1'))
+ .map((s) => s === '' ? '%EMPTY' : s) // convert '' to %EMPTY
.join(',');
}
const hideEnumSelect = () => hideDropDown(tblDropDownId(tbl_id));
+ const onClear = () => {
+ dispatchValueChange({fieldKey, groupKey, value: '', valid: true});
+ };
const onApply = () => {
- let value = getFieldVal(groupKey, fieldKey, '');
+ let value = getFieldVal(groupKey, fieldKey);
if (value) {
+ value = value.split(',').map((s) => s === '%EMPTY' ? '' : s).join(); // convert %EMPTY back into ''
value = isNumericType(col) ? value :
value.split(',')
.map((s) => `'${s.trim()}'`).join(',');
@@ -149,7 +158,8 @@ function EnumSelect({col, tbl_id, filterInfo, filterInfoCls, onFilter}) {
return (
-
filter
+
filter
+
clear
diff --git a/src/firefly/test/edu/caltech/ipac/astro/IpacTableReaderTest.java b/src/firefly/test/edu/caltech/ipac/astro/IpacTableReaderTest.java
index 7fb49575ab..75f2cd1622 100644
--- a/src/firefly/test/edu/caltech/ipac/astro/IpacTableReaderTest.java
+++ b/src/firefly/test/edu/caltech/ipac/astro/IpacTableReaderTest.java
@@ -406,6 +406,35 @@ public void testNoData() {
}
+ /**
+ * This test IpacTableReader handling of NULL related values
+ */
+ @Test
+ public void testNullValues() {
+ String input = "|ra |dec |spec |SpType |\n" +
+ "|double |double |char |char |\n" +
+ "| | | | |\n" +
+ "|null | | |null |\n" +
+ " 123.4 5.67 null K6Ve-1 \n" +
+ " null \n" +
+ " 123.4 5.67 abc null \n";
+ try{
+ DataGroup dg = IpacTableReader.read(asInputStream(input));
+
+ Assert.assertNull(dg.getData("ra", 1)); // value is same as NULL_STR, so value should be read in as null
+ Assert.assertNull(dg.getData("dec", 1)); // value is same as NULL_STR, so value should be read in as null
+
+ Assert.assertEquals("null", dg.getData("spec", 0)); // NULL_STR is empty_str, so null should be interpreted as a string
+ Assert.assertNull(dg.getData("spec", 1)); // value is same as NULL_STR, so value should be read in as null
+
+ Assert.assertEquals("", dg.getData("SpType", 1)); // NULL_STR is 'null', so blank should be interpreted as an empty string
+ Assert.assertNull(dg.getData("SpType", 2)); // value is same as NULL_STR, so value should be read in as null
+ } catch (IOException e) {
+ Assert.fail("Unexpected read error:" + e.getMessage());
+ }
+
+ }
+
@Test
/**
* This test calls the method IpacTableReader.readIpacTable to read a table which has one datum under "|".