DM-12616 Partial decimated data returned by a duplicate request

src/firefly/java/edu/caltech/ipac/firefly/server/query/DecimationProcessor.java

@@ -28,8 +28,6 @@ protected String getResultSetTablePrefix() {
 
     protected DataGroup fetchData(TableServerRequest treq, File dbFile, DbAdapter dbAdapter) throws DataAccessException {
 
-        treq.setPageSize(Integer.MAX_VALUE);
-
         DecimateInfo decimateInfo = getDecimateInfo(treq);
         TableServerRequest sreq = getSearchRequest(treq);
         sreq.setPageSize(Integer.MAX_VALUE);        // we want all of the data.  no paging

src/firefly/java/edu/caltech/ipac/firefly/server/query/EmbeddedDbProcessor.java

@@ -123,7 +123,7 @@ public DataGroupPart getData(ServerRequest request) throws DataAccessException {
             lockChecker.unlock();
         }
 
-        // make sure multiple requests for the same data waits for the first one to create before acessing.
+        // make sure multiple requests for the same data waits for the first one to create before accessing.
         lock.lock();
         try {
             boolean dbFileCreated = false;

src/firefly/js/charts/ChartUtil.js

@@ -387,6 +387,8 @@ export function getDataChangesForMappings({tableModel, mappings, traceNum}) {
         });
         getDataVal = (v) => transposed[cols.indexOf(v)];
     } else {
+        // no tableModel case is for pre-fetch changes
+        changes[`fireflyData.${traceNum}.error`] = undefined;
         getDataVal = (v) => v;
     }
 

src/firefly/js/charts/ChartsCntlr.js

@@ -15,6 +15,7 @@ import {getPointIdx, getRowIdx, handleTableSourceConnections, clearChartConn, ne
         applyDefaults, HIGHLIGHTED_PROPS, SELECTED_PROPS, TBL_SRC_PATTERN} from './ChartUtil.js';
 import {FilterInfo} from '../tables/FilterInfo.js';
 import {SelectInfo} from '../tables/SelectInfo.js';
+import {REINIT_APP} from '../core/AppDataCntlr.js';
 
 export const CHART_SPACE_PATH = 'charts';
 export const UI_PREFIX = `${CHART_SPACE_PATH}.ui`;
@@ -674,7 +675,11 @@ export function chartDataUpdate(payload) {
 export function reducer(state={ui:{}, data:{}}, action={}) {
 
     if (!action.type.startsWith(TablesCntlr.DATA_PREFIX) && !action.type.startsWith(CHART_SPACE_PATH)){
-        return state;
+        if (action.type === REINIT_APP) {
+            return {ui:{}, data:{}};
+        } else {
+            return state;
+        }
     }
 
     const nstate = {...state};

src/firefly/js/charts/dataTypes/FireflyGenericData.js

@@ -40,10 +40,7 @@ function fetchData(chartId, traceNum, tablesource) {
     const originalTableModel = getTblById(tbl_id);
     const {request, highlightedRow, selectInfo} = originalTableModel;
 
-    const req = makeTableFunctionRequest(request, 'XYGeneric');
-    req.startIdx = 0;
-    req.pageSize = MAX_ROW;
-
+    const req = makeTableFunctionRequest(request, 'XYGeneric', 'chart', {pageSize: MAX_ROW});
 
     Object.entries(options).forEach(([k,v]) => v && (req[k]=v));
 

src/firefly/js/charts/dataTypes/FireflyHeatmap.js

@@ -2,7 +2,7 @@
  * License information at https://github.com/Caltech-IPAC/firefly/blob/master/License.txt
  */
 import {get} from 'lodash';
-import {getTblById, getColumn, cloneRequest, doFetchTable} from '../../tables/TableUtil.js';
+import {getTblById, getColumn, doFetchTable} from '../../tables/TableUtil.js';
 import {makeTableFunctionRequest, MAX_ROW} from '../../tables/TableRequestUtil.js';
 import {dispatchChartUpdate, dispatchError, getChartData} from '../ChartsCntlr.js';
 import {serializeDecimateInfo, parseDecimateKey} from '../../tables/Decimate.js';
@@ -56,7 +56,8 @@ function fetchData(chartId, traceNum, tablesource) {
 
     const {xColOrExpr, yColOrExpr, maxbins, xyratio, xmin, xmax, ymin, ymax} = options;
     // min rows for decimation is 0
-    const req = makeTableFunctionRequest(request, 'DecimateTable', 'heatmap',  {decimate: serializeDecimateInfo(xColOrExpr, yColOrExpr, maxbins, xyratio, xmin, xmax, ymin, ymax, 0)})
+    const req = makeTableFunctionRequest(request, 'DecimateTable', 'heatmap',
+        {decimate: serializeDecimateInfo(xColOrExpr, yColOrExpr, maxbins, xyratio, xmin, xmax, ymin, ymax, 0), pageSize: MAX_ROW});
 
     doFetchTable(req).then((tableModel) => {
         if (tableModel.tableData && tableModel.tableData.data) {

src/firefly/js/charts/dataTypes/FireflyHistogram.js

@@ -4,7 +4,7 @@
 import {get, isArray} from 'lodash';
 import {logError} from '../../util/WebUtil.js';
 import {getTblById, doFetchTable} from '../../tables/TableUtil.js';
-import {makeTableFunctionRequest} from '../../tables/TableRequestUtil.js';
+import {makeTableFunctionRequest, MAX_ROW} from '../../tables/TableRequestUtil.js';
 import {dispatchChartUpdate, dispatchError, getChartData} from '../ChartsCntlr.js';
 
 
@@ -59,7 +59,7 @@ function fetchData(chartId, traceNum, tablesource) {
     const tableModel = getTblById(tbl_id);
     const {request} = tableModel;
 
-    const req = makeTableFunctionRequest(request, 'HistogramProcessor');
+    const req = makeTableFunctionRequest(request, 'HistogramProcessor', 'histogram', {pageSize: MAX_ROW});
 
     Object.entries(options).forEach(([k,v]) => req[k] = v);
 
@@ -106,7 +106,6 @@ function fetchData(chartId, traceNum, tablesource) {
         }
     ).catch(
         (reason) => {
-            //console.error(`Failed to fetch histogram data for ${chartId} trace ${traceNum}: ${reason}`);
             dispatchError(chartId, traceNum, reason);
         }
     );

src/firefly/js/charts/dataTypes/HistogramCDT.js

@@ -6,7 +6,7 @@ import {get} from 'lodash';
 
 import {fetchTable} from '../../rpc/SearchServicesJson.js';
 import {getTblById, isFullyLoaded} from '../../tables/TableUtil.js';
-import {makeTableFunctionRequest} from '../../tables/TableRequestUtil.js';
+import {makeTableFunctionRequest, MAX_ROW} from '../../tables/TableRequestUtil.js';
 import {getChartDataElement, chartDataUpdate} from './../ChartsCntlr.js';
 import {logError} from '../../util/WebUtil.js';
 
@@ -99,7 +99,7 @@ function fetchColData(dispatch, chartId, chartDataElementId) {
     const activeTableServerRequest = activeTableModel['request'];
     const tblSource = get(activeTableModel, 'tableMeta.resultSetID');
 
-    const req = makeTableFunctionRequest(activeTableServerRequest, 'HistogramProcessor');
+    const req = makeTableFunctionRequest(activeTableServerRequest, 'HistogramProcessor', 'histogram', {pageSize: MAX_ROW});
 
     // histogram parameters
     req.columnExpression = histogramParams.columnOrExpr;

src/firefly/js/visualize/saga/CatalogWatcher.js

@@ -121,7 +121,8 @@ function handleCatalogUpdate(tbl_id) {
     let req = cloneRequest(sourceTable.request, params);
     var dataTooBigForSelection= false;
     if (totalRows>50000) {
-        req = makeTableFunctionRequest(sourceTable.request, 'DecimateTable', 'heatmap',  {decimate: serializeDecimateInfo(columns.lonCol, columns.latCol, 10000)});
+        req = makeTableFunctionRequest(sourceTable.request, 'DecimateTable', 'heatmap',
+            {decimate: serializeDecimateInfo(columns.lonCol, columns.latCol, 10000), pageSize: MAX_ROW});
         dataTooBigForSelection= true;
     }
 

src/firefly/js/visualize/saga/CoverageWatcher.js

@@ -201,7 +201,7 @@ function removeCoverage(tbl_id, decimatedTables) {
 }
 
 /**
- * 
+ * @param {boolean} useHiPS
  * @param {string} tbl_id
  * @param {string} viewerId
  * @param decimatedTables
@@ -227,7 +227,8 @@ function updateCoverage(useHiPS, tbl_id, viewerId, decimatedTables, options) {
     if (table.totalRows>10000) {
         const cenCol= options.getCenterColumns(table);
         params.decimate=  serializeDecimateInfo(cenCol.lonCol, cenCol.latCol, 10000);
-        req = makeTableFunctionRequest(table.request, 'DecimateTable', 'coverage',  {decimate: serializeDecimateInfo(cenCol.lonCol, cenCol.latCol, 10000)});
+        req = makeTableFunctionRequest(table.request, 'DecimateTable', 'coverage',
+            {decimate: serializeDecimateInfo(cenCol.lonCol, cenCol.latCol, 10000), pageSize: MAX_ROW});
     }
 
     req.tbl_id = `cov-${tbl_id}`;
@@ -259,6 +260,7 @@ function updateCoverage(useHiPS, tbl_id, viewerId, decimatedTables, options) {
 /**
  *
  * @param {string} viewerId
+ * @param {boolean} useHiPS
  * @param {TableData} table
  * @param {CoverageOptions} options
  * @param {string} tbl_id