Skip to content

Commit 0c98957

Browse files
loitlyloitly
committed
IRSA-811: create framework for submitting download tasks with masking and 'send to background' feature
- adapter new finderchart to the newly created framework. - introduce 'watchers' concept to MasterSaga - use web.xml from firefly by default - use web-fragment.xml to add additional settings at the project level
1 parent 0f8063a commit 0c98957

File tree

9 files changed

+239
-44
lines changed

9 files changed

+239
-44
lines changed

‎config/web.xml

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -42,8 +42,7 @@
4242
</filter-mapping>
4343

4444

45-
46-
<!-- Servlet and its mappings -->
45+
<!-- Firefly servlet and its mappings -->
4746
<servlet>
4847
<servlet-class>edu.caltech.ipac.firefly.server.servlets.AnyFileDownload</servlet-class>
4948
<servlet-name>FireFly Any FileDownload</servlet-name>
@@ -89,11 +88,14 @@
8988
<url-pattern>/sticky/CmdSrv</url-pattern>
9089
</servlet-mapping>
9190

92-
9391
<servlet>
9492
<servlet-name>H2Console</servlet-name>
9593
<servlet-class>org.h2.server.web.WebServlet</servlet-class>
96-
<load-on-startup>1</load-on-startup>
94+
<init-param>
95+
<param-name>-webAllowOthers</param-name>
96+
<param-value>true</param-value>
97+
</init-param>
98+
<load-on-startup>2</load-on-startup>
9799
</servlet>
98100
<servlet-mapping>
99101
<servlet-name>H2Console</servlet-name>

‎src/firefly/java/edu/caltech/ipac/firefly/server/query/SearchServerCommands.java

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -161,8 +161,7 @@ public String doCommand(SrvParam params) throws Exception {
161161
TableServerRequest serverRequest = params.getTableServerRequest();
162162
int waitMil = params.getRequiredInt(ServerParams.WAIT_MILS);
163163

164-
BackgroundStatus bgStat= new SearchServicesImpl().submitBackgroundSearch(
165-
serverRequest, null,waitMil);
164+
BackgroundStatus bgStat = new SearchManager().getRawDataSetBackground(serverRequest, null, waitMil);
166165
return QueryUtil.convertToJsonObject(bgStat).toJSONString();
167166
}
168167
}

‎src/firefly/java/edu/caltech/ipac/firefly/server/query/StatisticsProcessor.java

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -78,13 +78,15 @@ protected DataGroup fetchData(TableServerRequest treq, File dbFile, DbAdapter db
7878
}
7979

8080
}
81-
DataObject data = EmbeddedDbUtil.runQuery(dbAdapter, dbFile, String.format("select %s from %s",StringUtils.toString(sqlCols), origDataTblName), null).get(0);
82-
for (int i = 0; i < stats.size(); i++) {
83-
DataObject col = stats.get(i);
84-
String cname = col.getStringData("columnName").toUpperCase();
85-
col.setDataElement(columns[3], getDouble(data.getDataElement(cname + "_min")));
86-
col.setDataElement(columns[4], getDouble(data.getDataElement(cname + "_max")));
87-
col.setDataElement(columns[5], data.getDataElement(cname + "_count"));
81+
if (sqlCols.size() > 0) {
82+
DataObject data = EmbeddedDbUtil.runQuery(dbAdapter, dbFile, String.format("select %s from %s",StringUtils.toString(sqlCols), origDataTblName), null).get(0);
83+
for (int i = 0; i < stats.size(); i++) {
84+
DataObject col = stats.get(i);
85+
String cname = col.getStringData("columnName").toUpperCase();
86+
col.setDataElement(columns[3], getDouble(data.getDataElement(cname + "_min")));
87+
col.setDataElement(columns[4], getDouble(data.getDataElement(cname + "_max")));
88+
col.setDataElement(columns[5], data.getDataElement(cname + "_count"));
89+
}
8890
}
8991
return stats;
9092
}

‎src/firefly/java/edu/caltech/ipac/firefly/server/servlets/ServerStatus.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,7 @@ private static void showDatabaseStatus(PrintWriter writer) {
9898
writer.println("Open: " + BaseDbAdapter.getDbInstances().size());
9999
writer.println("Details: touched time in (mm:ss)");
100100
BaseDbAdapter.getDbInstances().values().stream()
101-
.forEach((db) -> writer.println(String.format("\ttouched: %2$tM:%2$tS %s", db.getDbFile().getName(), db.getLastAccessed())));
101+
.forEach((db) -> writer.println(String.format("\ttouched: %2$tM:%2$tS %s", db.getDbFile().getPath(), db.getLastAccessed())));
102102
}
103103

104104
private static String getStats(Ehcache c) {

‎src/firefly/js/core/MasterSaga.js

Lines changed: 93 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,13 @@
33
*/
44

55
import {flux} from '../Firefly.js';
6-
import {fork, take} from 'redux-saga/effects';
6+
import {take, fork, cancel} from 'redux-saga/effects';
7+
import {get} from 'lodash';
8+
79
export const ADD_SAGA= 'MasterSaga.addSaga';
10+
export const ADD_WATCHER= 'MasterSaga.addWatcher';
11+
export const CANCEL_WATCHER= 'MasterSaga.cancelWatcher';
12+
export const CANCEL_ALL_WATCHER= 'MasterSaga.cancelAllWatcher';
813

914

1015
/**
@@ -16,18 +21,99 @@ export function dispatchAddSaga(saga, params={}) {
1621
flux.process({ type: ADD_SAGA, payload: { saga,params}});
1722
}
1823

24+
/**
25+
* @param {object} p
26+
* @param {string} p.id a unique identifier for this watcher
27+
* @param {string[]} p.actions an array of action types to watch
28+
* @param {function} p.callback a callback function to handle the action(s). The triggered action will be passed back as a parameter.
29+
*/
30+
export function dispatchAddWatcher({id, actions, callback}) {
31+
flux.process({ type: ADD_WATCHER, payload: {id, actions, callback}});
32+
}
33+
34+
/**
35+
* cancel the watcher with the given id.
36+
* @param {string} id a unique identifier of the watcher to cancel
37+
*/
38+
export function dispatchCancelWatcher(id) {
39+
flux.process({ type: CANCEL_WATCHER, payload: {id}});
40+
}
41+
42+
/**
43+
* Cancel all watchers. Should only be called during re-init scenarios.
44+
*/
45+
export function dispatchCancelAllWatchers() {
46+
flux.process({ type: CANCEL_ALL_WATCHER});
47+
}
48+
1949

2050
/**
2151
* This saga launches all the predefined Sagas then loops and waits for any ADD_SAGA actions and launches those Segas
2252
*/
2353
export function* masterSaga() {
24-
54+
let watchers = {};
2555

2656
// Start a saga from any action
2757
while (true) {
28-
var action= yield take(ADD_SAGA);
29-
const {getState, dispatch}= flux.getRedux();
30-
const {saga,params}= action.payload;
31-
if (typeof saga === 'function') yield fork( saga, params, dispatch, getState);
58+
const action= yield take([ADD_SAGA, ADD_WATCHER, CANCEL_WATCHER, CANCEL_ALL_WATCHER]);
59+
60+
switch (action.type) {
61+
case ADD_SAGA: {
62+
const {getState, dispatch}= flux.getRedux();
63+
const {saga,params}= action.payload;
64+
if (typeof saga === 'function') yield fork( saga, params, dispatch, getState);
65+
break;
66+
}
67+
case ADD_WATCHER: {
68+
const {getState, dispatch}= flux.getRedux();
69+
const {id, actions, callback}= action.payload;
70+
if (id && actions && callback) {
71+
if (watchers[id]) {
72+
yield cancel(watchers[id]);
73+
}
74+
const saga = createSaga({id, actions, callback});
75+
const task = yield fork(saga, dispatch, getState);
76+
watchers[id] = task;
77+
isDebug() && console.log(`watcher ${id} added. #watcher: ${Object.keys(watchers).length}`);
78+
}
79+
break;
80+
}
81+
case CANCEL_WATCHER: {
82+
const {id}= action.payload;
83+
const task = watchers[id];
84+
if (task) {
85+
yield cancel(task);
86+
Reflect.deleteProperty(watchers, id);
87+
isDebug() && console.log(`watcher ${id} cancelled. #watcher: ${Object.keys(watchers).length}`);
88+
}
89+
break;
90+
}
91+
case CANCEL_ALL_WATCHER: {
92+
const ids = Object.keys(watchers);
93+
for (let i = 0; i < ids.length; i++) {
94+
const task = watchers[ids[i]];
95+
yield cancel(task);
96+
}
97+
watchers = {};
98+
break;
99+
}
100+
}
32101
}
33-
}
102+
}
103+
104+
105+
function createSaga({id, actions=[], callback}) {
106+
const saga = function* () {
107+
while (true) {
108+
const action= yield take(actions);
109+
try {
110+
callback && callback(action);
111+
} catch (e) {
112+
console.log(`Encounter error while executing watcher: ${id} error: ${e}`);
113+
}
114+
}
115+
};
116+
return saga;
117+
}
118+
119+
const isDebug = () => get(window, 'firefly.debug', true);

‎src/firefly/js/core/background/BackgroundCntlr.js

Lines changed: 3 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@ export const BG_JOB_ADD = `${BACKGROUND_PATH}.bgJobAdd`;
2222
export const BG_JOB_UPDATE = `${BACKGROUND_PATH}.bgJobUpdate`;
2323
export const BG_JOB_REMOVE = `${BACKGROUND_PATH}.bgJobRemove`;
2424
export const BG_JOB_CANCEL = `${BACKGROUND_PATH}.bgJobCancel`;
25-
export const BG_JOB_IMMEDIATE = `${BACKGROUND_PATH}.bgJobImmediate`;
2625
export const BG_SET_EMAIL = `${BACKGROUND_PATH}.bgSetEmail`;
2726
export const BG_Package = `${BACKGROUND_PATH}.bgPackage`;
2827
export const BG_ALLOW_DATA_TAG = `${BACKGROUND_PATH}.bgAllowDataTag`;
@@ -119,22 +118,6 @@ export function dispatchPackage(dlRequest, searchRequest, selectionInfo) {
119118
}
120119

121120

122-
/**
123-
* this saga will trigger callback when a package request is either added or handled immediately
124-
* @param {Object} p props
125-
* @param {string} p.title download request's title
126-
* @param {function} p.callback callback to execute when package request returned.
127-
*/
128-
export function* doOnPackage({title, callback}) {
129-
130-
var isDone = false;
131-
while (!(isDone)) {
132-
const action = yield take([BG_JOB_IMMEDIATE, BG_JOB_ADD]);
133-
isDone = !title || title === get(action, 'payload.Title');
134-
}
135-
callback && callback();
136-
}
137-
138121
/*---------------------------- private -----------------------------*/
139122

140123

@@ -190,11 +173,10 @@ function bgPackage(action) {
190173
SearchServices.packageRequest(dlRequest, searchRequest, selectionInfo)
191174
.then((bgStatus) => {
192175
if (bgStatus) {
193-
bgStatus = transform(bgStatus);
176+
bgStatus = bgStatusTransform(bgStatus);
194177
const url = get(bgStatus, ['ITEMS', 0, 'url']);
195178
if (url && isSuccess(get(bgStatus, 'STATE'))) {
196179
download(url);
197-
dispatch({type: BG_JOB_IMMEDIATE, payload: bgStatus}); // allow saga to catch flow.
198180
} else {
199181
dispatchJobAdd(bgStatus);
200182
}
@@ -244,7 +226,7 @@ function handleBgStatusUpdate(state, action) {
244226

245227
function handleBgJobAdd(state, action) {
246228
var bgstats = action.payload;
247-
bgstats = transform(bgstats);
229+
bgstats = bgStatusTransform(bgstats);
248230
const nState = set({}, ['jobs', bgstats.ID], bgstats);
249231
if (!nState.email && !isNil(bgstats.email)) nState.email = bgstats.email; // use email from server if one is not set
250232
return smartMerge(state, nState);
@@ -264,7 +246,7 @@ function handleBgJobRemove(state, action) {
264246
* @param bgStatus
265247
* @returns {{ITEMS: Array.<*>}}
266248
*/
267-
function transform(bgStatus) {
249+
export function bgStatusTransform(bgStatus) {
268250
const ITEMS = Object.keys(bgStatus)
269251
.filter( (k) => k.startsWith('ITEMS_') )
270252
.map( (k) => {

‎src/firefly/js/core/background/BackgroundUtil.js

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,16 @@
44

55
import {get, omit, isNil} from 'lodash';
66
import Enum from 'enum';
7+
import {take} from 'redux-saga/effects';
78

89
import {flux} from '../../Firefly.js';
910
import {BACKGROUND_PATH} from './BackgroundCntlr.js';
1011
import {getModuleName} from '../../util/WebUtil.js';
12+
import {packageRequest} from '../../rpc/SearchServicesJson.js';
13+
import {SelectInfo} from '../../tables/SelectInfo.js';
14+
import {dispatchComponentStateChange} from '../ComponentCntlr.js';
15+
import {dispatchAddSaga} from '../MasterSaga.js';
16+
import {BG_JOB_ADD, BG_STATUS, bgStatusTransform} from './BackgroundCntlr.js';
1117

1218

1319
/**
@@ -144,3 +150,55 @@ export const BG_STATE = new Enum([
144150
*/
145151
'UNKNOWN_PACKAGE_ID'
146152
]);
153+
154+
155+
export function bgDownload({dlRequest, searchRequest, selectInfo}, {key, onComplete, sentToBg}) {
156+
dispatchComponentStateChange(key, {inProgress:true});
157+
packageRequest(dlRequest, searchRequest, SelectInfo.newInstance(selectInfo).toString())
158+
.then((bgStatus) => {
159+
if (bgStatus) {
160+
dispatchComponentStateChange(key, {bgStatus});
161+
bgStatus = bgStatusTransform(bgStatus);
162+
if (isSuccess(get(bgStatus, 'STATE'))) {
163+
onComplete && onComplete(bgStatus);
164+
dispatchComponentStateChange(key, {inProgress:false, bgStatus:undefined});
165+
} else {
166+
dispatchAddSaga(bgTracker(bgStatus.ID, key, onComplete, sentToBg));
167+
}
168+
}
169+
});
170+
}
171+
172+
function bgTracker(bgID, key, onComplete, sentToBg) {
173+
return function* () {
174+
let done = false;
175+
while (!done) {
176+
const action= yield take([BG_STATUS,BG_JOB_ADD]);
177+
try {
178+
const bgStatus = bgStatusTransform(action.payload || {});
179+
const {STATE, ID} = bgStatus;
180+
if (ID === bgID) {
181+
switch (action.type) {
182+
case BG_STATUS:
183+
{
184+
if (isSuccess(STATE)) {
185+
done = true;
186+
dispatchComponentStateChange(key, {inProgress:false, bgStatus:undefined});
187+
onComplete && onComplete(bgStatus);
188+
}
189+
break;
190+
}
191+
case BG_JOB_ADD:
192+
{
193+
done = true;
194+
dispatchComponentStateChange(key, {inProgress:false});
195+
sentToBg && sentToBg(bgStatus);
196+
}
197+
}
198+
}
199+
} catch (e) {
200+
console.log(`'Encounter error while tracking bgStatus:${bgID} error:${e}`);
201+
}
202+
}
203+
};
204+
}

‎src/firefly/js/core/background/BackgroundablePanel.jsx

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
import React from 'react';
2+
import PropTypes from 'prop-types';
3+
4+
import {dispatchJobAdd} from './BackgroundCntlr.js';
5+
import {getComponentState} from '../../core/ComponentCntlr.js';
6+
import {SimpleComponent} from '../../ui/SimpleComponent.jsx';
7+
8+
9+
/**
10+
* This component uses ComponentCntlr state persistence. It is keyed by the given props's componentKey.
11+
* The data structure is described below.
12+
* @typedef {Object} data BackgroundablePanel's data structure
13+
* @prop {boolean} data.inProgress true when a download is in progress
14+
* @prop {boolean} data.bgStatus the bgStatus given to this background request
15+
*/
16+
17+
18+
export class BgMaskPanel extends SimpleComponent {
19+
20+
getNextState(np) {
21+
return getComponentState(this.props.componentKey);
22+
}
23+
24+
render() {
25+
const {inProgress, bgStatus} = this.state;
26+
const sendToBg = () => {
27+
const {bgStatus} = this.state;
28+
bgStatus && dispatchJobAdd(bgStatus);
29+
};
30+
31+
if (!inProgress) return <div style={{display: 'none'}} />;
32+
return (
33+
<div style={maskStyle}>
34+
<div className='loading-mask'/>
35+
{bgStatus &&
36+
<div style={{display: 'flex', alignItems: 'center'}}>
37+
<button type='button' style={maskButton} className='button std' onClick={sendToBg}>Send to background</button>
38+
</div>
39+
}
40+
</div>
41+
);
42+
}
43+
}
44+
45+
BgMaskPanel.propTypes = {
46+
componentKey: PropTypes.string.isRequired, // key used to identify this background job
47+
};
48+
49+
const maskStyle = {
50+
position: 'relative',
51+
width: '100%',
52+
height: '100%',
53+
boxSizing: 'border-box',
54+
border: '1px solid rgba(0, 0, 0, 0.3)',
55+
display: 'flex',
56+
justifyContent: 'center'
57+
};
58+
59+
const maskButton = {
60+
zIndex: 1,
61+
marginTop: '80px',
62+
border: '1px solid rgb(125,125,125)',
63+
boxSizing: 'content-box',
64+
backgroundColor: 'rgb(220,220,220)',
65+
borderRadius: 4
66+
};

0 commit comments

Comments
 (0)