Skip to content

DM-10345: provide management for long running searches #415

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Jul 20, 2017
Merged

DM-10345: provide management for long running searches #415

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ce5cec5
DM-10345: provide management for long running searches
loitly Jul 12, 2017
cdadabb
update made per review
loitly Jul 20, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions src/firefly/html/css/global.css
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@
-webkit-order: 1;
-ms-flex-order: 1;
order: 1;
height: 75px;
}

#App > footer {
Expand Down
1 change: 1 addition & 0 deletions src/firefly/java/edu/caltech/ipac/firefly/data/ServerParams.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,7 @@ public class ServerParams {
public static final String PROGRESS = "CmdProgress";
public static final String SUB_BACKGROUND_SEARCH= "subBackgroundSearch";
public static final String GET_STATUS= "status";
public static final String ADD_JOB = "addBgJob";
public static final String REMOVE_JOB = "removeBgJob";
public static final String CANCEL= "cancel";
public static final String ADD_ID_TO_CRITERIA= "addIDToCriteria";
Expand Down
1 change: 1 addition & 0 deletions src/firefly/java/edu/caltech/ipac/firefly/server/ServerCommandAccess.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@ private static void initCommand() {
_cmdMap.put(ServerParams.GET_ENUM_VALUES, new SearchServerCommands.GetEnumValues());
_cmdMap.put(ServerParams.SUB_BACKGROUND_SEARCH, new SearchServerCommands.SubmitBackgroundSearch());
_cmdMap.put(ServerParams.GET_STATUS, new SearchServerCommands.GetStatus());
_cmdMap.put(ServerParams.ADD_JOB, new SearchServerCommands.AddBgJob());
_cmdMap.put(ServerParams.REMOVE_JOB, new SearchServerCommands.RemoveBgJob());
_cmdMap.put(ServerParams.CANCEL, new SearchServerCommands.Cancel());
_cmdMap.put(ServerParams.ADD_ID_TO_CRITERIA, new SearchServerCommands.AddIDToPushCriteria());
Expand Down
7 changes: 5 additions & 2 deletions src/firefly/java/edu/caltech/ipac/firefly/server/SrvParam.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -245,9 +245,12 @@ public WebPlotRequest getRequiredWebPlotRequest(String key) {
// Table related convenience methods
//====================================================================
public TableServerRequest getTableServerRequest() {
String reqString = getRequired(ServerParams.REQUEST);
return QueryUtil.convertToServerRequest(reqString);
return getTableServerRequest(ServerParams.REQUEST);
}

public TableServerRequest getTableServerRequest(String key) {
String reqString = getRequired(key);
return QueryUtil.convertToServerRequest(reqString);
}
}

6 changes: 4 additions & 2 deletions src/firefly/java/edu/caltech/ipac/firefly/server/packagedata/BackgroundInfoCacher.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,8 +50,10 @@ public BackgroundInfoCacher(String key, String email, String baseFileName, Strin
}

public static void fireBackgroundJobAdd(BackgroundStatus bgStat) {
FluxAction addAction = new FluxAction("background.bgJobAdd", QueryUtil.convertToJsonObject(bgStat));
ServerEventManager.fireAction(addAction, ServerEvent.Scope.USER);
if (bgStat != null) {
FluxAction addAction = new FluxAction("background.bgJobAdd", QueryUtil.convertToJsonObject(bgStat));
ServerEventManager.fireAction(addAction, ServerEvent.Scope.USER);
}
}

//======================================================================
Expand Down
2 changes: 1 addition & 1 deletion src/firefly/java/edu/caltech/ipac/firefly/server/packagedata/PackageMaster.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -168,7 +168,7 @@ public BackgroundStatus packageData(DownloadRequest req,
req.getTitle(),req.getEmail(),
req.getDataSource(),
ServerContext.getRequestOwner());
BackgroundStatus bgStat= BackgroundEnv.backgroundProcess(WAIT_MILLS,backProcess);
BackgroundStatus bgStat= BackgroundEnv.backgroundProcess(WAIT_MILLS,backProcess, BgType.PACKAGE);
checkForLongQueue(bgStat);
return bgStat;

Expand Down
9 changes: 5 additions & 4 deletions src/firefly/java/edu/caltech/ipac/firefly/server/query/BackgroundEnv.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -290,21 +290,22 @@ public static SearchServices.DownloadProgress getDownloadProgress(String fileKey
}
return retval;
}


public static BackgroundStatus backgroundProcess(int waitMills, BackgroundProcessor processor) {
return backgroundProcess(waitMills, processor, null);
}

public static BackgroundStatus backgroundProcess(int waitMills, BackgroundProcessor processor, BackgroundStatus.BgType type) {
String bid= processor.getBID();
runBackgroundThread(waitMills, processor);
Logger.briefDebug("Background thread returned");
BackgroundStatus bgStat= processor.getBackgroundStatus();
if (bgStat==null) {
bgStat = processor.getPiCacher().getStatus();
bgStat = bgStat == null ? new BackgroundStatus(bid, BackgroundState.WAITING) : bgStat;
bgStat = bgStat == null ? new BackgroundStatus(bid, BackgroundState.WAITING, type) : bgStat;
}
if (!bgStat.isDone()) {
// it's not done within the same request.. add it to background and enable email notification.
bgStat.addAttribute(JobAttributes.CanSendEmail);
BackgroundEnv.addUserBackgroundInfo(bgStat);
}
processor.getPiCacher().setStatus(bgStat);
Logger.briefDebug("Background report returned");
Expand Down
19 changes: 14 additions & 5 deletions src/firefly/java/edu/caltech/ipac/firefly/server/query/SearchManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import edu.caltech.ipac.firefly.server.util.ipactable.DataGroupPart;
import edu.caltech.ipac.firefly.server.util.ipactable.DataGroupReader;
import edu.caltech.ipac.firefly.server.util.ipactable.IpacTableParser;
import edu.caltech.ipac.firefly.server.util.ipactable.JsonTableUtil;
import edu.caltech.ipac.firefly.server.util.ipactable.TableDef;
import edu.caltech.ipac.util.Assert;
import edu.caltech.ipac.util.IpacTableUtil;
Expand All @@ -32,6 +33,9 @@
import java.util.Collections;
import java.util.List;

import static edu.caltech.ipac.firefly.core.background.BackgroundStatus.CLIENT_REQ;
import static edu.caltech.ipac.firefly.core.background.BackgroundStatus.SERVER_REQ;


/**
* @author loi, tatianag
Expand Down Expand Up @@ -198,14 +202,15 @@ public FileInfo getFileInfo(TableServerRequest request) throws DataAccessExcepti
public BackgroundStatus getRawDataSetBackground(TableServerRequest request, Request clientRequest, int waitMillis) throws RPCException {

Logger.briefDebug("Backgrounded search started:" + waitMillis + " wait, req:" + request);
String email= request.containsParam(ServerParams.EMAIL)? request.getParam(ServerParams.EMAIL) : "";
String email= request.getMeta(ServerParams.EMAIL) == null ? "" : request.getMeta(ServerParams.EMAIL);
SearchWorker worker= new SearchWorker(request, clientRequest);
String title = request.getTblTitle() == null ? request.getRequestId() : request.getTblTitle();
BackgroundEnv.BackgroundProcessor processor=
new BackgroundEnv.BackgroundProcessor(worker, null,
request.getRequestId(),
title,
email, request.getRequestId(),
ServerContext.getRequestOwner() );
return BackgroundEnv.backgroundProcess(waitMillis, processor);
return BackgroundEnv.backgroundProcess(waitMillis, processor, BackgroundStatus.BgType.SEARCH);
}

public RawDataSet getEnumValues(File file) throws IOException {
Expand Down Expand Up @@ -233,8 +238,12 @@ public SearchWorker(TableServerRequest request, Request clientRequest) {
public BackgroundStatus work(BackgroundEnv.BackgroundProcessor p) throws Exception {
RawDataSet data= getRawDataSet(request);
BackgroundStatus bgStat= new BackgroundStatus(p.getBID(), BackgroundState.SUCCESS, BackgroundStatus.BgType.SEARCH);
bgStat.setServerRequest(request);
bgStat.setClientRequest(clientRequest);
if (request != null) {
bgStat.setParam(SERVER_REQ, JsonTableUtil.toJsonTableRequest(request).toJSONString());
}
if (clientRequest != null) {
bgStat.setParam(CLIENT_REQ, JsonTableUtil.toJsonTableRequest(clientRequest).toJSONString());
}
if (data.getTotalRows() > 0) bgStat.setFilePath(data.getMeta().getSource());
return bgStat;
}
Expand Down
23 changes: 13 additions & 10 deletions src/firefly/java/edu/caltech/ipac/firefly/server/query/SearchServerCommands.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -149,18 +149,12 @@ public String doCommand(SrvParam params) throws Exception {
public static class SubmitBackgroundSearch extends ServCommand {

public String doCommand(SrvParam params) throws Exception {
String servReqStr = params.getRequired(ServerParams.REQUEST);
TableServerRequest serverRequest = params.getTableServerRequest();
int waitMil = params.getRequiredInt(ServerParams.WAIT_MILS);

TableServerRequest serverRequest= ServerRequest.parse(servReqStr, new TableServerRequest());
Request clientRequest= null;
if (params.contains(ServerParams.CLIENT_REQUEST)) {
clientRequest= ServerRequest.parse(params.getOptional(ServerParams.CLIENT_REQUEST),
new Request());
}
BackgroundStatus bgStat= new SearchServicesImpl().submitBackgroundSearch(
serverRequest,clientRequest,waitMil);
return bgStat.serialize();
serverRequest, null,waitMil);
return QueryUtil.convertToJsonObject(bgStat).toJSONString();
}
}

Expand All @@ -169,7 +163,7 @@ public static class GetStatus extends ServCommand {
public String doCommand(SrvParam params) throws Exception {
BackgroundStatus bgStat= new SearchServicesImpl().getStatus(params.getID(),
params.getRequiredBoolean(ServerParams.POLLING));
return bgStat.serialize();
return QueryUtil.convertToJsonObject(bgStat).toJSONString();
}
}

Expand All @@ -181,6 +175,15 @@ public String doCommand(SrvParam params) throws Exception {
}
}

public static class AddBgJob extends ServCommand {

public String doCommand(SrvParam params) throws Exception {
BackgroundStatus bgStatus = QueryUtil.convertToBackgroundStatus(params.getRequired("bgStats"));
BackgroundEnv.addUserBackgroundInfo(bgStatus);
return "true";
}
}

public static class RemoveBgJob extends ServCommand {

public String doCommand(SrvParam params) throws Exception {
Expand Down
17 changes: 17 additions & 0 deletions src/firefly/java/edu/caltech/ipac/firefly/server/util/QueryUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,23 @@ public static TableServerRequest convertToServerRequest(String searchReqStr) {
return retval;
}

public static BackgroundStatus convertToBackgroundStatus(String jsonBgStatus) {
BackgroundStatus retval = new BackgroundStatus();
if (!StringUtils.isEmpty(jsonBgStatus)) {
try {
JSONObject jsonReq = (JSONObject) new JSONParser().parse(jsonBgStatus);
for (Object key : jsonReq.keySet()) {
String skey = String.valueOf(key);
Object val = jsonReq.get(key);
retval.setParam(skey, String.valueOf(val));
}
} catch (ParseException e) {
LOGGER.error(e);
}
}
return retval;
}

public static JSONObject convertToJsonObject(BackgroundStatus bgStat) {
List<String> intParams = Arrays.asList(MESSAGE_CNT, PACKAGE_CNT, TOTAL_BYTES, RESPONSE_CNT, ACTIVE_REQUEST_CNT);

Expand Down
1 change: 1 addition & 0 deletions src/firefly/js/api/ApiHighlevelBuild.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,7 @@ function buildTablePart(llApi) {
* @prop {string} tbl_group the group this table belongs to. Defaults to 'main'.
* @prop {number} pageSize the starting page size. Will use the request's pageSize if not given.
* @prop {boolean} removable true if this table can be removed from view. Defaults to true.
* @prop {boolean} backgroundable true if this search can be sent to background. Defaults to false.
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

is backgroundable set to be true in default?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good catch. I made it default to true and did not update this jsdoc.

* @prop {boolean} showUnits defaults to false
* @prop {boolean} showFilters defaults to false
* @prop {boolean} selectable defaults to true
Expand Down
5 changes: 3 additions & 2 deletions src/firefly/js/core/LayoutCntlr.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import {clone} from '../util/WebUtil.js';
import {smartMerge} from '../tables/TableUtil.js';
import {getDropDownNames} from '../ui/Menu.jsx';
import ImagePlotCntlr from '../visualize/ImagePlotCntlr.js';
import {TBL_RESULTS_ADDED, TABLE_REMOVE} from '../tables/TablesCntlr.js';
import {TBL_RESULTS_ADDED, TBL_RESULTS_REMOVE, TABLE_REMOVE} from '../tables/TablesCntlr.js';
import {CHART_ADD, CHART_REMOVE} from '../charts/ChartsCntlr.js';
import {REPLACE_VIEWER_ITEMS} from '../visualize/MultiViewCntlr.js';

Expand Down Expand Up @@ -239,6 +239,7 @@ export function dropDownHandler(layoutInfo, action) {

case SHOW_DROPDOWN:
case TABLE_REMOVE:
case TBL_RESULTS_REMOVE:
case ImagePlotCntlr.DELETE_PLOT_VIEW:
if (!get(layoutInfo, 'dropDown.visible', false)) {
if (count===0) {
Expand All @@ -262,7 +263,7 @@ export function* dropDownManager() {
const action = yield take([
ImagePlotCntlr.PLOT_IMAGE_START, ImagePlotCntlr.PLOT_IMAGE,
REPLACE_VIEWER_ITEMS,
TABLE_REMOVE, TBL_RESULTS_ADDED,
TABLE_REMOVE, TBL_RESULTS_ADDED, TBL_RESULTS_REMOVE,
CHART_ADD, CHART_REMOVE,
SHOW_DROPDOWN, SET_LAYOUT_MODE
]);
Expand Down
13 changes: 13 additions & 0 deletions src/firefly/js/core/background/BackgroundCntlr.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ function actionCreators() {
[BG_MONITOR_SHOW]: bgMonitorShow,
[BG_SET_EMAIL]: bgSetEmail,
[BG_Package]: bgPackage,
[BG_JOB_ADD]: bgJobAdd,
[BG_JOB_REMOVE]: bgJobRemove,
[BG_JOB_CANCEL]: bgJobCancel
};
Expand Down Expand Up @@ -133,6 +134,16 @@ function bgMonitorShow(action) {
};
}

function bgJobAdd(action) {
return (dispatch) => {
const bgStatus = action.payload;
if (bgStatus) {
SearchServices.addBgJob(bgStatus);
dispatch(action);
}
};
}

function bgJobRemove(action) {
return (dispatch) => {
const {id} = action.payload;
Expand Down Expand Up @@ -174,6 +185,8 @@ function bgPackage(action) {
if (url && isSuccess(get(bgStatus, 'STATE'))) {
download(url);
dispatch({type: BG_JOB_IMMEDIATE, payload: bgStatus}); // allow saga to catch flow.
} else {
dispatchJobAdd(bgStatus);
}
}
});
Expand Down
Loading