Skip to content

Simplify API version validation #1556

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
May 29, 2024
Merged

Simplify API version validation #1556

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
d5b058f
Simplify API version message
thangckt May 27, 2024
8c4210b
u
thangckt May 27, 2024
cb6e88e
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] May 27, 2024
e1d7c0c
u
thangckt May 28, 2024
ee5b52a
Update Dispatcher.py
thangckt May 28, 2024
d7fe8af
Merge branch 'devel' into simplify
thangckt May 28, 2024
ba027fd
Update utils.py
thangckt May 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
142 changes: 70 additions & 72 deletions dpgen/data/gen.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1158,27 +1158,26 @@
# relax_run_tasks.append(ii)
run_tasks = [os.path.basename(ii) for ii in relax_run_tasks]

api_version = mdata.get("api_version", "1.0")
if Version(api_version) < Version("1.0"):
### Submit the jobs
if Version(mdata.get("api_version", "1.0")) < Version("1.0"):

Check warning on line 1162 in dpgen/data/gen.py

View check run for this annotation

Codecov / codecov/patch

dpgen/data/gen.py#L1162 

Added line #L1162 was not covered by tests
raise RuntimeError(
f"API version {api_version} has been removed. Please upgrade to 1.0."
"API version below 1.0 is no longer supported. Please upgrade to version 1.0 or newer."
)

elif Version(api_version) >= Version("1.0"):
submission = make_submission(
mdata["fp_machine"],
mdata["fp_resources"],
commands=[fp_command],
work_path=work_dir,
run_tasks=run_tasks,
group_size=fp_group_size,
forward_common_files=forward_common_files,
forward_files=forward_files,
backward_files=backward_files,
outlog="fp.log",
errlog="fp.log",
)
submission.run_submission()
submission = make_submission(

Check warning on line 1167 in dpgen/data/gen.py

View check run for this annotation

Codecov / codecov/patch

dpgen/data/gen.py#L1167 

Added line #L1167 was not covered by tests
mdata["fp_machine"],
mdata["fp_resources"],
commands=[fp_command],
work_path=work_dir,
run_tasks=run_tasks,
group_size=fp_group_size,
forward_common_files=forward_common_files,
forward_files=forward_files,
backward_files=backward_files,
outlog="fp.log",
errlog="fp.log",
)
submission.run_submission()

Check warning on line 1180 in dpgen/data/gen.py

View check run for this annotation

Codecov / codecov/patch

dpgen/data/gen.py#L1180 

Added line #L1180 was not covered by tests


def coll_abacus_md(jdata):
Expand Down Expand Up @@ -1298,27 +1297,26 @@
# relax_run_tasks.append(ii)
run_tasks = [os.path.basename(ii) for ii in relax_run_tasks]

api_version = mdata.get("api_version", "1.0")
if Version(api_version) < Version("1.0"):
### Submit the jobs
if Version(mdata.get("api_version", "1.0")) < Version("1.0"):

Check warning on line 1301 in dpgen/data/gen.py

View check run for this annotation

Codecov / codecov/patch

dpgen/data/gen.py#L1301 

Added line #L1301 was not covered by tests
raise RuntimeError(
f"API version {api_version} has been removed. Please upgrade to 1.0."
"API version below 1.0 is no longer supported. Please upgrade to version 1.0 or newer."
)

elif Version(api_version) >= Version("1.0"):
submission = make_submission(
mdata["fp_machine"],
mdata["fp_resources"],
commands=[fp_command],
work_path=work_dir,
run_tasks=run_tasks,
group_size=fp_group_size,
forward_common_files=forward_common_files,
forward_files=forward_files,
backward_files=backward_files,
outlog="fp.log",
errlog="fp.log",
)
submission.run_submission()
submission = make_submission(

Check warning on line 1306 in dpgen/data/gen.py

View check run for this annotation

Codecov / codecov/patch

dpgen/data/gen.py#L1306 

Added line #L1306 was not covered by tests
mdata["fp_machine"],
mdata["fp_resources"],
commands=[fp_command],
work_path=work_dir,
run_tasks=run_tasks,
group_size=fp_group_size,
forward_common_files=forward_common_files,
forward_files=forward_files,
backward_files=backward_files,
outlog="fp.log",
errlog="fp.log",
)
submission.run_submission()

Check warning on line 1319 in dpgen/data/gen.py

View check run for this annotation

Codecov / codecov/patch

dpgen/data/gen.py#L1319 

Added line #L1319 was not covered by tests


def run_vasp_md(jdata, mdata):
Expand Down Expand Up @@ -1359,27 +1357,27 @@
run_tasks = [ii.replace(work_dir + "/", "") for ii in md_run_tasks]
# dlog.info("md_work_dir", work_dir)
# dlog.info("run_tasks",run_tasks)
api_version = mdata.get("api_version", "1.0")
if Version(api_version) < Version("1.0"):

### Submit the jobs
if Version(mdata.get("api_version", "1.0")) < Version("1.0"):

Check warning on line 1362 in dpgen/data/gen.py

View check run for this annotation

Codecov / codecov/patch

dpgen/data/gen.py#L1362 

Added line #L1362 was not covered by tests
raise RuntimeError(
f"API version {api_version} has been removed. Please upgrade to 1.0."
"API version below 1.0 is no longer supported. Please upgrade to version 1.0 or newer."
)

elif Version(api_version) >= Version("1.0"):
submission = make_submission(
mdata["fp_machine"],
mdata["fp_resources"],
commands=[fp_command],
work_path=work_dir,
run_tasks=run_tasks,
group_size=fp_group_size,
forward_common_files=forward_common_files,
forward_files=forward_files,
backward_files=backward_files,
outlog="fp.log",
errlog="fp.log",
)
submission.run_submission()
submission = make_submission(

Check warning on line 1367 in dpgen/data/gen.py

View check run for this annotation

Codecov / codecov/patch

dpgen/data/gen.py#L1367 

Added line #L1367 was not covered by tests
mdata["fp_machine"],
mdata["fp_resources"],
commands=[fp_command],
work_path=work_dir,
run_tasks=run_tasks,
group_size=fp_group_size,
forward_common_files=forward_common_files,
forward_files=forward_files,
backward_files=backward_files,
outlog="fp.log",
errlog="fp.log",
)
submission.run_submission()

Check warning on line 1380 in dpgen/data/gen.py

View check run for this annotation

Codecov / codecov/patch

dpgen/data/gen.py#L1380 

Added line #L1380 was not covered by tests


def run_abacus_md(jdata, mdata):
Expand Down Expand Up @@ -1435,27 +1433,27 @@
run_tasks = [ii.replace(work_dir + "/", "") for ii in md_run_tasks]
# dlog.info("md_work_dir", work_dir)
# dlog.info("run_tasks",run_tasks)
api_version = mdata.get("api_version", "1.0")
if Version(api_version) < Version("1.0"):

### Submit the jobs
if Version(mdata.get("api_version", "1.0")) < Version("1.0"):

Check warning on line 1438 in dpgen/data/gen.py

View check run for this annotation

Codecov / codecov/patch

dpgen/data/gen.py#L1438 

Added line #L1438 was not covered by tests
raise RuntimeError(
f"API version {api_version} has been removed. Please upgrade to 1.0."
"API version below 1.0 is no longer supported. Please upgrade to version 1.0 or newer."
)

elif Version(api_version) >= Version("1.0"):
submission = make_submission(
mdata["fp_machine"],
mdata["fp_resources"],
commands=[fp_command],
work_path=work_dir,
run_tasks=run_tasks,
group_size=fp_group_size,
forward_common_files=forward_common_files,
forward_files=forward_files,
backward_files=backward_files,
outlog="fp.log",
errlog="fp.log",
)
submission.run_submission()
submission = make_submission(

Check warning on line 1443 in dpgen/data/gen.py

View check run for this annotation

Codecov / codecov/patch

dpgen/data/gen.py#L1443 

Added line #L1443 was not covered by tests
mdata["fp_machine"],
mdata["fp_resources"],
commands=[fp_command],
work_path=work_dir,
run_tasks=run_tasks,
group_size=fp_group_size,
forward_common_files=forward_common_files,
forward_files=forward_files,
backward_files=backward_files,
outlog="fp.log",
errlog="fp.log",
)
submission.run_submission()

Check warning on line 1456 in dpgen/data/gen.py

View check run for this annotation

Codecov / codecov/patch

dpgen/data/gen.py#L1456 

Added line #L1456 was not covered by tests


def gen_init_bulk(args):
Expand Down
108 changes: 53 additions & 55 deletions dpgen/generator/run.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -874,31 +874,30 @@
except Exception:
train_group_size = 1

api_version = mdata.get("api_version", "1.0")

user_forward_files = mdata.get("train" + "_user_forward_files", [])
forward_files += [os.path.basename(file) for file in user_forward_files]
backward_files += mdata.get("train" + "_user_backward_files", [])
if Version(api_version) < Version("1.0"):

### Submit the jobs
if Version(mdata.get("api_version", "1.0")) < Version("1.0"):
raise RuntimeError(
f"API version {api_version} has been removed. Please upgrade to 1.0."
"API version below 1.0 is no longer supported. Please upgrade to version 1.0 or newer."
)

elif Version(api_version) >= Version("1.0"):
submission = make_submission(
mdata["train_machine"],
mdata["train_resources"],
commands=commands,
work_path=work_path,
run_tasks=run_tasks,
group_size=train_group_size,
forward_common_files=trans_comm_data,
forward_files=forward_files,
backward_files=backward_files,
outlog="train.log",
errlog="train.log",
)
submission.run_submission()
submission = make_submission(
mdata["train_machine"],
mdata["train_resources"],
commands=commands,
work_path=work_path,
run_tasks=run_tasks,
group_size=train_group_size,
forward_common_files=trans_comm_data,
forward_files=forward_files,
backward_files=backward_files,
outlog="train.log",
errlog="train.log",
)
submission.run_submission()


def post_train(iter_index, jdata, mdata):
Expand Down Expand Up @@ -2090,31 +2089,31 @@
user_forward_files = mdata.get("model_devi" + "_user_forward_files", [])
forward_files += [os.path.basename(file) for file in user_forward_files]
backward_files += mdata.get("model_devi" + "_user_backward_files", [])
api_version = mdata.get("api_version", "1.0")
if len(run_tasks) == 0:
raise RuntimeError(
"run_tasks for model_devi should not be empty! Please check your files."
)
if Version(api_version) < Version("1.0"):

### Submit the jobs
if Version(mdata.get("api_version", "1.0")) < Version("1.0"):
raise RuntimeError(
f"API version {api_version} has been removed. Please upgrade to 1.0."
"API version below 1.0 is no longer supported. Please upgrade to version 1.0 or newer."
)

elif Version(api_version) >= Version("1.0"):
submission = make_submission(
mdata["model_devi_machine"],
mdata["model_devi_resources"],
commands=commands,
work_path=work_path,
run_tasks=run_tasks,
group_size=model_devi_group_size,
forward_common_files=model_names,
forward_files=forward_files,
backward_files=backward_files,
outlog="model_devi.log",
errlog="model_devi.log",
)
submission.run_submission()
submission = make_submission(
mdata["model_devi_machine"],
mdata["model_devi_resources"],
commands=commands,
work_path=work_path,
run_tasks=run_tasks,
group_size=model_devi_group_size,
forward_common_files=model_names,
forward_files=forward_files,
backward_files=backward_files,
outlog="model_devi.log",
errlog="model_devi.log",
)
submission.run_submission()


def run_model_devi(iter_index, jdata, mdata):
Expand Down Expand Up @@ -3964,27 +3963,26 @@
forward_files += [os.path.basename(file) for file in user_forward_files]
backward_files += mdata.get("fp" + "_user_backward_files", [])

api_version = mdata.get("api_version", "1.0")
if Version(api_version) < Version("1.0"):
### Submit the jobs
if Version(mdata.get("api_version", "1.0")) < Version("1.0"):

Check warning on line 3967 in dpgen/generator/run.py

View check run for this annotation

Codecov / codecov/patch

dpgen/generator/run.py#L3967 

Added line #L3967 was not covered by tests
raise RuntimeError(
f"API version {api_version} has been removed. Please upgrade to 1.0."
"API version below 1.0 is no longer supported. Please upgrade to version 1.0 or newer."
)

elif Version(api_version) >= Version("1.0"):
submission = make_submission(
mdata["fp_machine"],
mdata["fp_resources"],
commands=[fp_command],
work_path=work_path,
run_tasks=run_tasks,
group_size=fp_group_size,
forward_common_files=forward_common_files,
forward_files=forward_files,
backward_files=backward_files,
outlog=log_file,
errlog=log_file,
)
submission.run_submission()
submission = make_submission(

Check warning on line 3972 in dpgen/generator/run.py

View check run for this annotation

Codecov / codecov/patch

dpgen/generator/run.py#L3972 

Added line #L3972 was not covered by tests
mdata["fp_machine"],
mdata["fp_resources"],
commands=[fp_command],
work_path=work_path,
run_tasks=run_tasks,
group_size=fp_group_size,
forward_common_files=forward_common_files,
forward_files=forward_files,
backward_files=backward_files,
outlog=log_file,
errlog=log_file,
)
submission.run_submission()

Check warning on line 3985 in dpgen/generator/run.py

View check run for this annotation

Codecov / codecov/patch

dpgen/generator/run.py#L3985 

Added line #L3985 was not covered by tests


def run_fp(iter_index, jdata, mdata):
Expand Down
35 changes: 17 additions & 18 deletions dpgen/simplify/simplify.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -255,27 +255,26 @@ def run_model_devi(iter_index, jdata, mdata):
commands.append(command_true_error)
backward_files.append(true_error_file_name)

api_version = mdata.get("api_version", "1.0")
if Version(api_version) < Version("1.0"):
### Submit the jobs
if Version(mdata.get("api_version", "1.0")) < Version("1.0"):
raise RuntimeError(
f"API version {api_version} has been removed. Please upgrade to 1.0."
"API version below 1.0 is no longer supported. Please upgrade to version 1.0 or newer."
)

elif Version(api_version) >= Version("1.0"):
submission = make_submission(
mdata["model_devi_machine"],
mdata["model_devi_resources"],
commands=commands,
work_path=work_path,
run_tasks=run_tasks,
group_size=model_devi_group_size,
forward_common_files=model_names,
forward_files=forward_files,
backward_files=backward_files,
outlog="model_devi.log",
errlog="model_devi.log",
)
submission.run_submission()
submission = make_submission(
mdata["model_devi_machine"],
mdata["model_devi_resources"],
commands=commands,
work_path=work_path,
run_tasks=run_tasks,
group_size=model_devi_group_size,
forward_common_files=model_names,
forward_files=forward_files,
backward_files=backward_files,
outlog="model_devi.log",
errlog="model_devi.log",
)
submission.run_submission()


def post_model_devi(iter_index, jdata, mdata):
Expand Down