Skip to content

Commit c8adf25

Browse files
committed
remove enum and put dataset types in database instead #10517
Also populate a few dataset types in database, with "dataset" being the default. Add default type to existing datasets. Also APIs for managing dataset types.
1 parent 3aab5c0 commit c8adf25

24 files changed

+436
-90
lines changed
+23
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,26 @@
11
### Initial Support for Dataset Types (Dataset, Software, Workflow)
22

33
Datasets now have types. By default the dataset type will be "dataset" but if you turn on support for additional types, datasets can have a type of "software" or "workflow" as well. For more details see <https://dataverse-guide--10694.org.readthedocs.build/en/10694/user/dataset-types.html> and #10517. Please note that this feature is highly experimental.
4+
5+
A handy query:
6+
7+
```
8+
% DOCKER_CLI_HINTS=false docker exec -it postgres-1 bash -c "PGPASSWORD=secret psql -h localhost -U dataverse dataverse -c 'select dst.name, count(*) from dataset ds, datasettype dst where ds.datasettype_id = dst.id group by dst.name;'"
9+
name | count
10+
----------+-------
11+
dataset | 136
12+
software | 14
13+
(2 rows)
14+
```
15+
16+
Most API tests are passing but we do see a few failures:
17+
18+
```
19+
[ERROR] Failures:
20+
[ERROR] HarvestingClientsIT.testHarvestingClientRun_AllowHarvestingMissingCVV_False:187->harvestingClientRun:301 expected: <7> but was: <0>
21+
[ERROR] HarvestingClientsIT.testHarvestingClientRun_AllowHarvestingMissingCVV_True:191->harvestingClientRun:301 expected: <8> but was: <0>
22+
[ERROR] MakeDataCountApiIT.testMakeDataCountGetMetric:68 1 expectation failed.
23+
Expected status code <200> but was <400>.
24+
```
25+
26+
select dst.name, count(*) from dataset ds, datasettype dst where ds.datasettype_id = dst.id group by dst.name;

src/main/java/edu/harvard/iq/dataverse/Dataset.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -129,8 +129,8 @@ public class Dataset extends DvObjectContainer {
129129
*/
130130
private boolean useGenericThumbnail;
131131

132-
@OneToOne(cascade = {CascadeType.MERGE, CascadeType.PERSIST})
133-
@JoinColumn(name = "datasettype_id", nullable = true)
132+
@ManyToOne
133+
@JoinColumn(name="datasettype_id", nullable = false)
134134
private DatasetType datasetType;
135135

136136
@OneToOne(cascade = {CascadeType.MERGE, CascadeType.PERSIST})

src/main/java/edu/harvard/iq/dataverse/EjbDataverseEngine.java

+10-1
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
1313
import edu.harvard.iq.dataverse.confirmemail.ConfirmEmailServiceBean;
1414
import edu.harvard.iq.dataverse.datacapturemodule.DataCaptureModuleServiceBean;
15+
import edu.harvard.iq.dataverse.dataset.DatasetTypeServiceBean;
1516
import edu.harvard.iq.dataverse.engine.command.Command;
1617
import edu.harvard.iq.dataverse.engine.command.CommandContext;
1718
import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
@@ -127,7 +128,10 @@ public class EjbDataverseEngine {
127128

128129
@EJB
129130
MetadataBlockServiceBean metadataBlockService;
130-
131+
132+
@EJB
133+
DatasetTypeServiceBean datasetTypeService;
134+
131135
@EJB
132136
DataverseLinkingServiceBean dvLinking;
133137

@@ -603,6 +607,11 @@ public MetadataBlockServiceBean metadataBlocks() {
603607
return metadataBlockService;
604608
}
605609

610+
@Override
611+
public DatasetTypeServiceBean datasetTypes() {
612+
return datasetTypeService;
613+
}
614+
606615
@Override
607616
public void beginCommandSequence() {
608617
this.commandsCalled = new Stack();

src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java

+5-1
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
import edu.harvard.iq.dataverse.authorization.users.User;
1212
import edu.harvard.iq.dataverse.confirmemail.ConfirmEmailServiceBean;
1313
import edu.harvard.iq.dataverse.datacapturemodule.DataCaptureModuleServiceBean;
14+
import edu.harvard.iq.dataverse.dataset.DatasetTypeServiceBean;
1415
import edu.harvard.iq.dataverse.engine.command.Command;
1516
import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
1617
import edu.harvard.iq.dataverse.engine.command.exception.CommandException;
@@ -164,6 +165,9 @@ String getWrappedMessageWhenJson() {
164165
@EJB
165166
protected LicenseServiceBean licenseSvc;
166167

168+
@EJB
169+
protected DatasetTypeServiceBean datasetTypeSvc;
170+
167171
@EJB
168172
protected UserServiceBean userSvc;
169173

@@ -247,7 +251,7 @@ public enum Format {
247251
private final LazyRef<JsonParser> jsonParserRef = new LazyRef<>(new Callable<JsonParser>() {
248252
@Override
249253
public JsonParser call() throws Exception {
250-
return new JsonParser(datasetFieldSvc, metadataBlockSvc,settingsSvc, licenseSvc);
254+
return new JsonParser(datasetFieldSvc, metadataBlockSvc,settingsSvc, licenseSvc, datasetTypeSvc);
251255
}
252256
});
253257

src/main/java/edu/harvard/iq/dataverse/api/Datasets.java

+105
Original file line numberDiff line numberDiff line change
@@ -99,9 +99,12 @@
9999
import java.util.stream.Collectors;
100100

101101
import static edu.harvard.iq.dataverse.api.ApiConstants.*;
102+
import edu.harvard.iq.dataverse.dataset.DatasetType;
103+
import edu.harvard.iq.dataverse.dataset.DatasetTypeServiceBean;
102104
import static edu.harvard.iq.dataverse.util.json.JsonPrinter.*;
103105
import static edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder.jsonObjectBuilder;
104106
import static jakarta.ws.rs.core.Response.Status.BAD_REQUEST;
107+
import static jakarta.ws.rs.core.Response.Status.NOT_FOUND;
105108

106109
@Path("datasets")
107110
public class Datasets extends AbstractApiBean {
@@ -187,6 +190,9 @@ public class Datasets extends AbstractApiBean {
187190
@Inject
188191
DatasetVersionFilesServiceBean datasetVersionFilesServiceBean;
189192

193+
@Inject
194+
DatasetTypeServiceBean datasetTypeSvc;
195+
190196
/**
191197
* Used to consolidate the way we parse and handle dataset versions.
192198
* @param <T>
@@ -5071,4 +5077,103 @@ public Response resetPidGenerator(@Context ContainerRequestContext crc, @PathPar
50715077
return ok("Pid Generator reset to default: " + dataset.getEffectivePidGenerator().getId());
50725078
}
50735079

5080+
@GET
5081+
@Path("datasetTypes")
5082+
public Response getDatasetTypes() {
5083+
JsonArrayBuilder jab = Json.createArrayBuilder();
5084+
List<DatasetType> datasetTypes = datasetTypeSvc.listAll();
5085+
for (DatasetType datasetType : datasetTypes) {
5086+
JsonObjectBuilder job = Json.createObjectBuilder();
5087+
job.add("id", datasetType.getId());
5088+
job.add("name", datasetType.getName());
5089+
jab.add(job);
5090+
}
5091+
return ok(jab.build());
5092+
}
5093+
5094+
@GET
5095+
@Path("datasetTypes/byName/{name}")
5096+
public Response getDatasetTypes(@PathParam("name") String name) {
5097+
DatasetType datasetType = datasetTypeSvc.getByName(name);
5098+
if (datasetType != null) {
5099+
return ok(datasetType.toJson());
5100+
} else {
5101+
return error(NOT_FOUND, "Could not find a dataset type with name " + name);
5102+
}
5103+
}
5104+
5105+
@POST
5106+
@AuthRequired
5107+
@Path("datasetTypes")
5108+
public Response addDatasetType(@Context ContainerRequestContext crc, String jsonIn) {
5109+
System.out.println("json in: " + jsonIn);
5110+
AuthenticatedUser user;
5111+
try {
5112+
user = getRequestAuthenticatedUserOrDie(crc);
5113+
} catch (WrappedResponse ex) {
5114+
return error(Response.Status.BAD_REQUEST, "Authentication is required.");
5115+
}
5116+
if (!user.isSuperuser()) {
5117+
return error(Response.Status.FORBIDDEN, "Superusers only.");
5118+
}
5119+
5120+
if (jsonIn == null || jsonIn.isEmpty()) {
5121+
throw new IllegalArgumentException("JSON input was null or empty!");
5122+
}
5123+
JsonObject jsonObject = JsonUtil.getJsonObject(jsonIn);
5124+
String nameIn = jsonObject.getString("name", null);
5125+
if (nameIn == null) {
5126+
throw new IllegalArgumentException("A name for the dataset type is required");
5127+
}
5128+
5129+
try {
5130+
DatasetType datasetType = new DatasetType();
5131+
datasetType.setName(nameIn);
5132+
DatasetType saved = datasetTypeSvc.save(datasetType);
5133+
Long typeId = saved.getId();
5134+
String name = saved.getName();
5135+
actionLogSvc.log(new ActionLogRecord(ActionLogRecord.ActionType.Admin, "addDatasetType").setInfo("Dataset type added with id " + typeId + " and name " + name + "."));
5136+
return ok(saved.toJson());
5137+
} catch (WrappedResponse ex) {
5138+
return error(BAD_REQUEST, ex.getMessage());
5139+
}
5140+
}
5141+
5142+
@DELETE
5143+
@AuthRequired
5144+
@Path("datasetTypes/{id}")
5145+
public Response deleteDatasetType(@Context ContainerRequestContext crc, @PathParam("id") String doomed) {
5146+
AuthenticatedUser user;
5147+
try {
5148+
user = getRequestAuthenticatedUserOrDie(crc);
5149+
} catch (WrappedResponse ex) {
5150+
return error(Response.Status.BAD_REQUEST, "Authentication is required.");
5151+
}
5152+
if (!user.isSuperuser()) {
5153+
return error(Response.Status.FORBIDDEN, "Superusers only.");
5154+
}
5155+
5156+
if (doomed == null || doomed.isEmpty()) {
5157+
throw new IllegalArgumentException("ID is required!");
5158+
}
5159+
5160+
long idToDelete;
5161+
try {
5162+
idToDelete = Long.parseLong(doomed);
5163+
} catch (NumberFormatException e) {
5164+
throw new IllegalArgumentException("ID must be a number");
5165+
}
5166+
5167+
try {
5168+
int numDeleted = datasetTypeSvc.deleteById(idToDelete);
5169+
if (numDeleted == 1) {
5170+
return ok("deleted");
5171+
} else {
5172+
return error(BAD_REQUEST, "Something went wrong. Number of dataset types deleted: " + numDeleted);
5173+
}
5174+
} catch (WrappedResponse ex) {
5175+
return error(BAD_REQUEST, ex.getMessage());
5176+
}
5177+
}
5178+
50745179
}

src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java

+4-9
Original file line numberDiff line numberDiff line change
@@ -242,13 +242,6 @@ public Response createDataset(@Context ContainerRequestContext crc, String jsonB
242242
//Throw BadRequestException if metadataLanguage isn't compatible with setting
243243
DataverseUtil.checkMetadataLangauge(ds, owner, settingsService.getBaseMetadataLanguageMap(null, true));
244244

245-
try {
246-
logger.info("about to call checkDatasetType...");
247-
DataverseUtil.checkDatasetType(ds, FeatureFlags.DATASET_TYPES.enabled());
248-
} catch (BadRequestException ex) {
249-
return badRequest(ex.getLocalizedMessage());
250-
}
251-
252245
// clean possible version metadata
253246
DatasetVersion version = ds.getVersions().get(0);
254247

@@ -311,7 +304,7 @@ public Response createDatasetFromJsonLd(@Context ContainerRequestContext crc, St
311304
Dataset ds = new Dataset();
312305

313306
ds.setOwner(owner);
314-
ds = JSONLDUtil.updateDatasetMDFromJsonLD(ds, jsonLDBody, metadataBlockSvc, datasetFieldSvc, false, false, licenseSvc);
307+
ds = JSONLDUtil.updateDatasetMDFromJsonLD(ds, jsonLDBody, metadataBlockSvc, datasetFieldSvc, false, false, licenseSvc, datasetTypeSvc);
315308

316309
ds.setOwner(owner);
317310

@@ -508,7 +501,7 @@ public Response recreateDataset(@Context ContainerRequestContext crc, String jso
508501
Dataset ds = new Dataset();
509502

510503
ds.setOwner(owner);
511-
ds = JSONLDUtil.updateDatasetMDFromJsonLD(ds, jsonLDBody, metadataBlockSvc, datasetFieldSvc, false, true, licenseSvc);
504+
ds = JSONLDUtil.updateDatasetMDFromJsonLD(ds, jsonLDBody, metadataBlockSvc, datasetFieldSvc, false, true, licenseSvc, datasetTypeSvc);
512505
//ToDo - verify PID is one Dataverse can manage (protocol/authority/shoulder match)
513506
if (!PidUtil.getPidProvider(ds.getGlobalId().getProviderId()).canManagePID()) {
514507
throw new BadRequestException(
@@ -551,6 +544,8 @@ private Dataset parseDataset(String datasetJson) throws WrappedResponse {
551544
try {
552545
return jsonParser().parseDataset(JsonUtil.getJsonObject(datasetJson));
553546
} catch (JsonParsingException | JsonParseException jpe) {
547+
String message = jpe.getLocalizedMessage();
548+
logger.log(Level.SEVERE, "Error parsing dataset JSON. message: {0}", message);
554549
logger.log(Level.SEVERE, "Error parsing dataset json. Json: {0}", datasetJson);
555550
throw new WrappedResponse(error(Status.BAD_REQUEST, "Error parsing Json: " + jpe.getMessage()));
556551
}

src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java

+6-1
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
import edu.harvard.iq.dataverse.api.dto.*;
1515
import edu.harvard.iq.dataverse.api.dto.FieldDTO;
1616
import edu.harvard.iq.dataverse.api.dto.MetadataBlockDTO;
17+
import edu.harvard.iq.dataverse.dataset.DatasetTypeServiceBean;
1718
import edu.harvard.iq.dataverse.license.LicenseServiceBean;
1819
import edu.harvard.iq.dataverse.pidproviders.doi.AbstractDOIProvider;
1920
import edu.harvard.iq.dataverse.pidproviders.handle.HandlePidProvider;
@@ -71,9 +72,13 @@ public class ImportGenericServiceBean {
7172

7273
@EJB
7374
SettingsServiceBean settingsService;
75+
7476
@EJB
7577
LicenseServiceBean licenseService;
7678

79+
@EJB
80+
DatasetTypeServiceBean datasetTypeService;
81+
7782
@PersistenceContext(unitName = "VDCNet-ejbPU")
7883
private EntityManager em;
7984

@@ -110,7 +115,7 @@ public void importXML(String xmlToParse, String foreignFormat, DatasetVersion da
110115
logger.fine(json);
111116
JsonReader jsonReader = Json.createReader(new StringReader(json));
112117
JsonObject obj = jsonReader.readObject();
113-
DatasetVersion dv = new JsonParser(datasetFieldSvc, blockService, settingsService, licenseService).parseDatasetVersion(obj, datasetVersion);
118+
DatasetVersion dv = new JsonParser(datasetFieldSvc, blockService, settingsService, licenseService, datasetTypeService).parseDatasetVersion(obj, datasetVersion);
114119
} catch (XMLStreamException ex) {
115120
//Logger.getLogger("global").log(Level.SEVERE, null, ex);
116121
throw new EJBException("ERROR occurred while parsing XML fragment ("+xmlToParse.substring(0, 64)+"...); ", ex);

src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java

+8-2
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import edu.harvard.iq.dataverse.MetadataBlockServiceBean;
2424
import edu.harvard.iq.dataverse.api.dto.DatasetDTO;
2525
import edu.harvard.iq.dataverse.api.imports.ImportUtil.ImportType;
26+
import edu.harvard.iq.dataverse.dataset.DatasetTypeServiceBean;
2627
import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
2728
import edu.harvard.iq.dataverse.engine.command.exception.CommandException;
2829
import edu.harvard.iq.dataverse.engine.command.impl.CreateDatasetVersionCommand;
@@ -104,8 +105,13 @@ public class ImportServiceBean {
104105

105106
@EJB
106107
IndexServiceBean indexService;
108+
107109
@EJB
108110
LicenseServiceBean licenseService;
111+
112+
@EJB
113+
DatasetTypeServiceBean datasetTypeService;
114+
109115
/**
110116
* This is just a convenience method, for testing migration. It creates
111117
* a dummy dataverse with the directory name as dataverse name & alias.
@@ -264,7 +270,7 @@ public Dataset doImportHarvestedDataset(DataverseRequest dataverseRequest, Harve
264270
JsonObject obj = JsonUtil.getJsonObject(json);
265271
//and call parse Json to read it into a dataset
266272
try {
267-
JsonParser parser = new JsonParser(datasetfieldService, metadataBlockService, settingsService, licenseService, harvestingClient);
273+
JsonParser parser = new JsonParser(datasetfieldService, metadataBlockService, settingsService, licenseService, datasetTypeService, harvestingClient);
268274
parser.setLenient(true);
269275
Dataset ds = parser.parseDataset(obj);
270276

@@ -417,7 +423,7 @@ public JsonObjectBuilder doImport(DataverseRequest dataverseRequest, Dataverse o
417423
JsonObject obj = JsonUtil.getJsonObject(json);
418424
//and call parse Json to read it into a dataset
419425
try {
420-
JsonParser parser = new JsonParser(datasetfieldService, metadataBlockService, settingsService, licenseService);
426+
JsonParser parser = new JsonParser(datasetfieldService, metadataBlockService, settingsService, licenseService, datasetTypeService);
421427
parser.setLenient(!importType.equals(ImportType.NEW));
422428
Dataset ds = parser.parseDataset(obj);
423429

0 commit comments

Comments
 (0)