Skip to content

Commit bedfc43

Browse files
lfarrelllfarrell
authored
Merge pull request #957 from UNC-Libraries/bxc-2540-migr-deposits
BXC-2540 - Migrate deposit records
2 parents a7270aa + b580cab commit bedfc43

30 files changed

+1956
-60
lines changed

‎fcrepo-clients/src/main/java/edu/unc/lib/dl/fcrepo4/DepositRecord.java

+8-3
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ public BinaryObject addManifest(URI manifestUri, String mimetype)
5757
throws FedoraException {
5858

5959
String filename = StringUtils.substringAfterLast(manifestUri.toString(), "/");
60-
return addManifest(manifestUri, filename, mimetype);
60+
return addManifest(manifestUri, filename, mimetype, null, null);
6161
}
6262

6363
/**
@@ -66,13 +66,18 @@ public BinaryObject addManifest(URI manifestUri, String mimetype)
6666
* @param manifestUri URI of the binary content for this manifest
6767
* @param filename filename for the manifest
6868
* @param mimetype mimetype for the content of the manifest
69+
* @param sha1
70+
* @param md5
6971
* @return representing the newly created manifest object
7072
* @throws FedoraException
7173
*/
72-
public BinaryObject addManifest(URI manifestUri, String filename, String mimetype)
74+
public BinaryObject addManifest(URI manifestUri, String filename, String mimetype, String sha1, String md5)
7375
throws FedoraException {
7476
PID manifestPid = getDepositManifestPid(getPid(), filename);
75-
return repoObjFactory.createOrUpdateBinary(manifestPid, manifestUri, filename, mimetype, null, null, null);
77+
if (mimetype == null) {
78+
mimetype = "text/plain";
79+
}
80+
return repoObjFactory.createOrUpdateBinary(manifestPid, manifestUri, filename, mimetype, sha1, md5, null);
7681
}
7782

7883
/**

‎fcrepo-clients/src/main/java/edu/unc/lib/dl/fcrepo4/LdpContainerFactory.java

+7-1
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@
1616
package edu.unc.lib.dl.fcrepo4;
1717

1818
import static edu.unc.lib.dl.util.RDFModelUtil.TURTLE_MIMETYPE;
19+
import static java.nio.charset.StandardCharsets.UTF_8;
20+
import static org.slf4j.LoggerFactory.getLogger;
1921

2022
import java.io.ByteArrayInputStream;
2123
import java.io.IOException;
@@ -27,6 +29,7 @@
2729
import org.fcrepo.client.FcrepoOperationFailedException;
2830
import org.fcrepo.client.FcrepoResponse;
2931
import org.fcrepo.client.FedoraTypes;
32+
import org.slf4j.Logger;
3033

3134
import edu.unc.lib.dl.fedora.FedoraException;
3235

@@ -38,6 +41,8 @@
3841
*/
3942
public class LdpContainerFactory {
4043

44+
private static final Logger log = getLogger(LdpContainerFactory.class);
45+
4146
private FcrepoClient client;
4247

4348
private static final String DIRECT_CONTAINER_TTL =
@@ -127,9 +132,10 @@ public URI createIndirectContainer(URI membershipResource, Property memberRelati
127132

128133
private URI createLdpContainer(URI membershipResource, String interactionModel, String relationTtl, String name)
129134
throws FedoraException, IOException {
135+
log.debug("Creating {} container in {} with name {}", interactionModel, membershipResource, name);
130136
try (FcrepoResponse response = client.post(membershipResource)
131137
.addInteractionModel(interactionModel)
132-
.body(new ByteArrayInputStream(relationTtl.getBytes(StandardCharsets.UTF_8)), TURTLE_MIMETYPE)
138+
.body(new ByteArrayInputStream(relationTtl.getBytes(UTF_8)), TURTLE_MIMETYPE)
133139
.slug(name)
134140
.perform()) {
135141

‎fcrepo-clients/src/main/java/edu/unc/lib/dl/fcrepo4/RepositoryObjectFactory.java

+29-13
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import static org.fcrepo.client.ExternalContentHandling.PROXY;
2222
import static org.fcrepo.client.FedoraTypes.LDP_NON_RDF_SOURCE;
2323
import static org.fcrepo.client.LinkHeaderConstants.DESCRIBEDBY_REL;
24+
import static org.slf4j.LoggerFactory.getLogger;
2425

2526
import java.io.ByteArrayInputStream;
2627
import java.io.IOException;
@@ -41,6 +42,7 @@
4142
import org.fcrepo.client.FcrepoClient;
4243
import org.fcrepo.client.FcrepoOperationFailedException;
4344
import org.fcrepo.client.FcrepoResponse;
45+
import org.slf4j.Logger;
4446

4547
import edu.unc.lib.dl.fedora.ChecksumMismatchException;
4648
import edu.unc.lib.dl.fedora.FedoraException;
@@ -61,6 +63,7 @@
6163
*
6264
*/
6365
public class RepositoryObjectFactory {
66+
private static final Logger log = getLogger(RepositoryObjectFactory.class);
6467

6568
private LdpContainerFactory ldpFactory;
6669

@@ -88,17 +91,24 @@ public DepositRecord createDepositRecord(Model model) throws FedoraException {
8891
public DepositRecord createDepositRecord(PID pid, Model model) throws FedoraException {
8992
URI path = pid.getRepositoryUri();
9093

94+
log.debug("Creating deposit record {}", pid.getId());
9195
// Add the deposit record type to the object being created
9296
model = populateModelTypes(path, model, Arrays.asList(Cdr.DepositRecord));
9397

94-
try (FcrepoResponse response = getClient().put(path)
95-
.body(RDFModelUtil.streamModel(model), TURTLE_MIMETYPE)
96-
.perform()) {
97-
URI createdUri = response.getLocation();
98+
log.debug("Streaming model and requesting creation of {}", pid.getId());
99+
try {
100+
URI createdUri;
101+
try (FcrepoResponse response = getClient().put(path)
102+
.body(RDFModelUtil.streamModel(model), TURTLE_MIMETYPE)
103+
.perform()) {
104+
createdUri = response.getLocation();
105+
}
98106
// Add the manifests container
107+
log.debug("Created with location {}, adding manifest container", createdUri);
99108
ldpFactory.createDirectContainer(createdUri, Cdr.hasManifest,
100109
RepositoryPathConstants.DEPOSIT_MANIFEST_CONTAINER);
101110

111+
log.debug("Adding metadata container to {}", createdUri);
102112
// Add container for metadata objects
103113
addMetadataContainer(createdUri);
104114

@@ -108,6 +118,7 @@ public DepositRecord createDepositRecord(PID pid, Model model) throws FedoraExce
108118
throw ClientFaultResolver.resolve(e);
109119
}
110120

121+
log.debug("Retrieving created deposit record object {}", pid.getId());
111122
DepositRecord depositRecord = new DepositRecord(pid, repoObjDriver, this);
112123
return depositRecord;
113124
}
@@ -284,10 +295,13 @@ public FileObject createFileObject(PID pid, Model model) throws FedoraException
284295
// Add types to the object being created
285296
model = populateModelTypes(path, model, Arrays.asList(Cdr.FileObject, PcdmModels.Object));
286297

287-
try (FcrepoResponse response = getClient().put(path)
288-
.body(RDFModelUtil.streamModel(model), TURTLE_MIMETYPE)
289-
.perform()) {
290-
URI createdUri = response.getLocation();
298+
try {
299+
URI createdUri;
300+
try (FcrepoResponse response = getClient().put(path)
301+
.body(RDFModelUtil.streamModel(model), TURTLE_MIMETYPE)
302+
.perform()) {
303+
createdUri = response.getLocation();
304+
}
291305

292306
// Add container for metadata objects
293307
addMetadataContainer(createdUri);
@@ -673,11 +687,13 @@ private void persistTripleToFedora(URI subject, String sparqlUpdate) {
673687
}
674688

675689
private URI createContentContainerObject(URI path, Model model) throws FedoraException {
676-
try (FcrepoResponse response = getClient().put(path)
677-
.body(RDFModelUtil.streamModel(model), TURTLE_MIMETYPE)
678-
.perform()) {
679-
680-
URI createdUri = response.getLocation();
690+
try {
691+
URI createdUri;
692+
try (FcrepoResponse response = getClient().put(path)
693+
.body(RDFModelUtil.streamModel(model), TURTLE_MIMETYPE)
694+
.perform()) {
695+
createdUri = response.getLocation();
696+
}
681697

682698
// Add container for metadata objects
683699
addMetadataContainer(createdUri);

‎fcrepo-clients/src/main/java/edu/unc/lib/dl/fcrepo4/RepositoryPaths.java

+2
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
import static edu.unc.lib.dl.fcrepo4.RepositoryPathConstants.CONTENT_BASE;
1919
import static edu.unc.lib.dl.fcrepo4.RepositoryPathConstants.CONTENT_ROOT_ID;
20+
import static edu.unc.lib.dl.fcrepo4.RepositoryPathConstants.DEPOSIT_RECORD_BASE;
2021
import static edu.unc.lib.dl.fcrepo4.RepositoryPathConstants.METADATA_CONTAINER;
2122
import static edu.unc.lib.dl.fcrepo4.RepositoryPathConstants.REPOSITORY_ROOT_ID;
2223

@@ -183,5 +184,6 @@ private static void setContentBase(String uri) {
183184
contentBase = URIUtil.join(baseUri, CONTENT_BASE);
184185
contentBasePid = new FedoraPID(CONTENT_BASE, REPOSITORY_ROOT_ID, null, URI.create(contentBase));
185186
contentRootPid = PIDs.get(URIUtil.join(contentBase, CONTENT_ROOT_ID));
187+
depositRecordBase = URIUtil.join(baseUri, DEPOSIT_RECORD_BASE);
186188
}
187189
}

‎fcrepo-clients/src/main/java/edu/unc/lib/dl/model/DatastreamPids.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ public static PID getTechnicalMetadataPid(PID pid) {
6161
}
6262

6363
public static PID getDepositManifestPid(PID pid, String name) {
64-
String path = URIUtil.join(pid.getRepositoryPath(), DEPOSIT_MANIFEST_CONTAINER, name);
64+
String path = URIUtil.join(pid.getRepositoryPath(), DEPOSIT_MANIFEST_CONTAINER, name.toLowerCase());
6565
return PIDs.get(path);
6666
}
6767

‎fcrepo-clients/src/test/java/edu/unc/lib/dl/fcrepo4/DepositRecordIT.java

+2-4
Original file line numberDiff line numberDiff line change
@@ -93,18 +93,17 @@ public void addManifestsTest() throws Exception {
9393
String mimetype1 = "text/plain";
9494
Path manifestPath = createTempFile("manifest", ".txt");
9595
writeStringToFile(manifestPath.toFile(), bodyString1, UTF_8);
96-
BinaryObject manifest1 = record.addManifest(manifestPath.toUri(), filename1, mimetype1);
96+
BinaryObject manifest1 = record.addManifest(manifestPath.toUri(), filename1, mimetype1, null, null);
9797

9898
assertNotNull(manifest1);
9999
assertEquals(filename1, manifest1.getFilename());
100100
assertEquals(mimetype1, manifest1.getMimetype());
101101

102102
String bodyString2 = "Second manifest";
103-
String filename2 = "manifest2.txt";
104103
String mimetype2 = "text/plain";
105104
Path manifestPath2 = createTempFile("manifest", ".txt");
106105
writeStringToFile(manifestPath2.toFile(), bodyString2, UTF_8);
107-
BinaryObject manifest2 = record.addManifest(manifestPath2.toUri(), filename2, mimetype2);
106+
BinaryObject manifest2 = record.addManifest(manifestPath2.toUri(), mimetype2);
108107

109108
assertNotNull(manifest2);
110109

@@ -122,7 +121,6 @@ public void addManifestsTest() throws Exception {
122121
// Verify that retrieving the manifest returns the correct object
123122
BinaryObject gotManifest2 = record.getManifest(manifest2.getPid());
124123
assertNotNull("Get manifest did not return", gotManifest2);
125-
assertEquals(filename2, gotManifest2.getFilename());
126124
assertEquals(mimetype2, gotManifest2.getMimetype());
127125

128126
String respString2 = new BufferedReader(new InputStreamReader(manifest2.getBinaryStream()))

‎metadata/src/main/java/edu/unc/lib/dl/sparql/SparqlUpdateHelper.java

+3-2
Original file line numberDiff line numberDiff line change
@@ -196,8 +196,9 @@ public static String getObjectAsString(Object object) {
196196
} else if (object instanceof String) {
197197
return '"' + object.toString() + '"';
198198
} else if (object instanceof Literal) {
199-
RDFDatatype type = ((Literal) object).getDatatype();
200-
return '"' + object.toString() + "\"^^<" + type.getURI() + ">";
199+
Literal literal = (Literal) object;
200+
RDFDatatype type = literal.getDatatype();
201+
return '"' + literal.getLexicalForm() + "\"^^<" + type.getURI() + ">";
201202
} else {
202203
RDFDatatype type = TypeMapper.getInstance().getTypeByClass(object.getClass());
203204
if (type == null) {

‎migration-util/pom.xml

+7-1
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
<version>4.0-SNAPSHOT</version>
1313
</parent>
1414
<properties>
15-
<h2.version>1.4.200</h2.version>
15+
<h2.version>1.4.197</h2.version>
1616
</properties>
1717
<build>
1818
<finalName>dcr-migration-util</finalName>
@@ -34,6 +34,12 @@
3434
"org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
3535
<mainClass>edu.unc.lib.dcr.migration.MigrationCLI</mainClass>
3636
</transformer>
37+
<transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
38+
<resource>META-INF/spring.handlers</resource>
39+
</transformer>
40+
<transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
41+
<resource>META-INF/spring.schemas</resource>
42+
</transformer>
3743
</transformers>
3844
<createDependencyReducedPom>false</createDependencyReducedPom>
3945
<artifactSet>

‎migration-util/src/main/java/edu/unc/lib/dcr/migration/MigrationCLI.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
TransformPremis.class,
3838
PathIndexCommand.class,
3939
TransformContentCommand.class,
40+
TransformDepositRecordsCommand.class,
4041
ViewDepositModelCommand.class,
4142
SubmitDepositCommand.class
4243
})
@@ -78,7 +79,7 @@ public class MigrationCLI implements Callable<Integer> {
7879
description = "Port for redis. Default 6379.")
7980
protected int redisPort;
8081

81-
private MigrationCLI() {
82+
protected MigrationCLI() {
8283
}
8384

8485
@Override

‎migration-util/src/main/java/edu/unc/lib/dcr/migration/PathIndexCommand.java

+36-16
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,11 @@ public int populateIndex(
5555
@Parameters(index = "0", description = "Path to file listing FOXML documents.")
5656
Path objectListPath,
5757
@Parameters(index = "1", description = "Path to file listing datastream files.")
58-
Path dsListPath) {
58+
Path dsListPath,
59+
@Option(names = {"-l", "--list-dirs"}, description =
60+
"If set, the object and datastream paths will be treated as"
61+
+ " directories, recursively indexing the files within them")
62+
boolean listDirectories) {
5963

6064
long start = System.currentTimeMillis();
6165
PathIndexingService service = getPathIndexingService();
@@ -65,9 +69,17 @@ public int populateIndex(
6569
output.info("Creating index at path {}", parentCommand.databaseUrl);
6670
service.createIndexTable();
6771
output.info("Populating object files from {}", objectListPath);
68-
service.indexObjects(objectListPath);
72+
if (listDirectories) {
73+
service.indexObjectsFromPath(objectListPath);
74+
} else {
75+
service.indexObjects(objectListPath);
76+
}
6977
output.info("Populating datastream files from {}", dsListPath);
70-
service.indexDatastreams(dsListPath);
78+
if (listDirectories) {
79+
service.indexDatastreamsFromPath(dsListPath);
80+
} else {
81+
service.indexDatastreams(dsListPath);
82+
}
7183
output.info("Finished populating index in {}ms", System.currentTimeMillis() - start);
7284
output.info("{} files were indexed", getPathIndex().countFiles());
7385

@@ -92,22 +104,30 @@ public int getPaths(
92104
}
93105

94106
@Command(name = "list_paths",
95-
description = "List file paths objects and datastreams all objects in the provided list")
107+
description = "List file paths for objects and datastreams")
96108
public int listPaths(
97-
@Parameters(index = "0", description = "File containing the list of object ids")
109+
@Option(names = { "-p", "--pids-file" }, description = "File containing the list of object ids to list")
98110
Path listPath) {
99111

100-
output.info("Listing paths objects listed in {}:", listPath);
101-
try {
102-
Files.lines(listPath).forEach(id -> {
103-
id = id.replace("uuid:", "");
104-
String paths = getPathIndex().getPaths(id).values().stream()
105-
.map(Path::toString)
106-
.collect(joining("\n"));
107-
output.info(paths);
108-
});
109-
} catch (IOException e) {
110-
output.error("Failed to read list file: {}", e.getMessage());
112+
if (listPath == null) {
113+
output.info("Listing all paths:");
114+
String paths = getPathIndex().listPaths().values().stream()
115+
.map(Path::toString)
116+
.collect(joining("\n"));
117+
output.info(paths);
118+
} else {
119+
output.info("Listing paths objects listed in {}:", listPath);
120+
try {
121+
Files.lines(listPath).forEach(id -> {
122+
id = id.replace("uuid:", "");
123+
String paths = getPathIndex().getPaths(id).values().stream()
124+
.map(Path::toString)
125+
.collect(joining("\n"));
126+
output.info(paths);
127+
});
128+
} catch (IOException e) {
129+
output.error("Failed to read list file: {}", e.getMessage());
130+
}
111131
}
112132

113133
getPathIndex().close();

0 commit comments

Comments
 (0)