Skip to content

Commit 6d15e14

Browse files
bbpennelbbpennel
committed
Allow connections to close before making additional requests during object creation, to avoid exhausting the connection pool
1 parent 967d876 commit 6d15e14

File tree

10 files changed

+85
-32
lines changed

10 files changed

+85
-32
lines changed

‎.travis.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ matrix:
77
- gem install sass
88
services:
99
- redis-server
10-
script: mvn verify -pl access,access-common,admin,deposit,fcrepo-clients,metadata,persistence,security,services,services-camel,solr-ingest,solr-search,sword-server,migration-util
10+
script: mvn verify -pl migration-util
1111
sudo: false
1212
cache:
1313
directories:

‎fcrepo-clients/src/main/java/edu/unc/lib/dl/fcrepo4/LdpContainerFactory.java

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@
1616
package edu.unc.lib.dl.fcrepo4;
1717

1818
import static edu.unc.lib.dl.util.RDFModelUtil.TURTLE_MIMETYPE;
19+
import static java.nio.charset.StandardCharsets.UTF_8;
20+
import static org.slf4j.LoggerFactory.getLogger;
1921

2022
import java.io.ByteArrayInputStream;
2123
import java.io.IOException;
@@ -27,6 +29,7 @@
2729
import org.fcrepo.client.FcrepoOperationFailedException;
2830
import org.fcrepo.client.FcrepoResponse;
2931
import org.fcrepo.client.FedoraTypes;
32+
import org.slf4j.Logger;
3033

3134
import edu.unc.lib.dl.fedora.FedoraException;
3235

@@ -38,6 +41,8 @@
3841
*/
3942
public class LdpContainerFactory {
4043

44+
private static final Logger log = getLogger(LdpContainerFactory.class);
45+
4146
private FcrepoClient client;
4247

4348
private static final String DIRECT_CONTAINER_TTL =
@@ -127,9 +132,10 @@ public URI createIndirectContainer(URI membershipResource, Property memberRelati
127132

128133
private URI createLdpContainer(URI membershipResource, String interactionModel, String relationTtl, String name)
129134
throws FedoraException, IOException {
135+
log.debug("Creating {} container in {} with name {}", interactionModel, membershipResource, name);
130136
try (FcrepoResponse response = client.post(membershipResource)
131137
.addInteractionModel(interactionModel)
132-
.body(new ByteArrayInputStream(relationTtl.getBytes(StandardCharsets.UTF_8)), TURTLE_MIMETYPE)
138+
.body(new ByteArrayInputStream(relationTtl.getBytes(UTF_8)), TURTLE_MIMETYPE)
133139
.slug(name)
134140
.perform()) {
135141

‎fcrepo-clients/src/main/java/edu/unc/lib/dl/fcrepo4/RepositoryObjectFactory.java

Lines changed: 29 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import static org.fcrepo.client.ExternalContentHandling.PROXY;
2222
import static org.fcrepo.client.FedoraTypes.LDP_NON_RDF_SOURCE;
2323
import static org.fcrepo.client.LinkHeaderConstants.DESCRIBEDBY_REL;
24+
import static org.slf4j.LoggerFactory.getLogger;
2425

2526
import java.io.ByteArrayInputStream;
2627
import java.io.IOException;
@@ -41,6 +42,7 @@
4142
import org.fcrepo.client.FcrepoClient;
4243
import org.fcrepo.client.FcrepoOperationFailedException;
4344
import org.fcrepo.client.FcrepoResponse;
45+
import org.slf4j.Logger;
4446

4547
import edu.unc.lib.dl.fedora.ChecksumMismatchException;
4648
import edu.unc.lib.dl.fedora.FedoraException;
@@ -61,6 +63,7 @@
6163
*
6264
*/
6365
public class RepositoryObjectFactory {
66+
private static final Logger log = getLogger(RepositoryObjectFactory.class);
6467

6568
private LdpContainerFactory ldpFactory;
6669

@@ -88,17 +91,24 @@ public DepositRecord createDepositRecord(Model model) throws FedoraException {
8891
public DepositRecord createDepositRecord(PID pid, Model model) throws FedoraException {
8992
URI path = pid.getRepositoryUri();
9093

94+
log.debug("Creating deposit record {}", pid.getId());
9195
// Add the deposit record type to the object being created
9296
model = populateModelTypes(path, model, Arrays.asList(Cdr.DepositRecord));
9397

94-
try (FcrepoResponse response = getClient().put(path)
95-
.body(RDFModelUtil.streamModel(model), TURTLE_MIMETYPE)
96-
.perform()) {
97-
URI createdUri = response.getLocation();
98+
log.debug("Streaming model and requesting creation of {}", pid.getId());
99+
try {
100+
URI createdUri;
101+
try (FcrepoResponse response = getClient().put(path)
102+
.body(RDFModelUtil.streamModel(model), TURTLE_MIMETYPE)
103+
.perform()) {
104+
createdUri = response.getLocation();
105+
}
98106
// Add the manifests container
107+
log.debug("Created with location {}, adding manifest container", createdUri);
99108
ldpFactory.createDirectContainer(createdUri, Cdr.hasManifest,
100109
RepositoryPathConstants.DEPOSIT_MANIFEST_CONTAINER);
101110

111+
log.debug("Adding metadata container to {}", createdUri);
102112
// Add container for metadata objects
103113
addMetadataContainer(createdUri);
104114

@@ -108,6 +118,7 @@ public DepositRecord createDepositRecord(PID pid, Model model) throws FedoraExce
108118
throw ClientFaultResolver.resolve(e);
109119
}
110120

121+
log.debug("Retrieving created deposit record object {}", pid.getId());
111122
DepositRecord depositRecord = new DepositRecord(pid, repoObjDriver, this);
112123
return depositRecord;
113124
}
@@ -284,10 +295,13 @@ public FileObject createFileObject(PID pid, Model model) throws FedoraException
284295
// Add types to the object being created
285296
model = populateModelTypes(path, model, Arrays.asList(Cdr.FileObject, PcdmModels.Object));
286297

287-
try (FcrepoResponse response = getClient().put(path)
288-
.body(RDFModelUtil.streamModel(model), TURTLE_MIMETYPE)
289-
.perform()) {
290-
URI createdUri = response.getLocation();
298+
try {
299+
URI createdUri;
300+
try (FcrepoResponse response = getClient().put(path)
301+
.body(RDFModelUtil.streamModel(model), TURTLE_MIMETYPE)
302+
.perform()) {
303+
createdUri = response.getLocation();
304+
}
291305

292306
// Add container for metadata objects
293307
addMetadataContainer(createdUri);
@@ -673,11 +687,13 @@ private void persistTripleToFedora(URI subject, String sparqlUpdate) {
673687
}
674688

675689
private URI createContentContainerObject(URI path, Model model) throws FedoraException {
676-
try (FcrepoResponse response = getClient().put(path)
677-
.body(RDFModelUtil.streamModel(model), TURTLE_MIMETYPE)
678-
.perform()) {
679-
680-
URI createdUri = response.getLocation();
690+
try {
691+
URI createdUri;
692+
try (FcrepoResponse response = getClient().put(path)
693+
.body(RDFModelUtil.streamModel(model), TURTLE_MIMETYPE)
694+
.perform()) {
695+
createdUri = response.getLocation();
696+
}
681697

682698
// Add container for metadata objects
683699
addMetadataContainer(createdUri);

‎fcrepo-clients/src/main/java/edu/unc/lib/dl/model/DatastreamPids.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ public static PID getTechnicalMetadataPid(PID pid) {
6161
}
6262

6363
public static PID getDepositManifestPid(PID pid, String name) {
64-
String path = URIUtil.join(pid.getRepositoryPath(), DEPOSIT_MANIFEST_CONTAINER, name);
64+
String path = URIUtil.join(pid.getRepositoryPath(), DEPOSIT_MANIFEST_CONTAINER, name.toLowerCase());
6565
return PIDs.get(path);
6666
}
6767

‎migration-util/src/main/java/edu/unc/lib/dcr/migration/deposit/DepositRecordTransformer.java

Lines changed: 25 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import java.net.URI;
3030
import java.nio.file.Files;
3131
import java.nio.file.Path;
32+
import java.nio.file.Paths;
3233
import java.util.List;
3334
import java.util.concurrent.RecursiveAction;
3435

@@ -132,11 +133,14 @@ protected void compute() {
132133
log.info("Ingesting deposit record {} as {}", bxc3Pid.getId(), bxc5Pid.getRepositoryPath());
133134
DepositRecord depRecord = repoObjFactory.createDepositRecord(bxc5Pid, bxc5Model);
134135

135-
transformAndPopulatePremis(depRecord);
136+
log.info("Adding manifests for {}", bxc3Pid.getId());
136137
addManifests();
138+
log.info("Transforming premis for {}", bxc3Pid.getId());
139+
transformAndPopulatePremis(depRecord);
137140
// Need this to be last
141+
log.info("Overriding modification time for {}", bxc3Pid.getId());
138142
overrideLastModified(bxc3Resc, depRecord);
139-
} catch(Exception e) {
143+
} catch (Exception e) {
140144
tx.cancelAndIgnore();
141145
throw e;
142146
} finally {
@@ -203,7 +207,8 @@ private void transformAndPopulatePremis(DepositRecord depRecord) {
203207
Path transformedPremisPath = Files.createTempFile("premis", ".xml");
204208
try {
205209
PID bxc5Pid = depRecord.getPid();
206-
PremisLogger filePremisLogger = premisLoggerFactory.createPremisLogger(bxc5Pid, transformedPremisPath.toFile());
210+
PremisLogger filePremisLogger = premisLoggerFactory.createPremisLogger(
211+
bxc5Pid, transformedPremisPath.toFile());
207212
DepositRecordPremisToRdfTransformer premisTransformer =
208213
new DepositRecordPremisToRdfTransformer(bxc5Pid, filePremisLogger, originalPremisPath);
209214

@@ -252,9 +257,17 @@ private void addManifests() {
252257
continue;
253258
}
254259

260+
if (Files.notExists(manifestPath)) {
261+
manifestNum++;
262+
log.error("Manifest file {} does not exist for {}", manifestPath, bxc3Pid);
263+
continue;
264+
}
265+
255266
PID manifestPid = getDepositManifestPid(bxc5Pid, dsName);
256267
// Transfer the manifest to its permanent storage location
257268
URI manifestStoredUri = transferSession.transfer(manifestPid, manifestPath.toUri());
269+
log.error("Transferred manifest {}, exists? {}", manifestStoredUri,
270+
Files.exists(Paths.get(manifestStoredUri)));
258271

259272
// Populate manifest timestamps
260273
Model manifestModel = ModelFactory.createDefaultModel();
@@ -264,8 +277,15 @@ private void addManifests() {
264277
selfResc.addProperty(Fcrepo4Repository.created, created, XSDDatatype.XSDdateTime);
265278

266279
// Create the manifest in fedora
267-
repoObjFactory.createOrUpdateBinary(manifestPid, manifestStoredUri, label,
268-
mimetype, null, md5, manifestModel);
280+
try {
281+
repoObjFactory.createOrUpdateBinary(manifestPid, manifestStoredUri, dsName,
282+
mimetype, null, md5, manifestModel);
283+
} catch (Exception e) {
284+
log.error("Failed stuff for {}", manifestStoredUri, e);
285+
} finally {
286+
log.error("After binary create of manifest {}, exists? {}", manifestStoredUri,
287+
Files.exists(Paths.get(manifestStoredUri)));
288+
}
269289

270290
manifestNum++;
271291
// Repeat until no more manifests found

‎migration-util/src/main/java/edu/unc/lib/dcr/migration/premis/DepositRecordPremisToRdfTransformer.java

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,10 @@ private void transformNormalizationEvent(Element eventEl) {
139139
return;
140140
}
141141

142+
if (eventDetail.contains("Deposit recorded")) {
143+
return;
144+
}
145+
142146
// PID assignment normalization jobs become info package creation
143147
if (eventDetail.contains("Assigned PID")) {
144148
createEventBuilder(Premis.InformationPackageCreation, eventEl)
@@ -154,9 +158,6 @@ private void transformNormalizationEvent(Element eventEl) {
154158
log.error("Unknown deposit normalization event for {}, with detail: {}", pid, eventDetail);
155159
return;
156160
}
157-
if (eventDetail.equals("Deposit recorded")) {
158-
return;
159-
}
160161

161162
String format = matcher.group(1);
162163
createEventBuilder(Premis.Ingestion, eventEl)

‎migration-util/src/test/java/edu/unc/lib/dcr/migration/TransformDepositRecordsCommandIT.java

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,7 @@ public void setUp() throws Exception {
9696

9797
out.reset();
9898
System.setOut(new PrintStream(out));
99+
99100
datastreamsPath = tmpFolder.newFolder("datastreams").toPath();
100101
objectsPath = tmpFolder.newFolder("objects").toPath();
101102

@@ -107,7 +108,7 @@ public void setUp() throws Exception {
107108
}
108109

109110
@After
110-
public void restoreStreams() {
111+
public void cleanup() {
111112
System.setOut(originalOut);
112113
System.clearProperty("dcr.migration.index.url");
113114
System.clearProperty("dcr.it.tdr.ingestSource");
@@ -122,14 +123,16 @@ public void transformDepositRecords() throws Exception {
122123

123124
String[] args = new String[] { "tdr", pidListFile.getAbsolutePath(),
124125
"-s", "loc1" };
125-
migrationCommand.execute(args);
126+
int result = migrationCommand.execute(args);
126127

128+
assertEquals("Incorrect exit status", 0, result);
127129
String output = out.toString();
128130
assertTrue("Expected one transformation successful",
129131
output.contains(" 1/1 "));
130132
assertTrue("Expected transformation completed message",
131133
output.contains("Finished transformation"));
132134

135+
133136
DepositRecord depRec = repoObjLoader.getDepositRecord(bxc3Pid);
134137
assertTrue(depRec.getResource().hasProperty(DC.title, "Deposit Record with Manifest"));
135138
assertTrue(depRec.getResource().hasLiteral(Cdr.depositedOnBehalfOf, DEPOSITOR));
@@ -195,7 +198,8 @@ public void transformDepositRecordsGeneratedIds() throws Exception {
195198
String[] args = new String[] { "tdr", pidListFile.getAbsolutePath(),
196199
"-g",
197200
"-s", "loc1" };
198-
migrationCommand.execute(args);
201+
int result = migrationCommand.execute(args);
202+
assertEquals("Incorrect exit status", 0, result);
199203

200204
String output = out.toString();
201205
assertTrue("Expected one transformation successful",

‎migration-util/src/test/java/edu/unc/lib/dcr/migration/deposit/AbstractDepositRecordTransformationIT.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,7 @@ protected void addPremisLog(PID originalPid) throws IOException {
112112

113113
protected BinaryObject getManifestByName(List<BinaryObject> binList, String dsName) {
114114
return binList.stream()
115-
.filter(b -> b.getPid().getComponentPath().endsWith(dsName))
115+
.filter(b -> b.getPid().getComponentPath().endsWith(dsName.toLowerCase()))
116116
.findFirst()
117117
.get();
118118
}

‎migration-util/src/test/java/edu/unc/lib/dcr/migration/deposit/DepositRecordTransformerIT.java

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
import static java.util.stream.Collectors.toList;
2020
import static org.junit.Assert.assertEquals;
2121
import static org.junit.Assert.assertTrue;
22+
import static org.slf4j.LoggerFactory.getLogger;
2223

2324
import java.io.IOException;
2425
import java.nio.file.Files;
@@ -35,6 +36,7 @@
3536
import org.junit.Before;
3637
import org.junit.Test;
3738
import org.junit.runner.RunWith;
39+
import org.slf4j.Logger;
3840
import org.springframework.beans.factory.annotation.Autowired;
3941
import org.springframework.test.context.ContextConfiguration;
4042
import org.springframework.test.context.ContextHierarchy;
@@ -72,6 +74,7 @@
7274
@ContextConfiguration("/spring-test/cdr-client-container.xml")
7375
})
7476
public class DepositRecordTransformerIT extends AbstractDepositRecordTransformationIT {
77+
private static final Logger log = getLogger(DepositRecordTransformerIT.class);
7578

7679
private static Path ingestSourcePath;
7780

@@ -228,7 +231,7 @@ public void transform_DepositRecord_withManifests() throws Exception {
228231

229232
String manifest0Name = "DATA_MANIFEST0";
230233
String manifest0Content = "content for m0";
231-
writeManifestFile(bxc3Pid, manifest0Name, manifest0Content);
234+
Path mPath = writeManifestFile(bxc3Pid, manifest0Name, manifest0Content);
232235
DatastreamVersion manifest0 = new DatastreamVersion(null,
233236
manifest0Name, "0",
234237
FoxmlDocumentBuilder.DEFAULT_CREATED_DATE,
@@ -238,13 +241,15 @@ public void transform_DepositRecord_withManifests() throws Exception {
238241

239242
String manifest1Name = "DATA_MANIFEST1";
240243
String manifest1Content = "additional content";
241-
writeManifestFile(bxc3Pid, manifest1Name, manifest1Content);
244+
Path mPath2 = writeManifestFile(bxc3Pid, manifest1Name, manifest1Content);
242245
DatastreamVersion manifest1 = new DatastreamVersion(null,
243246
manifest1Name, "0",
244247
FoxmlDocumentBuilder.DEFAULT_LAST_MODIFIED,
245248
Integer.toString(manifest1Content.length()),
246249
"text/plain",
247250
null);
251+
log.error("Mpath1 = {} {}", mPath, Files.exists(mPath));
252+
log.error("Mpath2 = {} {}", mPath2, Files.exists(mPath2));
248253

249254
Document foxml = new FoxmlDocumentBuilder(bxc3Pid, "Deposit Record with Manifests")
250255
.relsExtModel(bxc3Model)

‎migration-util/src/test/resources/allowed_external_paths.txt

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
11
http://
2-
https://
3-
file:///
2+
file:/
3+
file:///
4+
https://

0 commit comments

Comments
 (0)