Skip to content

Commit 1ddd40b

Browse files
krosenvoldkrosenvold
committed
Fixed uncontrolled file descriptor/memory usage. This closes #6 
1 parent 92c0628 commit 1ddd40b

File tree

3 files changed

+80
-40
lines changed

3 files changed

+80
-40
lines changed

‎src/main/java/org/codehaus/plexus/archiver/jar/JarArchiver.java

+17-12
Original file line numberDiff line numberDiff line change
@@ -23,10 +23,9 @@
2323
import java.io.*;
2424
import java.util.*;
2525

26-
import javax.annotation.WillClose;
27-
2826
import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
2927
import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream;
28+
import org.apache.commons.compress.parallel.InputStreamSupplier;
3029
import org.codehaus.plexus.archiver.ArchiverException;
3130
import org.codehaus.plexus.archiver.zip.ConcurrentJarCreator;
3231
import org.codehaus.plexus.archiver.zip.ZipArchiver;
@@ -333,7 +332,8 @@ private void writeManifest( ConcurrentJarCreator zOut, Manifest manifest )
333332
manifest.write( baos );
334333

335334
ByteArrayInputStream bais = new ByteArrayInputStream( baos.toByteArray() );
336-
super.zipFile( bais, zOut, MANIFEST_NAME, System.currentTimeMillis(), null, DEFAULT_FILE_MODE, null );
335+
super.zipFile( createInputStreamSupplier( bais ), zOut, MANIFEST_NAME, System.currentTimeMillis(), null,
336+
DEFAULT_FILE_MODE, null );
337337
super.initZipOutputStream( zOut );
338338
}
339339

@@ -434,21 +434,23 @@ private void createIndexList( ConcurrentJarCreator zOut )
434434

435435
ByteArrayInputStream bais = new ByteArrayInputStream( baos.toByteArray() );
436436

437-
super.zipFile( bais, zOut, INDEX_NAME, System.currentTimeMillis(), null, DEFAULT_FILE_MODE, null );
437+
super.zipFile( createInputStreamSupplier( bais ), zOut, INDEX_NAME, System.currentTimeMillis(), null,
438+
DEFAULT_FILE_MODE, null );
438439
}
439440

440441
/**
441442
* Overridden from Zip class to deal with manifests and index lists.
442443
*/
443-
protected void zipFile( @WillClose InputStream is, ConcurrentJarCreator zOut, String vPath, long lastModified, File fromArchive,
444+
protected void zipFile( InputStreamSupplier is, ConcurrentJarCreator zOut, String vPath,
445+
long lastModified, File fromArchive,
444446
int mode, String symlinkDestination )
445447
throws IOException, ArchiverException
446448
{
447449
if ( MANIFEST_NAME.equalsIgnoreCase( vPath ) )
448450
{
449451
if ( !doubleFilePass || skipWriting )
450452
{
451-
filesetManifest( fromArchive, is );
453+
filesetManifest( fromArchive, is.get() );
452454
}
453455
}
454456
else if ( INDEX_NAME.equalsIgnoreCase( vPath ) && index )
@@ -483,7 +485,7 @@ private void filesetManifest( File file, InputStream is )
483485
manifest = getManifest( file );
484486
}
485487
}
486-
else if ( ( filesetManifestConfig != null ) && filesetManifestConfig != FilesetManifestConfig.skip)
488+
else if ( ( filesetManifestConfig != null ) && filesetManifestConfig != FilesetManifestConfig.skip )
487489
{
488490
// we add this to our group of fileset manifests
489491
getLogger().debug( "Found manifest to merge in file " + file );
@@ -522,16 +524,17 @@ protected boolean createEmptyZip( File zipFile )
522524
try
523525
{
524526
getLogger().debug( "Building MANIFEST-only jar: " + getDestFile().getAbsolutePath() );
525-
zipArchiveOutputStream = new ZipArchiveOutputStream( bufferedOutputStream( fileOutputStream( getDestFile(), "jar" ) ));
527+
zipArchiveOutputStream =
528+
new ZipArchiveOutputStream( bufferedOutputStream( fileOutputStream( getDestFile(), "jar" ) ) );
526529

527-
zipArchiveOutputStream.setEncoding(getEncoding());
530+
zipArchiveOutputStream.setEncoding( getEncoding() );
528531
if ( isCompress() )
529532
{
530-
zipArchiveOutputStream.setMethod(ZipArchiveOutputStream.DEFLATED);
533+
zipArchiveOutputStream.setMethod( ZipArchiveOutputStream.DEFLATED );
531534
}
532535
else
533536
{
534-
zipArchiveOutputStream.setMethod(ZipArchiveOutputStream.STORED);
537+
zipArchiveOutputStream.setMethod( ZipArchiveOutputStream.STORED );
535538
}
536539
ConcurrentJarCreator ps = new ConcurrentJarCreator(Runtime.getRuntime().availableProcessors());
537540
initZipOutputStream( ps );
@@ -589,7 +592,9 @@ public void reset()
589592

590593
public enum FilesetManifestConfig
591594
{
592-
skip, merge, mergewithoutmain
595+
skip,
596+
merge,
597+
mergewithoutmain
593598
}
594599

595600
/**

‎src/main/java/org/codehaus/plexus/archiver/zip/AbstractZipArchiver.java

+60-24
Original file line numberDiff line numberDiff line change
@@ -21,9 +21,6 @@
2121
import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream;
2222
import org.apache.commons.compress.archivers.zip.ZipEncoding;
2323
import org.apache.commons.compress.archivers.zip.ZipEncodingHelper;
24-
import java.util.concurrent.ExecutionException;
25-
import java.util.zip.CRC32;
26-
2724
import org.apache.commons.compress.parallel.InputStreamSupplier;
2825
import org.codehaus.plexus.archiver.AbstractArchiver;
2926
import org.codehaus.plexus.archiver.ArchiveEntry;
@@ -47,6 +44,8 @@
4744
import java.io.SequenceInputStream;
4845
import java.util.Hashtable;
4946
import java.util.Stack;
47+
import java.util.concurrent.ExecutionException;
48+
import java.util.zip.CRC32;
5049

5150
import static org.codehaus.plexus.archiver.util.Streams.bufferedOutputStream;
5251
import static org.codehaus.plexus.archiver.util.Streams.fileOutputStream;
@@ -82,7 +81,7 @@ public abstract class AbstractZipArchiver
8281

8382
protected final Hashtable<String, String> entries = new Hashtable<String, String>();
8483

85-
protected final AddedDirs addedDirs = new AddedDirs();
84+
protected final AddedDirs addedDirs = new AddedDirs();
8685

8786
private static final long EMPTY_CRC = new CRC32().getValue();
8887

@@ -115,7 +114,8 @@ public abstract class AbstractZipArchiver
115114
* <p/>
116115
* plexus-archiver chooses to round up.
117116
* <p/>
118-
* Java versions up to java7 round timestamp down, which means we add a heuristic value (which is slightly questionable)
117+
* Java versions up to java7 round timestamp down, which means we add a heuristic value (which is slightly
118+
* questionable)
119119
* Java versions from 8 and up round timestamp up.
120120
* s
121121
*/
@@ -407,16 +407,16 @@ private void addParentDirs(ArchiveEntry archiveEntry, File baseDir, String entry
407407

408408
/**
409409
* Adds a new entry to the archive, takes care of duplicates as well.
410-
*
411-
* @param in the stream to read data for the entry from.
410+
* @param in the stream to read data for the entry from.
412411
* @param zOut the stream to write to.
413412
* @param vPath the name this entry shall have in the archive.
414413
* @param lastModified last modification time for the entry.
415414
* @param fromArchive the original archive we are copying this
416415
* @param symlinkDestination
417416
*/
418417
@SuppressWarnings( { "JavaDoc" } )
419-
protected void zipFile( @WillClose InputStream in, ConcurrentJarCreator zOut, String vPath, long lastModified,
418+
protected void zipFile( InputStreamSupplier in, ConcurrentJarCreator zOut, String vPath,
419+
long lastModified,
420420
File fromArchive, int mode, String symlinkDestination )
421421
throws IOException, ArchiverException
422422
{
@@ -429,16 +429,7 @@ protected void zipFile( @WillClose InputStream in, ConcurrentJarCreator zOut, St
429429
ZipArchiveEntry ze = new ZipArchiveEntry( vPath );
430430
setTime( ze, lastModified );
431431

432-
byte[] header = new byte[4];
433-
int read = in.read( header );
434-
435-
boolean compressThis = doCompress;
436-
if ( !recompressAddedZips && isZipHeader( header ) )
437-
{
438-
compressThis = false;
439-
}
440-
441-
ze.setMethod( compressThis ? ZipArchiveEntry.DEFLATED : ZipArchiveEntry.STORED );
432+
ze.setMethod( doCompress ? ZipArchiveEntry.DEFLATED : ZipArchiveEntry.STORED );
442433
ze.setUnixMode( UnixStat.FILE_FLAG | mode );
443434

444435
InputStream payload;
@@ -447,13 +438,12 @@ protected void zipFile( @WillClose InputStream in, ConcurrentJarCreator zOut, St
447438
ZipEncoding enc = ZipEncodingHelper.getZipEncoding( getEncoding() );
448439
final byte[] bytes = enc.encode( symlinkDestination ).array();
449440
payload = new ByteArrayInputStream( bytes );
441+
zOut.addArchiveEntry( ze, createInputStreamSupplier( payload ) );
450442
}
451443
else
452444
{
453-
payload = maybeSequence( header, read, in );
445+
zOut.addArchiveEntry( ze, wrappedRecompressor( ze, in ) );
454446
}
455-
zOut.addArchiveEntry( ze, createInputStreamSupplier( payload ) );
456-
457447
}
458448
}
459449

@@ -476,7 +466,7 @@ private boolean isZipHeader( byte[] header )
476466
* @param vPath the name this entry shall have in the archive
477467
*/
478468
@SuppressWarnings( { "JavaDoc" } )
479-
protected void zipFile( ArchiveEntry entry, ConcurrentJarCreator zOut, String vPath )
469+
protected void zipFile( final ArchiveEntry entry, ConcurrentJarCreator zOut, String vPath )
480470
throws IOException, ArchiverException
481471
{
482472
final PlexusIoResource resource = entry.getResource();
@@ -487,7 +477,21 @@ protected void zipFile( ArchiveEntry entry, ConcurrentJarCreator zOut, String vP
487477

488478
final boolean b = entry.getResource() instanceof SymlinkDestinationSupplier;
489479
String symlinkTarget = b ? ( (SymlinkDestinationSupplier) entry.getResource() ).getSymlinkDestination() : null;
490-
InputStream in = entry.getInputStream();
480+
InputStreamSupplier in = new InputStreamSupplier()
481+
{
482+
@Override
483+
public InputStream get()
484+
{
485+
try
486+
{
487+
return entry.getInputStream();
488+
}
489+
catch ( IOException e )
490+
{
491+
throw new RuntimeException( e );
492+
}
493+
}
494+
};
491495
try
492496
{
493497
zipFile( in, zOut, vPath, resource.getLastModified(), null, entry.getMode(), symlinkTarget );
@@ -584,7 +588,39 @@ protected void zipDir( PlexusIoResource dir, ConcurrentJarCreator zOut, String v
584588
}
585589
}
586590

587-
private InputStreamSupplier createInputStreamSupplier( final InputStream inputStream )
591+
592+
private InputStreamSupplier wrappedRecompressor( final ZipArchiveEntry ze, final InputStreamSupplier other )
593+
{
594+
595+
return new InputStreamSupplier()
596+
{
597+
public InputStream get()
598+
{
599+
InputStream is = other.get();
600+
byte[] header = new byte[4];
601+
try
602+
{
603+
int read = is.read( header );
604+
boolean compressThis = doCompress;
605+
if ( !recompressAddedZips && isZipHeader( header ) )
606+
{
607+
compressThis = false;
608+
}
609+
610+
ze.setMethod( compressThis ? ZipArchiveEntry.DEFLATED : ZipArchiveEntry.STORED );
611+
612+
return maybeSequence( header, read, is );
613+
}
614+
catch ( IOException e )
615+
{
616+
throw new RuntimeException( e );
617+
}
618+
619+
}
620+
};
621+
}
622+
623+
protected InputStreamSupplier createInputStreamSupplier( final InputStream inputStream )
588624
{
589625
return new InputStreamSupplier()
590626
{

‎src/test/java/org/codehaus/plexus/archiver/jar/JarArchiverTest.java

+3-4
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,13 @@
11
package org.codehaus.plexus.archiver.jar;
22

3+
import junit.framework.TestCase;
34
import org.codehaus.plexus.archiver.ArchiverException;
45

56
import java.io.File;
67
import java.io.FileOutputStream;
78
import java.io.IOException;
89
import java.util.Random;
910

10-
import junit.framework.TestCase;
11-
1211
public class JarArchiverTest
1312
extends TestCase
1413
{
@@ -53,12 +52,12 @@ public void testVeryLargeJar()
5352
tmpDir.delete();
5453
tmpDir.mkdirs();
5554
Random rand = new Random();
56-
for ( int i = 0; i < 15000; i++ )
55+
for ( int i = 0; i < 45000; i++ )
5756
{
5857
File f = new File( tmpDir, "file" + i );
5958
f.deleteOnExit();
6059
FileOutputStream out = new FileOutputStream(f);
61-
byte[] data = new byte[10240]; // 10kb per file
60+
byte[] data = new byte[512]; // 512bytes per file
6261
rand.nextBytes( data );
6362
out.write( data );
6463
out.flush();

0 commit comments

Comments
 (0)