Skip to content

Commit

Permalink
LDEV-5034 include directories with compressTar, set mode
Browse files Browse the repository at this point in the history 
https://luceeserver.atlassian.net/browse/LDEV-5034
  • Loading branch information
@zspitzer
zspitzer committed Dec 29, 2024
1 parent fb06afd commit 6952bac
Show file tree
Hide file tree
Showing 3 changed files with 194 additions and 85 deletions.
44 changes: 26 additions & 18 deletions core/src/main/java/lucee/commons/io/compress/CompressUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -203,11 +203,12 @@ private static void extractTar(Resource tarFile, Resource targetDir) throws IOEx
return;
}

// read the zip file and build a query from its contents
// read the tar file and extract its contents
TarArchiveInputStream tis = null;
try {
tis = new TarArchiveInputStream(IOUtil.toBufferedInputStream(tarFile.getInputStream()));
TarArchiveEntry entry;
int mode;
while ((entry = tis.getNextTarEntry()) != null) {
// print.ln(entry);
Resource target = targetDir.getRealResource(entry.getName());
Expand All @@ -220,6 +221,10 @@ private static void extractTar(Resource tarFile, Resource targetDir) throws IOEx
IOUtil.copy(tis, target, false);
}
target.setLastModified(entry.getModTime().getTime());
mode = entry.getMode();
if (mode > 0) {
target.setMode(ModeUtil.extractPermissions(mode, false));
}
}
}
finally {
Expand Down Expand Up @@ -544,29 +549,32 @@ public static void compressTar(String parent, Resource[] sources, TarArchiveOutp
}

private static void compressTar(String parent, Resource source, TarArchiveOutputStream tos, int mode) throws IOException {
if (source.isFile()) {
// TarEntry entry = (source instanceof FileResource)?new TarEntry((FileResource)source):new
// TarEntry(parent);
TarArchiveEntry entry = new TarArchiveEntry(parent);
String _parent = parent;
if (!source.isFile() && parent.charAt(parent.length() - 1) != '/')
_parent += "/"; // indicates this is a directory

entry.setName(parent);
TarArchiveEntry entry = new TarArchiveEntry(_parent);

// mode
if (mode > 0) entry.setMode(ModeUtil.extractPermissions(mode, false));
else if ((mode = source.getMode()) > 0) entry.setMode(ModeUtil.extractPermissions(mode, false));
entry.setName(parent);
// mode
if (mode > 0) entry.setMode(ModeUtil.extractPermissions(mode, false));
else if (source.getMode() > 0) entry.setMode(ModeUtil.extractPermissions(source.getMode(), false));

if (source.isFile())
entry.setSize(source.length());
entry.setModTime(source.lastModified());
tos.putArchiveEntry(entry);
try {
entry.setModTime(source.lastModified());
tos.putArchiveEntry(entry);
try {
if (source.isFile())
IOUtil.copy(source, tos, false);
}
finally {
tos.closeArchiveEntry();
}
}
else if (source.isDirectory()) {
compressTar(parent, source.listResources(), tos, mode);
finally {
tos.closeArchiveEntry();
}

if (source.isDirectory()) {
Resource[] sources = source.listResources();
compressTar(parent, sources, tos, mode);
}
}

Expand Down
2 changes: 1 addition & 1 deletion core/src/main/java/resource/fld/core-base.fld
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2638,7 +2638,7 @@ sort direction:
<required>no</required>
<default>777</default>
<description>
Mode of the values
Mode to store the files, pass 0 preserve the source file modes
</description>
</argument>
<return>
Expand Down
233 changes: 167 additions & 66 deletions test/tickets/LDEV5034.cfc
View file
Original file line number Diff line number Diff line change
@@ -1,80 +1,160 @@
component extends="org.lucee.cfml.test.LuceeTestCase" {
component extends="org.lucee.cfml.test.LuceeTestCase" {

function beforeAll(){
variables.dir = getTempDirectory() & "LDEV-5034/";
if ( directoryExists( dir ) )
directoryDelete( dir, true );
directoryCreate( dir );
variables.baseDir = getTempDirectory() & "LDEV-5034/";
if ( directoryExists( baseDir ) )
directoryDelete( baseDir, true );
directoryCreate( baseDir );
};

function afterAll(){
if ( directoryExists( dir ) ){
directoryDelete( dir, true );
if ( directoryExists( baseDir ) ){
directoryDelete( baseDir, true );
};
};

function run( testResults, testBox ){
describe( "fileSetAccessMode", function(){
it( title="test access modes", skip=isNotUnix(), body=function(){
var tests = [];

arrayAppend( tests, _dir( dir, "755", "755" ) );
arrayAppend( tests, _dir( dir, "777", "777" ) );
arrayAppend( tests, _dir( dir, "644", "644" ) );

var files = directoryList( dir, true, "query");
var st = QueryToStruct( files, "name" );

// loop array=st index="local.item"{
// systemOutput( item, true );
// }

arrayAppend( tests, _file( dir, "644.txt", "644" ) );
arrayAppend( tests, _file( dir, "743.txt", "743" ) );
arrayAppend( tests, _file( dir, "043.txt", "443" ) );
arrayAppend( tests, _file( dir, "400.txt", "400" ) );

var files = directoryList( dir, true, "query");
var st = QueryToStruct( files, "name" );

// loop collection=st item="local.item"{
// systemOutput( item, true );
// }
// loop array=tests item="local.test" {
// systemOutput( test, true );
// }
loop array=tests item="local.test" {
// systemOutput( test, true );
var key = mid( test.name, len( dir ) + 1 );
// systemOutput( key, true );
expect( st ).toHaveKey( key );
// systemOutput( st[ key ], true );
expect( test.mode ).toBe( st[ key ].mode );
}

var tar = getTempFile( getTempDirectory(), "LDEV-5034", ".tar.gz" );
compress( "tgz", dir, tar );

var dest = getTempDirectory() & "LDEV-5034-" & createUUID() & "/";
directoryCreate( dest );
extract( "tgz", tar, dest );

var extractedFiles = directoryList( dest, true, "query" );
var st2 = QueryToStruct( files, "name" );

files=justFiles(files);
extractedFiles=justFiles(extractedFiles);
expect( files.recordcount ).toBe( extractedFiles.recordcount );
function run( testResults, testBox ){
describe( "Round trip a set of files and dirs thru compress and extract tar", function(){
it( title="just files", skip=isNotUnix(), body=function(){
var srcDir = getTestDir( "files" );
var destDir = getTestDir( "files-extract" );
var sourceFiles = [];
// create some files
arrayAppend( sourceFiles, _file( srcDir, "644.txt", "644" ) );
arrayAppend( sourceFiles, _file( srcDir, "743.txt", "743" ) );
arrayAppend( sourceFiles, _file( srcDir, "043.txt", "443" ) );
arrayAppend( sourceFiles, _file( srcDir, "400.txt", "400" ) );

testRoundTripCompressExtract( srcDir, destDir, sourceFiles );
});

it( title="file and directory", skip=isNotUnix(), body=function(){
var srcDir = getTestDir( "mixed2" );
var destDir = getTestDir( "mixed2-extract" );
var sourceFiles = [];
arrayAppend( sourceFiles, _dir(srcDir, "dir-755", "755" ) );
arrayAppend( sourceFiles, _file(srcDir, "644.txt", "644" ) );
arrayAppend( sourceFiles, _file(srcDir & "dir-755/", "755-400.txt", "400" ) );

testRoundTripCompressExtract( srcDir, destDir, sourceFiles );
});

it( title="files and directories", skip=isNotUnix(), body=function(){
var srcDir = getTestDir( "mixed" );
var destDir = getTestDir( "mixed-extract" );
var sourceFiles = [];
// create some dirs
arrayAppend( sourceFiles, _dir(srcDir, "dir-755", "755" ) );
arrayAppend( sourceFiles, _dir(srcDir, "dir-777", "777" ) );
arrayAppend( sourceFiles, _dir(srcDir, "dir-743", "743" ) );
// create some files
arrayAppend( sourceFiles, _file(srcDir, "644.txt", "644" ) );
arrayAppend( sourceFiles, _file(srcDir, "743.txt", "743" ) );
arrayAppend( sourceFiles, _file(srcDir, "043.txt", "443" ) );
arrayAppend( sourceFiles, _file(srcDir, "400.txt", "400" ) );
// emptyDirs are skipped, add some files to them
arrayAppend( sourceFiles, _file(srcDir & "dir-777/", "777-443.txt", "743" ) );
arrayAppend( sourceFiles, _file(srcDir & "dir-755/", "755-400.txt", "400" ) );
arrayAppend( sourceFiles, _file(srcDir & "dir-743/", "743-777.txt", "777" ) );

testRoundTripCompressExtract( srcDir, destDir, sourceFiles );
});
// only disabled for faster CI
xit( title="test tomcat", skip=isNotUnix(), body=function(){
var download_url =" https://dlcdn.apache.org/tomcat/tomcat-11/v11.0.2/bin/apache-tomcat-11.0.2.tar.gz";
var filename = listLast( download_url, "/" );
var srcDir = getTestDir( "tomcat" );
var destDir = getTestDir( "tomcat-extract" );
http method="get" url=download_url path=getTempDirectory() file=filename throwOnError=true;
var info = fileInfo(getTempDirectory() & filename);
systemOutput(info, true);
extract(format="tgz", source=info.path, target=srcDir);
var sourceFiles = folderToTest( srcDir );
// for (var s in sourceFiles) systemOutput(s, true);
testRoundTripCompressExtract( srcDir, destDir, sourceFiles );
});

} );
}

private function justFiles(qry) {
qry=duplicate(qry);
for(var row=qry.recordcount;row>0;row--) {
if(qry.type[row]!="File") queryDeleteRow(qry,row);
private function testRoundTripCompressExtract( string srcDir, string destDir,
array sourceFiles, boolean debug=false ){
if (arrayLen( arguments.sourceFiles ) == 0)
throw "testRoundTripCompressExtract: sourceFiles was empty?";
if ( arguments.debug ){
systemOutput( "-----------------source files", true );
loop array=arguments.sourceFiles item="local.sourceFile" {
structDelete( sourceFile, "dateLastModified" );
systemOutput( sourceFile, true );
}
}

// compress the source files into a tar archive
var tarFile = getTempFile( getTempDirectory(), "LDEV-5034", ".tar.gz" );
// mode 0 here is important!!!!! default is 777 and ignore source permissions
compress( format="tgz", source=arguments.srcDir, target=tarFile, includeBaseFolder=false, mode=0 );

//systemOutput(tarFile, true);
var dest = arguments.destDir;
extract( "tgz", tarFile, dest );

var files = directoryList( arguments.srcDir, true, "query");
var extractedFiles = directoryList( dest, true, "query" );
QueryAddColumn( extractedFiles, "path" );
loop query=extractedFiles {
var _path = mid( extractedFiles.directory, len( arguments.destDir ) + 1 );
if ( len( _path ) eq 0 )
_path = extractedFiles.name;
else
_path = _path & "/" & extractedFiles.name;

QuerySetCell( extractedFiles, "path", _path, extractedFiles.currentrow );
}
if ( arguments.debug ){
QueryDeleteColumn( extractedFiles, "directory" );
QueryDeleteColumn( extractedFiles, "dateLastModified" );
QueryDeleteColumn( extractedFiles, "attributes" );
}
var stExtracted = QueryToStruct( extractedFiles, "path" );

if ( arguments.debug ){
systemOutput( "-----------------sourceFiles", true );
systemOutput( files.toString(), true );
systemOutput( "-----------------extractedFiles", true );
systemOutput( extractedFiles.toString(), true );
systemOutput( "-----------------stExtracted", true );
loop collection=stExtracted key="local.file" value="local.item" {
systemOutput( item, true );
}
systemOutput("-----------------", true);
}
querySort(qry, "type");

var files = justFiles( files );
var extractedFiles = justFiles( extractedFiles );
expect( files.recordcount ).toBe( extractedFiles.recordcount );

// check that after round tripping thru compress() and extract()
// the final files have the same permissions as the source files
loop array=arguments.sourceFiles item="local.sourceFile" {
if ( arguments.debug )
systemOutput( sourceFile, true );
var filename = mid( sourceFile.name, len( arguments.srcDir ) + 1 );
if ( arguments.debug )
systemOutput( "filename [#filename#]", true );
expect( stExtracted ).toHaveKey( filename );
if ( arguments.debug )
systemOutput( stExtracted[ filename ], true );
expect( stExtracted[ filename ].mode ).toBe( sourceFile.mode , sourceFile.name );
}
}

private function justFiles( srcQry ) {
var qry = duplicate( arguments.srcQry );
for(var row=qry.recordcount; row > 0; row--) {
if ( qry.type[ row ] != "File" )
queryDeleteRow( qry, row);
}
querySort( qry, "type" );
return qry;
}

Expand All @@ -98,10 +178,31 @@ component extends="org.lucee.cfml.test.LuceeTestCase" {
mode: mode,
type: "file"
};
}
}

private function folderToTest( srcDir ){
var files = directoryList( arguments.srcDir, true, "query");
var sources = [];
for (var file in files){
arrayAppend( sources, {
name: file.directory & "/" & file.name,
mode: file.mode,
type: file.type
} );
}
return sources;
}

private function isNotUnix(){
return (server.os.name contains "windows");
return ( server.os.name contains "windows" );
}

private function getTestDir( suffix ){
var testDir = variables.baseDir & arguments.suffix & "/";
if ( directoryExists( testDir ) )
directoryDelete( testDir, true );
directoryCreate( testDir );
return testDir;
};

}

0 comments on commit 6952bac

Please sign in to comment.