From 6dd0bb2c0b3e4006a06ff93756cd5ea76a6795b9 Mon Sep 17 00:00:00 2001 From: Raymond Lai Date: Fri, 14 Jul 2023 14:17:12 +0800 Subject: [PATCH] SSH code fixes - Fix broken session caused by unclosed sessions (again) - Use du command at remote to calculate directory size - Implement HybridFile.setLastModified for SSH using remote touch command - Skip calculate hash when given HybridFile.isDirectory is true --- .../hashcalculator/CalculateHashTask.kt | 4 +- .../filemanager/filesystem/HybridFile.java | 190 ++++++++------- .../filesystem/HybridFileParcelable.java | 5 + .../filemanager/filesystem/Operations.java | 15 +- .../filesystem/files/GenericCopyUtil.java | 6 +- .../ftp/NetCopyClientConnectionPool.kt | 1 - .../filesystem/ftp/NetCopyClientUtils.kt | 21 +- .../filesystem/ssh/SFTPClientExt.kt | 61 +++++ .../filesystem/ssh/SshClientUtils.java | 227 ------------------ .../filesystem/ssh/SshClientUtils.kt | 222 +++++++++++++++++ .../ui/activities/MainActivity.java | 1 + .../com/amaze/filemanager/utils/GenericExt.kt | 4 - build.gradle | 2 +- 13 files changed, 420 insertions(+), 339 deletions(-) create mode 100644 app/src/main/java/com/amaze/filemanager/filesystem/ssh/SFTPClientExt.kt delete mode 100644 app/src/main/java/com/amaze/filemanager/filesystem/ssh/SshClientUtils.java create mode 100644 app/src/main/java/com/amaze/filemanager/filesystem/ssh/SshClientUtils.kt diff --git a/app/src/main/java/com/amaze/filemanager/asynchronous/asynctasks/hashcalculator/CalculateHashTask.kt b/app/src/main/java/com/amaze/filemanager/asynchronous/asynctasks/hashcalculator/CalculateHashTask.kt index 43e0dc27bb..6067e9a7c5 100644 --- a/app/src/main/java/com/amaze/filemanager/asynchronous/asynctasks/hashcalculator/CalculateHashTask.kt +++ b/app/src/main/java/com/amaze/filemanager/asynchronous/asynctasks/hashcalculator/CalculateHashTask.kt @@ -45,9 +45,9 @@ class CalculateHashTask( private val log: Logger = LoggerFactory.getLogger(CalculateHashTask::class.java) - private val task: Callable = if (file.isSftp) { + private val task: Callable = if (file.isSftp && !file.isDirectory(context)) { CalculateHashSftpCallback(file) - } else if (file.isFtp) { + } else if (file.isFtp || file.isDirectory(context)) { // Don't do this. Especially when FTPClient requires thread safety. DoNothingCalculateHashCallback() } else { diff --git a/app/src/main/java/com/amaze/filemanager/filesystem/HybridFile.java b/app/src/main/java/com/amaze/filemanager/filesystem/HybridFile.java index 2f91894d0d..649143bc6f 100644 --- a/app/src/main/java/com/amaze/filemanager/filesystem/HybridFile.java +++ b/app/src/main/java/com/amaze/filemanager/filesystem/HybridFile.java @@ -26,6 +26,8 @@ import static com.amaze.filemanager.filesystem.ftp.NetCopyClientConnectionPool.SSH_URI_PREFIX; import static com.amaze.filemanager.filesystem.ftp.NetCopyConnectionInfo.MULTI_SLASH; import static com.amaze.filemanager.filesystem.smb.CifsContexts.SMB_URI_PREFIX; +import static com.amaze.filemanager.filesystem.ssh.SFTPClientExtKt.READ_AHEAD_MAX_UNCONFIRMED_READS; +import static com.amaze.filemanager.filesystem.ssh.SshClientUtils.sftpGetSize; import java.io.File; import java.io.FileInputStream; @@ -39,10 +41,7 @@ import java.net.URLDecoder; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; -import java.text.DateFormat; -import java.text.SimpleDateFormat; import java.util.ArrayList; -import java.util.Calendar; import java.util.EnumSet; import java.util.List; import java.util.Locale; @@ -74,6 +73,7 @@ import com.amaze.filemanager.filesystem.ftp.NetCopyConnectionInfo; import com.amaze.filemanager.filesystem.root.DeleteFileCommand; import com.amaze.filemanager.filesystem.root.ListFilesCommand; +import com.amaze.filemanager.filesystem.ssh.SFTPClientExtKt; import com.amaze.filemanager.filesystem.ssh.SFtpClientTemplate; import com.amaze.filemanager.filesystem.ssh.SshClientSessionTemplate; import com.amaze.filemanager.filesystem.ssh.SshClientUtils; @@ -82,7 +82,6 @@ import com.amaze.filemanager.ui.dialogs.GeneralDialogCreation; import com.amaze.filemanager.ui.fragments.preferencefragments.PreferencesConstants; import com.amaze.filemanager.utils.DataUtils; -import com.amaze.filemanager.utils.Function; import com.amaze.filemanager.utils.OTGUtil; import com.amaze.filemanager.utils.OnFileFound; import com.amaze.filemanager.utils.Utils; @@ -94,15 +93,16 @@ import android.content.Context; import android.net.Uri; import android.os.Build; +import android.text.TextUtils; import android.text.format.Formatter; import android.widget.Toast; import androidx.annotation.NonNull; import androidx.annotation.Nullable; +import androidx.arch.core.util.Function; import androidx.documentfile.provider.DocumentFile; import androidx.preference.PreferenceManager; -import io.reactivex.Flowable; import io.reactivex.Single; import io.reactivex.SingleObserver; import io.reactivex.android.schedulers.AndroidSchedulers; @@ -119,6 +119,7 @@ import net.schmizz.sshj.connection.channel.direct.Session; import net.schmizz.sshj.sftp.FileMode; import net.schmizz.sshj.sftp.RemoteFile; +import net.schmizz.sshj.sftp.RemoteResourceInfo; import net.schmizz.sshj.sftp.SFTPClient; import net.schmizz.sshj.sftp.SFTPException; @@ -302,11 +303,11 @@ public long lastModified() { switch (mode) { case SFTP: final Long returnValue = - NetCopyClientUtils.INSTANCE.execute( - new SFtpClientTemplate(path, false) { + SshClientUtils.execute( + new SFtpClientTemplate(path, true) { @Override public Long execute(@NonNull SFTPClient client) throws IOException { - return client.mtime(NetCopyClientUtils.INSTANCE.extractRemotePathFrom(path)); + return client.mtime(NetCopyClientUtils.extractRemotePathFrom(path)); } }); @@ -350,13 +351,7 @@ public long length(Context context) { if (this instanceof HybridFileParcelable) { return ((HybridFileParcelable) this).getSize(); } else { - return NetCopyClientUtils.INSTANCE.execute( - new SFtpClientTemplate(path, false) { - @Override - public Long execute(@NonNull SFTPClient client) throws IOException { - return client.size(NetCopyClientUtils.INSTANCE.extractRemotePathFrom(path)); - } - }); + return sftpGetSize.invoke(getPath()); } case SMB: s = @@ -513,8 +508,7 @@ public FTPFile getFtpFile() { new FtpClientTemplate(path, false) { public FTPFile executeWithFtpClient(@NonNull FTPClient ftpClient) throws IOException { String path = - NetCopyClientUtils.INSTANCE.extractRemotePathFrom( - getParent(AppConfig.getInstance())); + NetCopyClientUtils.extractRemotePathFrom(getParent(AppConfig.getInstance())); ftpClient.changeWorkingDirectory(path); for (FTPFile ftpFile : ftpClient.listFiles()) { if (ftpFile.getName().equals(getName(AppConfig.getInstance()))) return ftpFile; @@ -622,13 +616,13 @@ public boolean isDirectory(Context context) { switch (mode) { case SFTP: final Boolean returnValue = - NetCopyClientUtils.INSTANCE.execute( - new SFtpClientTemplate(path, false) { + SshClientUtils.execute( + new SFtpClientTemplate(path, true) { @Override public Boolean execute(@NonNull SFTPClient client) { try { return client - .stat(NetCopyClientUtils.INSTANCE.extractRemotePathFrom(path)) + .stat(NetCopyClientUtils.extractRemotePathFrom(path)) .getType() .equals(FileMode.Type.DIRECTORY); } catch (IOException notFound) { @@ -726,20 +720,24 @@ public long folderSize(Context context) { switch (mode) { case SFTP: - final Long returnValue = - NetCopyClientUtils.INSTANCE.execute( - new SFtpClientTemplate(path, false) { - @Override - public Long execute(@NonNull SFTPClient client) throws IOException { - return client.size(NetCopyClientUtils.INSTANCE.extractRemotePathFrom(path)); - } - }); - - if (returnValue == null) { - LOG.error("Error obtaining size of folder over SFTP"); + Long retval = -1L; + String result = SshClientUtils.execute(getRemoteShellCommandLineResult("du -bs \"%s\"")); + if (!TextUtils.isEmpty(result) && result.indexOf('\t') > 0) { + try { + retval = Long.valueOf(result.substring(0, result.lastIndexOf('\t'))); + } catch (NumberFormatException ifParseFailed) { + LOG.warn("Unable to parse result (Seen {\"\"}), resort to old method", result); + retval = -1L; + } } - - return returnValue == null ? 0L : returnValue; + if (retval == -1L) { + Long returnValue = sftpGetSize.invoke(getPath()); + if (returnValue == null) { + LOG.error("Error obtaining size of folder over SFTP"); + } + return returnValue == null ? 0L : returnValue; + } + return retval; case SMB: SmbFile smbFile = getSmbFile(); size = (smbFile != null) ? FileUtils.folderSize(smbFile) : 0L; @@ -805,8 +803,8 @@ public long getUsableSpace() { break; case SFTP: final Long returnValue = - NetCopyClientUtils.INSTANCE.execute( - new SFtpClientTemplate(path, false) { + SshClientUtils.execute( + new SFtpClientTemplate(path, true) { @Override public Long execute(@NonNull SFTPClient client) throws IOException { try { @@ -817,8 +815,7 @@ public Long execute(@NonNull SFTPClient client) throws IOException { .getSFTPEngine() .request( Statvfs.request( - client, - NetCopyClientUtils.INSTANCE.extractRemotePathFrom(path))) + client, NetCopyClientUtils.extractRemotePathFrom(path))) .retrieve()); return response.diskFreeSpace(); } catch (SFTPException e) { @@ -891,8 +888,8 @@ public long getTotal(Context context) { break; case SFTP: final Long returnValue = - NetCopyClientUtils.INSTANCE.execute( - new SFtpClientTemplate(path, false) { + SshClientUtils.execute( + new SFtpClientTemplate(path, true) { @Override public Long execute(@NonNull SFTPClient client) throws IOException { try { @@ -903,8 +900,7 @@ public Long execute(@NonNull SFTPClient client) throws IOException { .getSFTPEngine() .request( Statvfs.request( - client, - NetCopyClientUtils.INSTANCE.extractRemotePathFrom(path))) + client, NetCopyClientUtils.extractRemotePathFrom(path))) .retrieve()); return response.diskSize(); } catch (SFTPException e) { @@ -941,40 +937,30 @@ public Long execute(@NonNull SFTPClient client) throws IOException { public void forEachChildrenFile(Context context, boolean isRoot, OnFileFound onFileFound) { switch (mode) { case SFTP: - NetCopyClientUtils.INSTANCE.execute( - new SFtpClientTemplate(getPath(), false) { + SshClientUtils.execute( + new SFtpClientTemplate(getPath(), true) { @Override public Boolean execute(@NonNull SFTPClient client) { try { - Flowable.fromIterable( - client.ls(NetCopyClientUtils.INSTANCE.extractRemotePathFrom(getPath()))) - .onBackpressureBuffer() - .subscribeOn(Schedulers.computation()) - .map( - info -> { - boolean isDirectory = false; - try { - isDirectory = SshClientUtils.isDirectory(client, info); - } catch (IOException ifBrokenSymlink) { - LOG.warn("IOException checking isDirectory(): " + info.getPath()); - return Flowable.empty(); - } - return new HybridFileParcelable(getPath(), isDirectory, info); - }) - .doOnNext( - v -> { - if (v instanceof HybridFileParcelable) { - onFileFound.onFileFound((HybridFileParcelable) v); - } - }) - .blockingSubscribe(); + for (RemoteResourceInfo info : + client.ls(NetCopyClientUtils.extractRemotePathFrom(getPath()))) { + boolean isDirectory = false; + try { + isDirectory = SshClientUtils.isDirectory(client, info); + } catch (IOException ifBrokenSymlink) { + LOG.warn("IOException checking isDirectory(): " + info.getPath()); + continue; + } + HybridFileParcelable f = new HybridFileParcelable(getPath(), isDirectory, info); + onFileFound.onFileFound(f); + } } catch (IOException e) { LOG.warn("IOException", e); AppConfig.toast( context, context.getString( R.string.cannot_read_directory, - parseAndFormatUriForDisplay(path), + parseAndFormatUriForDisplay(getPath()), e.getMessage())); } return true; @@ -1002,7 +988,7 @@ public Boolean execute(@NonNull SFTPClient client) { } break; case FTP: - String thisPath = NetCopyClientUtils.INSTANCE.extractRemotePathFrom(getPath()); + String thisPath = NetCopyClientUtils.extractRemotePathFrom(getPath()); FTPFile[] ftpFiles = NetCopyClientUtils.INSTANCE.execute( new FtpClientTemplate(getPath(), false) { @@ -1097,13 +1083,18 @@ public InputStream getInputStream(Context context) { @Override public InputStream execute(@NonNull final SFTPClient client) throws IOException { final RemoteFile rf = - client.open(NetCopyClientUtils.INSTANCE.extractRemotePathFrom(getPath())); - return rf.new RemoteFileInputStream() { + SFTPClientExtKt.openWithReadAheadSupport( + client, NetCopyClientUtils.extractRemotePathFrom(getPath())); + return rf.new ReadAheadRemoteFileInputStream(READ_AHEAD_MAX_UNCONFIRMED_READS) { @Override public void close() throws IOException { try { + LOG.debug("Closing input stream for {}", getPath()); super.close(); + } catch (Throwable e) { + e.printStackTrace(); } finally { + LOG.debug("Closing client for {}", getPath()); rf.close(); client.close(); } @@ -1138,7 +1129,7 @@ public InputStream executeWithFtpClient(@NonNull FTPClient ftpClient) File tmpFile = File.createTempFile("ftp-transfer_", ".tmp"); tmpFile.deleteOnExit(); ftpClient.changeWorkingDirectory( - NetCopyClientUtils.INSTANCE.extractRemotePathFrom(parent)); + NetCopyClientUtils.extractRemotePathFrom(parent)); ftpClient.setFileType(FTP.BINARY_FILE_TYPE); InputStream fin = ftpClient.retrieveFileStream(getName(AppConfig.getInstance())); @@ -1196,14 +1187,14 @@ public OutputStream getOutputStream(Context context) { OutputStream outputStream; switch (mode) { case SFTP: - return NetCopyClientUtils.INSTANCE.execute( + return SshClientUtils.execute( new SFtpClientTemplate(getPath(), false) { @Nullable @Override public OutputStream execute(@NonNull SFTPClient client) throws IOException { final RemoteFile rf = client.open( - NetCopyClientUtils.INSTANCE.extractRemotePathFrom(getPath()), + NetCopyClientUtils.extractRemotePathFrom(getPath()), EnumSet.of( net.schmizz.sshj.sftp.OpenMode.WRITE, net.schmizz.sshj.sftp.OpenMode.CREAT)); @@ -1231,7 +1222,7 @@ public void close() throws IOException { public OutputStream executeWithFtpClient(@NonNull FTPClient ftpClient) throws IOException { ftpClient.setFileType(FTP.BINARY_FILE_TYPE); - String remotePath = NetCopyClientUtils.INSTANCE.extractRemotePathFrom(path); + String remotePath = NetCopyClientUtils.extractRemotePathFrom(path); OutputStream outputStream = ftpClient.storeFileStream(remotePath); if (outputStream != null) { return FTPClientImpl.wrap(outputStream, ftpClient); @@ -1289,8 +1280,7 @@ public boolean exists() { @Override public Boolean execute(SFTPClient client) throws IOException { try { - return client.stat(NetCopyClientUtils.INSTANCE.extractRemotePathFrom(path)) - != null; + return client.stat(NetCopyClientUtils.extractRemotePathFrom(path)) != null; } catch (SFTPException notFound) { return false; } @@ -1393,13 +1383,28 @@ public boolean setLastModified(final long date) { new FtpClientTemplate(path, false) { public Boolean executeWithFtpClient(@NonNull FTPClient ftpClient) throws IOException { - Calendar calendar = Calendar.getInstance(); - calendar.setTimeInMillis(date); - DateFormat df = new SimpleDateFormat("yyyyMMddHHmmss", Locale.US); - df.setCalendar(calendar); return ftpClient.setModificationTime( - NetCopyClientUtils.INSTANCE.extractRemotePathFrom(path), - df.format(calendar.getTime())); + NetCopyClientUtils.extractRemotePathFrom(path), + NetCopyClientUtils.getTimestampForTouch(date)); + } + })); + } else if (isSftp()) { + return Boolean.TRUE.equals( + SshClientUtils.execute( + new SshClientSessionTemplate(getPath()) { + @Override + public Boolean execute(@NonNull Session session) throws IOException { + Session.Command cmd = + session.exec( + String.format( + Locale.US, + "touch -m -t %s \"%s\"", + NetCopyClientUtils.getTimestampForTouch(date), + getPath())); + // Quirk: need to wait the command to finish + IOUtils.readFully(cmd.getInputStream()); + cmd.close(); + return 0 == cmd.getExitStatus(); } })); } else { @@ -1410,12 +1415,12 @@ public Boolean executeWithFtpClient(@NonNull FTPClient ftpClient) public void mkdir(Context context) { if (isSftp()) { - NetCopyClientUtils.INSTANCE.execute( + SshClientUtils.execute( new SFtpClientTemplate(path, true) { @Override public Boolean execute(@NonNull SFTPClient client) { try { - client.mkdir(NetCopyClientUtils.INSTANCE.extractRemotePathFrom(path)); + client.mkdir(NetCopyClientUtils.extractRemotePathFrom(path)); } catch (IOException e) { LOG.error("Error making directory over SFTP", e); } @@ -1427,7 +1432,7 @@ public Boolean execute(@NonNull SFTPClient client) { new FtpClientTemplate(getPath(), false) { public Boolean executeWithFtpClient(@NonNull FTPClient ftpClient) throws IOException { ExtensionsKt.makeDirectoryTree( - ftpClient, NetCopyClientUtils.INSTANCE.extractRemotePathFrom(getPath())); + ftpClient, NetCopyClientUtils.extractRemotePathFrom(getPath())); return true; } }); @@ -1471,11 +1476,11 @@ public boolean delete(Context context, boolean rootmode) throws ShellNotRunningException, SmbException { if (isSftp()) { Boolean retval = - NetCopyClientUtils.INSTANCE.execute( - new SFtpClientTemplate(path, false) { + SshClientUtils.execute( + new SFtpClientTemplate(path, true) { @Override public Boolean execute(@NonNull SFTPClient client) throws IOException { - String _path = NetCopyClientUtils.INSTANCE.extractRemotePathFrom(path); + String _path = NetCopyClientUtils.extractRemotePathFrom(path); if (isDirectory(AppConfig.getInstance())) client.rmdir(_path); else client.rm(_path); return client.statExistence(_path) == null; @@ -1489,8 +1494,7 @@ public Boolean execute(@NonNull SFTPClient client) throws IOException { @Override public Boolean executeWithFtpClient(@NonNull FTPClient ftpClient) throws IOException { - return ftpClient.deleteFile( - NetCopyClientUtils.INSTANCE.extractRemotePathFrom(path)); + return ftpClient.deleteFile(NetCopyClientUtils.extractRemotePathFrom(path)); } }); return retval != null && retval; @@ -1643,7 +1647,7 @@ public void getMd5Checksum(Context context, Function callback) { switch (mode) { case SFTP: String md5Command = "md5sum -b \"%s\" | cut -c -32"; - return SshClientUtils.execute(getSftpHash(md5Command)); + return SshClientUtils.execute(getRemoteShellCommandLineResult(md5Command)); default: byte[] b = createChecksum(context); String result = ""; @@ -1685,7 +1689,7 @@ public void getSha256Checksum(Context context, Function callback) switch (mode) { case SFTP: String shaCommand = "sha256sum -b \"%s\" | cut -c -64"; - return SshClientUtils.execute(getSftpHash(shaCommand)); + return SshClientUtils.execute(getRemoteShellCommandLineResult(shaCommand)); default: MessageDigest messageDigest = MessageDigest.getInstance("SHA-256"); byte[] input = new byte[GenericCopyUtil.DEFAULT_BUFFER_SIZE]; @@ -1733,11 +1737,11 @@ public void onError(Throwable e) { }); } - private SshClientSessionTemplate getSftpHash(String command) { + private SshClientSessionTemplate getRemoteShellCommandLineResult(String command) { return new SshClientSessionTemplate(path) { @Override public String execute(Session session) throws IOException { - String extractedPath = NetCopyClientUtils.INSTANCE.extractRemotePathFrom(path); + String extractedPath = NetCopyClientUtils.extractRemotePathFrom(getPath()); String fullCommand = String.format(command, extractedPath); Session.Command cmd = session.exec(fullCommand); String result = new String(IOUtils.readFully(cmd.getInputStream()).toByteArray()); diff --git a/app/src/main/java/com/amaze/filemanager/filesystem/HybridFileParcelable.java b/app/src/main/java/com/amaze/filemanager/filesystem/HybridFileParcelable.java index 50928e1e6b..20edfc696e 100644 --- a/app/src/main/java/com/amaze/filemanager/filesystem/HybridFileParcelable.java +++ b/app/src/main/java/com/amaze/filemanager/filesystem/HybridFileParcelable.java @@ -99,6 +99,11 @@ public HybridFileParcelable(String path, boolean isDirectory, RemoteResourceInfo Integer.toString(FilePermission.toMask(sshFile.getAttributes().getPermissions()), 8)); } + @Override + public long lastModified() { + return date; + } + public String getName() { if (!Utils.isNullOrEmpty(name)) return name; else return super.getSimpleName(); diff --git a/app/src/main/java/com/amaze/filemanager/filesystem/Operations.java b/app/src/main/java/com/amaze/filemanager/filesystem/Operations.java index 4125f726eb..5838f3d593 100644 --- a/app/src/main/java/com/amaze/filemanager/filesystem/Operations.java +++ b/app/src/main/java/com/amaze/filemanager/filesystem/Operations.java @@ -47,6 +47,7 @@ import com.amaze.filemanager.filesystem.root.MakeFileCommand; import com.amaze.filemanager.filesystem.root.RenameFileCommand; import com.amaze.filemanager.filesystem.ssh.SFtpClientTemplate; +import com.amaze.filemanager.filesystem.ssh.SshClientUtils; import com.amaze.filemanager.utils.DataUtils; import com.amaze.filemanager.utils.OTGUtil; import com.cloudrail.si.interfaces.CloudStorage; @@ -67,7 +68,7 @@ public class Operations { - private static Executor executor = AsyncTask.THREAD_POOL_EXECUTOR; + private static final Executor executor = AsyncTask.THREAD_POOL_EXECUTOR; private static final Logger LOG = LoggerFactory.getLogger(Operations.class); @@ -579,14 +580,14 @@ protected Void doInBackground(Void... params) { } return null; } else if (oldFile.isSftp()) { - NetCopyClientUtils.INSTANCE.execute( - new SFtpClientTemplate(oldFile.getPath(), false) { + SshClientUtils.execute( + new SFtpClientTemplate(oldFile.getPath(), true) { @Override public Void execute(@NonNull SFTPClient client) { try { client.rename( - NetCopyClientUtils.INSTANCE.extractRemotePathFrom(oldFile.getPath()), - NetCopyClientUtils.INSTANCE.extractRemotePathFrom(newFile.getPath())); + NetCopyClientUtils.extractRemotePathFrom(oldFile.getPath()), + NetCopyClientUtils.extractRemotePathFrom(newFile.getPath())); errorCallBack.done(newFile, true); } catch (IOException e) { String errmsg = @@ -620,8 +621,8 @@ public Boolean executeWithFtpClient(@NonNull FTPClient ftpClient) throws IOException { boolean result = ftpClient.rename( - NetCopyClientUtils.INSTANCE.extractRemotePathFrom(oldFile.getPath()), - NetCopyClientUtils.INSTANCE.extractRemotePathFrom(newFile.getPath())); + NetCopyClientUtils.extractRemotePathFrom(oldFile.getPath()), + NetCopyClientUtils.extractRemotePathFrom(newFile.getPath())); errorCallBack.done(newFile, result); return result; } diff --git a/app/src/main/java/com/amaze/filemanager/filesystem/files/GenericCopyUtil.java b/app/src/main/java/com/amaze/filemanager/filesystem/files/GenericCopyUtil.java index 068d6a95fe..82080fd658 100644 --- a/app/src/main/java/com/amaze/filemanager/filesystem/files/GenericCopyUtil.java +++ b/app/src/main/java/com/amaze/filemanager/filesystem/files/GenericCopyUtil.java @@ -243,10 +243,10 @@ private void startCopy( doCopy(inChannel, outChannel, updatePosition); } catch (IOException e) { - LOG.debug("I/O Error!", e); - throw new IOException(); + LOG.error("I/O Error copy {} to {}: {}", mSourceFile, mTargetFile, e); + throw new IOException(e); } catch (OutOfMemoryError e) { - LOG.warn("low memory while copying file", e); + LOG.warn("low memory while copying {} to {}: {}", mSourceFile, mTargetFile, e); onLowMemory.onLowMemory(); diff --git a/app/src/main/java/com/amaze/filemanager/filesystem/ftp/NetCopyClientConnectionPool.kt b/app/src/main/java/com/amaze/filemanager/filesystem/ftp/NetCopyClientConnectionPool.kt index e5fb262932..4cee083dff 100644 --- a/app/src/main/java/com/amaze/filemanager/filesystem/ftp/NetCopyClientConnectionPool.kt +++ b/app/src/main/java/com/amaze/filemanager/filesystem/ftp/NetCopyClientConnectionPool.kt @@ -70,7 +70,6 @@ object NetCopyClientConnectionPool { /** * Obtain a [NetCopyClient] connection from the underlying connection pool. * - * * Beneath it will return the connection if it exists; otherwise it will create a new one and * put it into the connection pool. * diff --git a/app/src/main/java/com/amaze/filemanager/filesystem/ftp/NetCopyClientUtils.kt b/app/src/main/java/com/amaze/filemanager/filesystem/ftp/NetCopyClientUtils.kt index 3fedf1b0eb..61f3c85f5c 100644 --- a/app/src/main/java/com/amaze/filemanager/filesystem/ftp/NetCopyClientUtils.kt +++ b/app/src/main/java/com/amaze/filemanager/filesystem/ftp/NetCopyClientUtils.kt @@ -48,6 +48,10 @@ import org.apache.commons.net.ftp.FTPClient import org.apache.commons.net.ftp.FTPReply import org.slf4j.LoggerFactory import java.io.IOException +import java.text.DateFormat +import java.text.SimpleDateFormat +import java.util.Calendar +import java.util.Locale object NetCopyClientUtils { @@ -74,7 +78,9 @@ object NetCopyClientUtils { * Execute the given NetCopyClientTemplate. * * This template pattern is borrowed from Spring Framework, to simplify code on operations - * using SftpClientTemplate. + * using NetCopyClientTemplate. + * + * FIXME: Over-simplification implementation causing unnecessarily closing SSHClient. * * @param template [NetCopyClientTemplate] to execute * @param Type of return value @@ -187,6 +193,7 @@ object NetCopyClientUtils { * @param fullUri Full SSH URL * @return The remote path part of the full SSH URL */ + @JvmStatic fun extractRemotePathFrom(fullUri: String): String { return NetCopyConnectionInfo(fullUri).let { connInfo -> if (true == connInfo.defaultPath?.isNotEmpty()) { @@ -291,4 +298,16 @@ object NetCopyClientUtils { SMB_URI_PREFIX -> 0 // SMB never requires explicit port number at URL else -> throw IllegalArgumentException("Cannot derive default port") } + + /** + * Convenience method to format given UNIX timestamp to yyyyMMddHHmmss format. + */ + @JvmStatic + fun getTimestampForTouch(date: Long): String { + val calendar = Calendar.getInstance() + calendar.timeInMillis = date + val df: DateFormat = SimpleDateFormat("yyyyMMddHHmmss", Locale.US) + df.calendar = calendar + return df.format(calendar.time) + } } diff --git a/app/src/main/java/com/amaze/filemanager/filesystem/ssh/SFTPClientExt.kt b/app/src/main/java/com/amaze/filemanager/filesystem/ssh/SFTPClientExt.kt new file mode 100644 index 0000000000..4f36e35996 --- /dev/null +++ b/app/src/main/java/com/amaze/filemanager/filesystem/ssh/SFTPClientExt.kt @@ -0,0 +1,61 @@ +/* + * Copyright (C) 2014-2023 Arpit Khurana , Vishal Nehra , + * Emmanuel Messulam, Raymond Lai and Contributors. + * + * This file is part of Amaze File Manager. + * + * Amaze File Manager is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package com.amaze.filemanager.filesystem.ssh + +import net.schmizz.sshj.sftp.FileAttributes +import net.schmizz.sshj.sftp.OpenMode +import net.schmizz.sshj.sftp.PacketType +import net.schmizz.sshj.sftp.RemoteFile +import net.schmizz.sshj.sftp.SFTPClient +import net.schmizz.sshj.sftp.SFTPEngine +import java.io.IOException +import java.util.EnumSet +import java.util.concurrent.TimeUnit + +const val READ_AHEAD_MAX_UNCONFIRMED_READS: Int = 16 + +/** + * Monkey-patch [SFTPEngine.open] until sshj adds back read ahead support in [RemoteFile]. + */ +@Throws(IOException::class) +fun SFTPEngine.openWithReadAheadSupport( + path: String, + modes: Set, + fa: FileAttributes +): RemoteFile { + val handle: ByteArray = request( + newRequest(PacketType.OPEN).putString(path, subsystem.remoteCharset) + .putUInt32(OpenMode.toMask(modes).toLong()).putFileAttributes(fa) + ).retrieve(timeoutMs.toLong(), TimeUnit.MILLISECONDS) + .ensurePacketTypeIs(PacketType.HANDLE).readBytes() + return RemoteFile(this, path, handle) +} + +/** + * Monkey-patch [SFTPClient.open] until sshj adds back read ahead support in [RemoteFile]. + */ +fun SFTPClient.openWithReadAheadSupport(path: String): RemoteFile { + return sftpEngine.openWithReadAheadSupport( + path, + EnumSet.of(OpenMode.READ), + FileAttributes.EMPTY + ) +} diff --git a/app/src/main/java/com/amaze/filemanager/filesystem/ssh/SshClientUtils.java b/app/src/main/java/com/amaze/filemanager/filesystem/ssh/SshClientUtils.java deleted file mode 100644 index 8af4363fa0..0000000000 --- a/app/src/main/java/com/amaze/filemanager/filesystem/ssh/SshClientUtils.java +++ /dev/null @@ -1,227 +0,0 @@ -/* - * Copyright (C) 2014-2020 Arpit Khurana , Vishal Nehra , - * Emmanuel Messulam, Raymond Lai and Contributors. - * - * This file is part of Amaze File Manager. - * - * Amaze File Manager is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -package com.amaze.filemanager.filesystem.ssh; - -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.amaze.filemanager.R; -import com.amaze.filemanager.fileoperations.filesystem.cloud.CloudStreamer; -import com.amaze.filemanager.filesystem.HybridFile; -import com.amaze.filemanager.filesystem.ftp.NetCopyClientUtils; -import com.amaze.filemanager.ui.activities.MainActivity; -import com.amaze.filemanager.ui.icons.MimeTypes; - -import android.content.ActivityNotFoundException; -import android.content.Intent; -import android.content.pm.PackageManager; -import android.content.pm.ResolveInfo; -import android.net.Uri; -import android.widget.Toast; - -import androidx.annotation.NonNull; -import androidx.annotation.Nullable; - -import net.schmizz.sshj.SSHClient; -import net.schmizz.sshj.connection.channel.direct.Session; -import net.schmizz.sshj.sftp.FileAttributes; -import net.schmizz.sshj.sftp.FileMode; -import net.schmizz.sshj.sftp.RemoteResourceInfo; -import net.schmizz.sshj.sftp.SFTPClient; - -public abstract class SshClientUtils { - - private static final Logger LOG = LoggerFactory.getLogger(SshClientUtils.class); - - /** - * Execute the given template with SshClientTemplate. - * - * @param template {@link SshClientSessionTemplate} to execute - * @param Type of return value - * @return Template execution results - */ - public static T execute(@NonNull final SshClientSessionTemplate template) { - return NetCopyClientUtils.INSTANCE.execute( - new SshClientTemplate(template.url, false) { - @Override - public T executeWithSSHClient(@NonNull SSHClient sshClient) { - Session session = null; - T retval = null; - try { - session = sshClient.startSession(); - retval = template.execute(session); - } catch (IOException e) { - LOG.error("Error executing template method", e); - } finally { - if (session != null && session.isOpen()) { - try { - session.close(); - } catch (IOException e) { - LOG.warn("Error closing SFTP client", e); - } - } - } - return retval; - } - }); - } - - /** - * Execute the given template with SshClientTemplate. - * - * @param template {@link SFtpClientTemplate} to execute - * @param Type of return value - * @return Template execution results - */ - @Nullable - public static T execute(@NonNull final SFtpClientTemplate template) { - final SshClientTemplate ftpClientTemplate = - new SshClientTemplate(template.url, false) { - @Override - @Nullable - public T executeWithSSHClient(SSHClient sshClient) { - SFTPClient sftpClient = null; - T retval = null; - try { - sftpClient = sshClient.newSFTPClient(); - retval = template.execute(sftpClient); - } catch (IOException e) { - LOG.error("Error executing template method", e); - } finally { - if (sftpClient != null && template.closeClientOnFinish) { - try { - sftpClient.close(); - } catch (IOException e) { - LOG.warn("Error closing SFTP client", e); - } - } - } - return retval; - } - }; - - return NetCopyClientUtils.INSTANCE.execute(ftpClientTemplate); - } - - /** - * Converts plain path smb://127.0.0.1/test.pdf to authorized path - * smb://test:123@127.0.0.1/test.pdf from server list - * - * @param path - * @return - */ - public static String formatPlainServerPathToAuthorised(ArrayList servers, String path) { - for (String[] serverEntry : servers) { - Uri inputUri = Uri.parse(path); - Uri serverUri = Uri.parse(serverEntry[1]); - if (inputUri.getScheme().equalsIgnoreCase(serverUri.getScheme()) - && serverUri.getAuthority().contains(inputUri.getAuthority())) { - String output = - inputUri - .buildUpon() - .encodedAuthority(serverUri.getEncodedAuthority()) - .build() - .toString(); - LOG.info("build authorised path {} from plain path {}", output, path); - return output; - } - } - return path; - } - - /** - * Disconnects the given {@link SSHClient} but wrap all exceptions beneath, so callers are free - * from the hassles of handling thrown exceptions. - * - * @param client {@link SSHClient} instance - */ - public static void tryDisconnect(SSHClient client) { - if (client != null && client.isConnected()) { - try { - client.disconnect(); - } catch (IOException e) { - LOG.warn("Error closing SSHClient connection", e); - } - } - } - - public static void launchFtp(final HybridFile baseFile, final MainActivity activity) { - final CloudStreamer streamer = CloudStreamer.getInstance(); - - new Thread( - () -> { - try { - boolean isDirectory = baseFile.isDirectory(activity); - long fileLength = baseFile.length(activity); - streamer.setStreamSrc( - baseFile.getInputStream(activity), baseFile.getName(activity), fileLength); - activity.runOnUiThread( - () -> { - try { - File file = - new File( - NetCopyClientUtils.INSTANCE.extractRemotePathFrom( - baseFile.getPath())); - Uri uri = - Uri.parse(CloudStreamer.URL + Uri.fromFile(file).getEncodedPath()); - Intent i = new Intent(Intent.ACTION_VIEW); - i.setDataAndType( - uri, MimeTypes.getMimeType(baseFile.getPath(), isDirectory)); - PackageManager packageManager = activity.getPackageManager(); - List resInfos = packageManager.queryIntentActivities(i, 0); - if (resInfos != null && resInfos.size() > 0) activity.startActivity(i); - else - Toast.makeText( - activity, - activity.getResources().getString(R.string.smb_launch_error), - Toast.LENGTH_SHORT) - .show(); - } catch (ActivityNotFoundException e) { - LOG.warn("failed to launch sftp file", e); - } - }); - } catch (Exception e) { - LOG.warn("failed to launch sftp file", e); - } - }) - .start(); - } - - public static boolean isDirectory(@NonNull SFTPClient client, @NonNull RemoteResourceInfo info) - throws IOException { - boolean isDirectory = info.isDirectory(); - if (info.getAttributes().getType().equals(FileMode.Type.SYMLINK)) { - try { - FileAttributes symlinkAttrs = client.stat(info.getPath()); - isDirectory = symlinkAttrs.getType().equals(FileMode.Type.DIRECTORY); - } catch (IOException ifSymlinkIsBroken) { - LOG.warn("Symbolic link {} is broken, skipping", info.getPath()); - throw ifSymlinkIsBroken; - } - } - return isDirectory; - } -} diff --git a/app/src/main/java/com/amaze/filemanager/filesystem/ssh/SshClientUtils.kt b/app/src/main/java/com/amaze/filemanager/filesystem/ssh/SshClientUtils.kt new file mode 100644 index 0000000000..12c8c40094 --- /dev/null +++ b/app/src/main/java/com/amaze/filemanager/filesystem/ssh/SshClientUtils.kt @@ -0,0 +1,222 @@ +/* + * Copyright (C) 2014-2020 Arpit Khurana , Vishal Nehra , + * Emmanuel Messulam, Raymond Lai and Contributors. + * + * This file is part of Amaze File Manager. + * + * Amaze File Manager is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package com.amaze.filemanager.filesystem.ssh + +import android.content.ActivityNotFoundException +import android.content.Intent +import android.net.Uri +import android.widget.Toast +import com.amaze.filemanager.R +import com.amaze.filemanager.fileoperations.filesystem.cloud.CloudStreamer +import com.amaze.filemanager.filesystem.HybridFile +import com.amaze.filemanager.filesystem.ftp.NetCopyClientUtils +import com.amaze.filemanager.filesystem.ftp.NetCopyClientUtils.extractRemotePathFrom +import com.amaze.filemanager.ui.activities.MainActivity +import com.amaze.filemanager.ui.icons.MimeTypes +import net.schmizz.sshj.SSHClient +import net.schmizz.sshj.connection.channel.direct.Session +import net.schmizz.sshj.sftp.FileMode +import net.schmizz.sshj.sftp.RemoteResourceInfo +import net.schmizz.sshj.sftp.SFTPClient +import org.slf4j.LoggerFactory +import java.io.File +import java.io.IOException +import kotlin.concurrent.thread + +object SshClientUtils { + + @JvmStatic + private val LOG = LoggerFactory.getLogger(SshClientUtils::class.java) + + @JvmField + val sftpGetSize: (String) -> Long? = { path -> + NetCopyClientUtils.execute(object : SFtpClientTemplate(path, true) { + override fun execute(client: SFTPClient): Long { + return client.size(extractRemotePathFrom(path)) + } + }) + } + + /** + * Execute the given template with SshClientTemplate. + * + * @param template [SshClientSessionTemplate] to execute + * @param Type of return value + * @return Template execution results + */ + @JvmStatic + fun execute(template: SshClientSessionTemplate): T? { + return NetCopyClientUtils.execute( + object : SshClientTemplate(template.url, false) { + override fun executeWithSSHClient(sshClient: SSHClient): T? { + var session: Session? = null + var retval: T? = null + try { + session = sshClient.startSession() + retval = template.execute(session) + } catch (e: IOException) { + LOG.error("Error executing template method", e) + } finally { + if (session != null && session.isOpen) { + try { + session.close() + } catch (e: IOException) { + LOG.warn("Error closing SFTP client", e) + } + } + } + return retval + } + } + ) + } + + /** + * Execute the given template with SshClientTemplate. + * + * @param template [SFtpClientTemplate] to execute + * @param Type of return value + * @return Template execution results + */ + @JvmStatic + fun execute(template: SFtpClientTemplate): T? { + return NetCopyClientUtils.execute(template) + } + + /** + * Converts plain path smb://127.0.0.1/test.pdf to authorized path + * smb://test:123@127.0.0.1/test.pdf from server list + * + * @param path + * @return + */ + @JvmStatic + fun formatPlainServerPathToAuthorised( + servers: ArrayList>, + path: String + ): String { + for (serverEntry in servers) { + val inputUri = Uri.parse(path) + val serverUri = Uri.parse(serverEntry[1]) + if (inputUri.scheme.equals(serverUri.scheme, ignoreCase = true) && + serverUri.authority!!.contains(inputUri.authority!!) + ) { + val output = inputUri + .buildUpon() + .encodedAuthority(serverUri.encodedAuthority) + .build() + .toString() + LOG.info("build authorised path {} from plain path {}", output, path) + return output + } + } + return path + } + + /** + * Disconnects the given [SSHClient] but wrap all exceptions beneath, so callers are free + * from the hassles of handling thrown exceptions. + * + * @param client [SSHClient] instance + */ + fun tryDisconnect(client: SSHClient?) { + if (client != null && client.isConnected) { + try { + client.disconnect() + } catch (e: IOException) { + LOG.warn("Error closing SSHClient connection", e) + } + } + } + + /** + * Open a remote SSH file on local Android device. It uses the [CloudStreamer] to stream the + * file. + */ + @JvmStatic + @Suppress("Detekt.TooGenericExceptionCaught") + fun launchFtp(baseFile: HybridFile, activity: MainActivity) { + val streamer = CloudStreamer.getInstance() + thread { + try { + val isDirectory = baseFile.isDirectory(activity) + val fileLength = baseFile.length(activity) + streamer.setStreamSrc( + baseFile.getInputStream(activity), + baseFile.getName(activity), + fileLength + ) + activity.runOnUiThread { + try { + val file = File( + extractRemotePathFrom( + baseFile.path + ) + ) + val uri = Uri.parse(CloudStreamer.URL + Uri.fromFile(file).encodedPath) + val i = Intent(Intent.ACTION_VIEW) + i.setDataAndType( + uri, + MimeTypes.getMimeType(baseFile.path, isDirectory) + ) + val packageManager = activity.packageManager + val resInfos = packageManager.queryIntentActivities(i, 0) + if (resInfos != null && resInfos.size > 0) { + activity.startActivity(i) + } else { + Toast.makeText( + activity, + activity.resources.getString(R.string.smb_launch_error), + Toast.LENGTH_SHORT + ) + .show() + } + } catch (e: ActivityNotFoundException) { + LOG.warn("failed to launch sftp file", e) + } + } + } catch (e: Exception) { + LOG.warn("failed to launch sftp file", e) + } + } + } + + /** + * Reads given [RemoteResourceInfo] and determines if the path it's related to is a directory. + * + * Will descend into corresponding target if given RemoteResourceInfo represents a symlink. + */ + @JvmStatic + @Throws(IOException::class) + fun isDirectory(client: SFTPClient, info: RemoteResourceInfo): Boolean { + var isDirectory = info.isDirectory + if (info.attributes.type == FileMode.Type.SYMLINK) { + try { + val symlinkAttrs = client.stat(info.path) + isDirectory = symlinkAttrs.type == FileMode.Type.DIRECTORY + } catch (ifSymlinkIsBroken: IOException) { + LOG.warn("Symbolic link {} is broken, skipping", info.path) + throw ifSymlinkIsBroken + } + } + return isDirectory + } +} diff --git a/app/src/main/java/com/amaze/filemanager/ui/activities/MainActivity.java b/app/src/main/java/com/amaze/filemanager/ui/activities/MainActivity.java index d268f41ad1..9a980d3b6f 100644 --- a/app/src/main/java/com/amaze/filemanager/ui/activities/MainActivity.java +++ b/app/src/main/java/com/amaze/filemanager/ui/activities/MainActivity.java @@ -1988,6 +1988,7 @@ public void showSftpDialog(String name, String path, boolean edit) { if (i != -1) name = dataUtils.getServers().get(i)[0]; } SftpConnectDialog sftpConnectDialog = new SftpConnectDialog(); + sftpConnectDialog.setCancelable(false); String finalName = name; Flowable.fromCallable(() -> new NetCopyConnectionInfo(path)) .flatMap( diff --git a/app/src/main/java/com/amaze/filemanager/utils/GenericExt.kt b/app/src/main/java/com/amaze/filemanager/utils/GenericExt.kt index 81ab58b4a4..c54a5d9f73 100644 --- a/app/src/main/java/com/amaze/filemanager/utils/GenericExt.kt +++ b/app/src/main/java/com/amaze/filemanager/utils/GenericExt.kt @@ -114,7 +114,3 @@ fun String.urlEncoded(charset: Charset = Charsets.UTF_8): String { fun String.urlDecoded(charset: Charset = Charsets.UTF_8): String { return decode(this, charset.name()) } - -interface Function { - fun apply(t: T): R -} diff --git a/build.gradle b/build.gradle index 3d018b2808..842e651fb4 100644 --- a/build.gradle +++ b/build.gradle @@ -5,7 +5,7 @@ buildscript { kotlin_version = "1.6.10" robolectricVersion = '4.9' glideVersion = '4.11.0' - sshjVersion = '0.34.0' + sshjVersion = '0.35.0' jcifsVersion = '2.1.6' fabSpeedDialVersion = '3.1.1' roomVersion = '2.4.3'