GH-3572: Migrate SFTP from jsch to sshd-sftp (#3892)

* GH-3572: Migrate SFTP from `jsch` to `sshd-sftp`

Fixes #3572

* Rework SFTP module from the JSch API to more modern `sshd-sftp`
* Migrate generics of most API from `ChannelSftp.LsEntry` to the `SftpClient.DirEntry`
* Rework `DefaultSftpSessionFactory` to deal with an `SshClient` and create `SftpClient`
wrapped to the `SftpSession`
* Implement a `ResourceKnownHostsServerKeyVerifier` to load `known-hosts` from any possible resource
* Implement an expected `SftpSession.list()` with just file name to take or pattern matching
* Remove some unused tests and their config
* Remove tests for custom `UserInfo` since we don't provide any custom out-of-the-box
* Test a new `ResourceKnownHostsServerKeyVerifier` against default `known-hosts` file

* * Some tests improvements

* * Improve generics handling for `FileUtils`

Author: artembilan

build.gradle

@@ -76,7 +76,6 @@ ext {
     jmsApiVersion = '3.0.0'
     jpaApiVersion = '3.0.3'
     jrubyVersion = '9.3.8.0'
-    jschVersion = '0.1.55'
     jsonpathVersion = '2.7.0'
     junit4Version = '4.13.2'
     junitJupiterVersion = '5.9.0'
@@ -900,13 +899,11 @@ project('spring-integration-sftp') {
     description = 'Spring Integration SFTP Support'
     dependencies {
         api project(':spring-integration-file')
-        api "com.jcraft:jsch:$jschVersion"
         api 'org.springframework:spring-context-support'
-        optionalApi ("org.apache.sshd:sshd-sftp:$apacheSshdVersion") {
+        api ("org.apache.sshd:sshd-sftp:$apacheSshdVersion") {
             exclude group: 'org.slf4j', module: 'jcl-over-slf4j'
         }
 
-        testImplementation "org.apache.sshd:sshd-core:$apacheSshdVersion"
         testImplementation project(':spring-integration-event')
         testImplementation project(':spring-integration-file').sourceSets.test.output
     }

spring-integration-file/src/main/java/org/springframework/integration/file/dsl/RemoteFileInboundChannelAdapterSpec.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2016-2020 the original author or authors.
+ * Copyright 2016-2022 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -290,7 +290,7 @@ public S scanner(DirectoryScanner scanner) {
 	 * @return the spec.
 	 * @since 5.2.9
 	 */
-	public S remoteComparator(Comparator<F> remoteComparator) {
+	public S remoteComparator(Comparator<? extends F> remoteComparator) {
 		this.synchronizer.setComparator(remoteComparator);
 		return _this();
 	}

spring-integration-file/src/main/java/org/springframework/integration/file/remote/AbstractRemoteFileStreamingMessageSource.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2016-2021 the original author or authors.
+ * Copyright 2016-2022 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -62,11 +62,11 @@
 public abstract class AbstractRemoteFileStreamingMessageSource<F>
 		extends AbstractFetchLimitingMessageSource<InputStream> implements ManageableLifecycle {
 
-	private final RemoteFileTemplate<F> remoteFileTemplate;
+	private final RemoteFileTemplate<? extends F> remoteFileTemplate;
 
 	private final BlockingQueue<AbstractFileInfo<F>> toBeReceived = new LinkedBlockingQueue<>();
 
-	private final Comparator<F> comparator;
+	private final Comparator<? extends F> comparator;
 
 	private final AtomicBoolean running = new AtomicBoolean();
 
@@ -86,8 +86,8 @@ public abstract class AbstractRemoteFileStreamingMessageSource<F>
 	 */
 	private FileListFilter<F> filter;
 
-	protected AbstractRemoteFileStreamingMessageSource(RemoteFileTemplate<F> template,
-			@Nullable Comparator<F> comparator) {
+	protected AbstractRemoteFileStreamingMessageSource(RemoteFileTemplate<? extends F> template,
+			@Nullable Comparator<? extends F> comparator) {
 
 		Assert.notNull(template, "'template' must not be null");
 		this.remoteFileTemplate = template;
@@ -143,7 +143,7 @@ public void setFileInfoJson(boolean fileInfoJson) {
 		this.fileInfoJson = fileInfoJson;
 	}
 
-	protected RemoteFileTemplate<F> getRemoteFileTemplate() {
+	protected RemoteFileTemplate<? extends F> getRemoteFileTemplate() {
 		return this.remoteFileTemplate;
 	}
 

spring-integration-file/src/main/java/org/springframework/integration/file/remote/synchronizer/AbstractInboundFileSynchronizer.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2002-2021 the original author or authors.
+ * Copyright 2002-2022 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -123,7 +123,7 @@ public abstract class AbstractInboundFileSynchronizer<F>
 	private BeanFactory beanFactory;
 
 	@Nullable
-	private Comparator<F> comparator;
+	private Comparator<? extends F> comparator;
 
 	private MetadataStore remoteFileMetadataStore = new SimpleMetadataStore();
 
@@ -141,7 +141,7 @@ public AbstractInboundFileSynchronizer(SessionFactory<F> sessionFactory) {
 	}
 
 	@Nullable
-	protected Comparator<F> getComparator() {
+	protected Comparator<? extends F> getComparator() {
 		return this.comparator;
 	}
 
@@ -151,7 +151,7 @@ protected Comparator<F> getComparator() {
 	 * @param comparator the comparator.
 	 * @since 5.1
 	 */
-	public void setComparator(@Nullable Comparator<F> comparator) {
+	public void setComparator(@Nullable Comparator<? extends F> comparator) {
 		this.comparator = comparator;
 	}
 

spring-integration-file/src/main/java/org/springframework/integration/file/support/FileUtils.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2017-2019 the original author or authors.
+ * Copyright 2017-2022 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -29,6 +29,7 @@
  * Utilities for operations on Files.
  *
  * @author Gary Russell
+ * @author Artem Bilan
  *
  * @since 5.0
  *
@@ -47,22 +48,22 @@ public final class FileUtils {
 	 * @since 5.0.7
 	 */
 	@SuppressWarnings("unchecked")
-	public static <F> F[] purgeUnwantedElements(F[] fileArray, Predicate<F> predicate,
-			@Nullable Comparator<F> comparator) {
+	public static <F> F[] purgeUnwantedElements(F[] fileArray, Predicate<? extends F> predicate,
+			@Nullable Comparator<? extends F> comparator) {
 
 		if (ObjectUtils.isEmpty(fileArray)) {
 			return fileArray;
 		}
 		else {
 			if (comparator == null) {
 				return Arrays.stream(fileArray)
-						.filter(predicate.negate())
+						.filter((Predicate<? super F>) predicate.negate())
 						.toArray(size -> (F[]) Array.newInstance(fileArray[0].getClass(), size));
 			}
 			else {
 				return Arrays.stream(fileArray)
-						.filter(predicate.negate())
-						.sorted(comparator)
+						.filter((Predicate<? super F>) predicate.negate())
+						.sorted((Comparator<? super F>) comparator)
 						.toArray(size -> (F[]) Array.newInstance(fileArray[0].getClass(), size));
 			}
 		}

spring-integration-sftp/src/main/java/org/springframework/integration/sftp/dsl/Sftp.java

@@ -19,6 +19,8 @@
 import java.io.File;
 import java.util.Comparator;
 
+import org.apache.sshd.sftp.client.SftpClient;
+
 import org.springframework.integration.file.remote.MessageSessionCallback;
 import org.springframework.integration.file.remote.RemoteFileTemplate;
 import org.springframework.integration.file.remote.gateway.AbstractRemoteFileOutboundGateway;
@@ -27,9 +29,6 @@
 import org.springframework.integration.sftp.gateway.SftpOutboundGateway;
 import org.springframework.integration.sftp.session.SftpRemoteFileTemplate;
 
-import com.jcraft.jsch.ChannelSftp;
-import com.jcraft.jsch.ChannelSftp.LsEntry;
-
 /**
  * The factory for SFTP components.
  *
@@ -46,7 +45,7 @@ public final class Sftp {
 	 * @param sessionFactory the session factory.
 	 * @return the spec.
 	 */
-	public static SftpInboundChannelAdapterSpec inboundAdapter(SessionFactory<ChannelSftp.LsEntry> sessionFactory) {
+	public static SftpInboundChannelAdapterSpec inboundAdapter(SessionFactory<SftpClient.DirEntry> sessionFactory) {
 		return inboundAdapter(sessionFactory, null);
 	}
 
@@ -56,7 +55,7 @@ public static SftpInboundChannelAdapterSpec inboundAdapter(SessionFactory<Channe
 	 * @param receptionOrderComparator the comparator.
 	 * @return the spec.
 	 */
-	public static SftpInboundChannelAdapterSpec inboundAdapter(SessionFactory<ChannelSftp.LsEntry> sessionFactory,
+	public static SftpInboundChannelAdapterSpec inboundAdapter(SessionFactory<SftpClient.DirEntry> sessionFactory,
 			Comparator<File> receptionOrderComparator) {
 
 		return new SftpInboundChannelAdapterSpec(sessionFactory, receptionOrderComparator);
@@ -69,7 +68,7 @@ public static SftpInboundChannelAdapterSpec inboundAdapter(SessionFactory<Channe
 	 * @return the spec.
 	 */
 	public static SftpStreamingInboundChannelAdapterSpec inboundStreamingAdapter(
-			RemoteFileTemplate<LsEntry> remoteFileTemplate) {
+			RemoteFileTemplate<SftpClient.DirEntry> remoteFileTemplate) {
 
 		return inboundStreamingAdapter(remoteFileTemplate, null);
 	}
@@ -82,8 +81,8 @@ public static SftpStreamingInboundChannelAdapterSpec inboundStreamingAdapter(
 	 * @return the spec.
 	 */
 	public static SftpStreamingInboundChannelAdapterSpec inboundStreamingAdapter(
-			RemoteFileTemplate<LsEntry> remoteFileTemplate,
-			Comparator<LsEntry> receptionOrderComparator) {
+			RemoteFileTemplate<SftpClient.DirEntry> remoteFileTemplate,
+			Comparator<SftpClient.DirEntry> receptionOrderComparator) {
 
 		return new SftpStreamingInboundChannelAdapterSpec(remoteFileTemplate, receptionOrderComparator);
 	}
@@ -93,7 +92,7 @@ public static SftpStreamingInboundChannelAdapterSpec inboundStreamingAdapter(
 	 * @param sessionFactory the session factory.
 	 * @return the spec.
 	 */
-	public static SftpMessageHandlerSpec outboundAdapter(SessionFactory<ChannelSftp.LsEntry> sessionFactory) {
+	public static SftpMessageHandlerSpec outboundAdapter(SessionFactory<SftpClient.DirEntry> sessionFactory) {
 		return new SftpMessageHandlerSpec(sessionFactory);
 	}
 
@@ -103,7 +102,7 @@ public static SftpMessageHandlerSpec outboundAdapter(SessionFactory<ChannelSftp.
 	 * @param fileExistsMode the file exists mode.
 	 * @return the spec.
 	 */
-	public static SftpMessageHandlerSpec outboundAdapter(SessionFactory<ChannelSftp.LsEntry> sessionFactory,
+	public static SftpMessageHandlerSpec outboundAdapter(SessionFactory<SftpClient.DirEntry> sessionFactory,
 			FileExistsMode fileExistsMode) {
 
 		return outboundAdapter(new SftpRemoteFileTemplate(sessionFactory), fileExistsMode);
@@ -141,7 +140,7 @@ public static SftpMessageHandlerSpec outboundAdapter(SftpRemoteFileTemplate sftp
 	 * @param expression the remoteFilePath SpEL expression.
 	 * @return the {@link SftpOutboundGatewaySpec}
 	 */
-	public static SftpOutboundGatewaySpec outboundGateway(SessionFactory<ChannelSftp.LsEntry> sessionFactory,
+	public static SftpOutboundGatewaySpec outboundGateway(SessionFactory<SftpClient.DirEntry> sessionFactory,
 			AbstractRemoteFileOutboundGateway.Command command, String expression) {
 
 		return outboundGateway(sessionFactory, command.getCommand(), expression);
@@ -157,7 +156,7 @@ public static SftpOutboundGatewaySpec outboundGateway(SessionFactory<ChannelSftp
 	 * @return the {@link SftpOutboundGatewaySpec}
 	 * @see RemoteFileTemplate
 	 */
-	public static SftpOutboundGatewaySpec outboundGateway(SessionFactory<ChannelSftp.LsEntry> sessionFactory,
+	public static SftpOutboundGatewaySpec outboundGateway(SessionFactory<SftpClient.DirEntry> sessionFactory,
 			String command, String expression) {
 
 		return new SftpOutboundGatewaySpec(new SftpOutboundGateway(sessionFactory, command, expression));
@@ -172,7 +171,7 @@ public static SftpOutboundGatewaySpec outboundGateway(SessionFactory<ChannelSftp
 	 * @return the {@link SftpOutboundGatewaySpec}
 	 * @see RemoteFileTemplate
 	 */
-	public static SftpOutboundGatewaySpec outboundGateway(RemoteFileTemplate<ChannelSftp.LsEntry> remoteFileTemplate,
+	public static SftpOutboundGatewaySpec outboundGateway(RemoteFileTemplate<SftpClient.DirEntry> remoteFileTemplate,
 			AbstractRemoteFileOutboundGateway.Command command, String expression) {
 
 		return outboundGateway(remoteFileTemplate, command.getCommand(), expression);
@@ -187,7 +186,7 @@ public static SftpOutboundGatewaySpec outboundGateway(RemoteFileTemplate<Channel
 	 * @return the {@link SftpOutboundGatewaySpec}
 	 * @see RemoteFileTemplate
 	 */
-	public static SftpOutboundGatewaySpec outboundGateway(RemoteFileTemplate<ChannelSftp.LsEntry> remoteFileTemplate,
+	public static SftpOutboundGatewaySpec outboundGateway(RemoteFileTemplate<SftpClient.DirEntry> remoteFileTemplate,
 			String command, String expression) {
 
 		return new SftpOutboundGatewaySpec(new SftpOutboundGateway(remoteFileTemplate, command, expression));
@@ -201,8 +200,8 @@ public static SftpOutboundGatewaySpec outboundGateway(RemoteFileTemplate<Channel
 	 * @return the {@link SftpOutboundGatewaySpec}
 	 * @see MessageSessionCallback
 	 */
-	public static SftpOutboundGatewaySpec outboundGateway(SessionFactory<ChannelSftp.LsEntry> sessionFactory,
-			MessageSessionCallback<ChannelSftp.LsEntry, ?> messageSessionCallback) {
+	public static SftpOutboundGatewaySpec outboundGateway(SessionFactory<SftpClient.DirEntry> sessionFactory,
+			MessageSessionCallback<SftpClient.DirEntry, ?> messageSessionCallback) {
 
 		return new SftpOutboundGatewaySpec(new SftpOutboundGateway(sessionFactory, messageSessionCallback));
 	}

spring-integration-sftp/src/main/java/org/springframework/integration/sftp/dsl/SftpInboundChannelAdapterSpec.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2014-2020 the original author or authors.
+ * Copyright 2014-2022 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -19,6 +19,8 @@
 import java.io.File;
 import java.util.Comparator;
 
+import org.apache.sshd.sftp.client.SftpClient;
+
 import org.springframework.integration.file.dsl.RemoteFileInboundChannelAdapterSpec;
 import org.springframework.integration.file.filters.CompositeFileListFilter;
 import org.springframework.integration.file.filters.FileListFilter;
@@ -30,8 +32,6 @@
 import org.springframework.integration.sftp.inbound.SftpInboundFileSynchronizer;
 import org.springframework.integration.sftp.inbound.SftpInboundFileSynchronizingMessageSource;
 
-import com.jcraft.jsch.ChannelSftp;
-
 /**
  * A {@link RemoteFileInboundChannelAdapterSpec} for an {@link SftpInboundFileSynchronizingMessageSource}.
  *
@@ -40,10 +40,10 @@
  * @since 5.0
  */
 public class SftpInboundChannelAdapterSpec
-		extends RemoteFileInboundChannelAdapterSpec<ChannelSftp.LsEntry, SftpInboundChannelAdapterSpec,
-				SftpInboundFileSynchronizingMessageSource> {
+		extends RemoteFileInboundChannelAdapterSpec<SftpClient.DirEntry, SftpInboundChannelAdapterSpec,
+		SftpInboundFileSynchronizingMessageSource> {
 
-	protected SftpInboundChannelAdapterSpec(SessionFactory<ChannelSftp.LsEntry> sessionFactory,
+	protected SftpInboundChannelAdapterSpec(SessionFactory<SftpClient.DirEntry> sessionFactory,
 			Comparator<File> comparator) {
 
 		super(new SftpInboundFileSynchronizer(sessionFactory));
@@ -70,9 +70,10 @@ public SftpInboundChannelAdapterSpec regexFilter(String regex) {
 		return filter(composeFilters(new SftpRegexPatternFileListFilter(regex)));
 	}
 
-	private CompositeFileListFilter<ChannelSftp.LsEntry> composeFilters(FileListFilter<ChannelSftp.LsEntry>
+	private CompositeFileListFilter<SftpClient.DirEntry> composeFilters(FileListFilter<SftpClient.DirEntry>
 			fileListFilter) {
-		CompositeFileListFilter<ChannelSftp.LsEntry> compositeFileListFilter = new CompositeFileListFilter<>();
+
+		CompositeFileListFilter<SftpClient.DirEntry> compositeFileListFilter = new CompositeFileListFilter<>();
 		compositeFileListFilter.addFilters(fileListFilter,
 				new SftpPersistentAcceptOnceFileListFilter(new SimpleMetadataStore(), "sftpMessageSource"));
 		return compositeFileListFilter;

spring-integration-sftp/src/main/java/org/springframework/integration/sftp/dsl/SftpMessageHandlerSpec.java

@@ -16,14 +16,14 @@
 
 package org.springframework.integration.sftp.dsl;
 
+import org.apache.sshd.sftp.client.SftpClient;
+
 import org.springframework.integration.file.dsl.FileTransferringMessageHandlerSpec;
 import org.springframework.integration.file.remote.session.SessionFactory;
 import org.springframework.integration.file.support.FileExistsMode;
 import org.springframework.integration.sftp.outbound.SftpMessageHandler;
 import org.springframework.integration.sftp.session.SftpRemoteFileTemplate;
 
-import com.jcraft.jsch.ChannelSftp;
-
 /**
  * @author Artem Bilan
  * @author Joaquin Santana
@@ -32,9 +32,9 @@
  * @since 5.0
  */
 public class SftpMessageHandlerSpec
-		extends FileTransferringMessageHandlerSpec<ChannelSftp.LsEntry, SftpMessageHandlerSpec> {
+		extends FileTransferringMessageHandlerSpec<SftpClient.DirEntry, SftpMessageHandlerSpec> {
 
-	protected SftpMessageHandlerSpec(SessionFactory<ChannelSftp.LsEntry> sessionFactory) {
+	protected SftpMessageHandlerSpec(SessionFactory<SftpClient.DirEntry> sessionFactory) {
 		this.target = new SftpMessageHandler(sessionFactory);
 	}
 

spring-integration-sftp/src/main/java/org/springframework/integration/sftp/dsl/SftpOutboundGatewaySpec.java

@@ -16,24 +16,24 @@
 
 package org.springframework.integration.sftp.dsl;
 
+import org.apache.sshd.sftp.client.SftpClient;
+
 import org.springframework.integration.file.dsl.RemoteFileOutboundGatewaySpec;
 import org.springframework.integration.file.remote.gateway.AbstractRemoteFileOutboundGateway;
 import org.springframework.integration.sftp.filters.SftpRegexPatternFileListFilter;
 import org.springframework.integration.sftp.filters.SftpSimplePatternFileListFilter;
 
-import com.jcraft.jsch.ChannelSftp;
-
 /**
  * @author Artem Bilan
  * @author Gary Russell
  *
  * @since 5.0
  */
 public class SftpOutboundGatewaySpec
-		extends RemoteFileOutboundGatewaySpec<ChannelSftp.LsEntry, SftpOutboundGatewaySpec> {
+		extends RemoteFileOutboundGatewaySpec<SftpClient.DirEntry, SftpOutboundGatewaySpec> {
 
 
-	protected SftpOutboundGatewaySpec(AbstractRemoteFileOutboundGateway<ChannelSftp.LsEntry> outboundGateway) {
+	protected SftpOutboundGatewaySpec(AbstractRemoteFileOutboundGateway<SftpClient.DirEntry> outboundGateway) {
 		super(outboundGateway);
 	}