Apache VFS移动FTP文件太慢原因及解决办法

1
2
3
4
5
6
7
8
9

protected FileObject findFile(final FileName name, final FileSystemOptions fileSystemOptions)
            throws FileSystemException {
    // Check in the cache for the file system
    final FileName rootName = getContext().getFileSystemManager().resolveName(name, FileName.ROOT_PATH);
    final FileSystem fs = getFileSystem(rootName, fileSystemOptions);
    // Locate the file
    // return fs.resolveFile(name.getPath());
    return fs.resolveFile(name);
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14

protected FtpFileObject(final AbstractFileName name, final FtpFileSystem fileSystem, final FileName rootName)
            throws FileSystemException {
    super(name, fileSystem);
    final String relPath = UriParser.decode(rootName.getRelativeName(name));
    if (".".equals(relPath)) {
        // do not use the "." as path against the ftp-server
        // e.g. the uu.net ftp-server do a recursive listing then
        // this.relPath = UriParser.decode(rootName.getPath());
        // this.relPath = ".";
        this.relPath = null;
    } else {
        this.relPath = relPath;
    }
}

1

private FTPFile fileInfo;

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14

public void putClient(final FtpClient client) {
    // Save client for reuse if none is idle.
    if (!idleClient.compareAndSet(null, client)) {
        // An idle client is already present so close the connection.
        closeConnection(client);
    }
}
public FtpClient getClient() throws FileSystemException {
    FtpClient client = idleClient.getAndSet(null);
    if (client == null || !client.isConnected()) {
        client = createWrapper();
    }
    return client;
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44

@Override
public void moveTo(final FileObject destFile) throws FileSystemException {
    if (canRenameTo(destFile)) {
        if (!getParent().isWriteable()) {
            throw new FileSystemException("vfs.provider/rename-parent-read-only.error", getName(),getParent().getName());
        }
    } else {
        if (!isWriteable()) {
            throw new FileSystemException("vfs.provider/rename-read-only.error", getName());
        }
    }
    if (destFile.exists() && !isSameFile(destFile)) {
        destFile.deleteAll();
        // throw new FileSystemException("vfs.provider/rename-dest-exists.error", destFile.getName());
    }
    if (canRenameTo(destFile)) {
        // issue rename on same filesystem
        try {
            attach();
            // remember type to avoid attach
            final FileType srcType = getType();
            doRename(destFile); 
            FileObjectUtils.getAbstractFileObject(destFile).handleCreate(srcType);
            destFile.close(); // now the destFile is no longer imaginary. force reattach.
            handleDelete(); // fire delete-events. This file-object (src) is like deleted.
            } catch (final RuntimeException re) {
                throw re;
            } catch (final Exception exc) {
                throw new FileSystemException("vfs.provider/rename.error", exc, getName(), destFile.getName());
            }
        } else {
            // different fs - do the copy/delete stuff
            destFile.copyFrom(this, Selectors.SELECT_SELF);
            if ((destFile.getType().hasContent()
                    && destFile.getFileSystem().hasCapability(Capability.SET_LAST_MODIFIED_FILE)
                    || destFile.getType().hasChildren()
                            && destFile.getFileSystem().hasCapability(Capability.SET_LAST_MODIFIED_FOLDER))
                    && fs.hasCapability(Capability.GET_LAST_MODIFIED)) {
            destFile.getContent().setLastModifiedTime(this.getContent().getLastModifiedTime());
        }
        deleteSelf();
    }

}

1
2
3
4

@Override
public boolean canRenameTo(final FileObject newfile) {
    return fs == newfile.getFileSystem();
}

1
2
3
4

@Override
public boolean exists() throws FileSystemException {
    return getType() != FileType.IMAGINARY;
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

@Override
protected FileType doGetType() throws Exception {
    // VFS-210
    synchronized (getFileSystem()) {
        if (this.fileInfo == null) {
            getInfo(false);
        }
        if (this.fileInfo == UNKNOWN) {
            return FileType.IMAGINARY;
        } else if (this.fileInfo.isDirectory()) {
            return FileType.FOLDER;
        } else if (this.fileInfo.isFile()) {
            return FileType.FILE;
        } else if (this.fileInfo.isSymbolicLink()) {
            final FileObject linkDest = getLinkDestination();
            // VFS-437: We need to check if the symbolic link links back to the symbolic link itself
            if (this.isCircular(linkDest)) {
                // If the symbolic link links back to itself, treat it as an imaginary file to prevent following
                // this link. If the user tries to access the link as a file or directory, the user will end up with
                // a FileSystemException warning that the file cannot be accessed. This is to prevent the infinite
                // call back to doGetType() to prevent the StackOverFlow
                return FileType.IMAGINARY;
            }
            return linkDest.getType();
        }
    }
    throw new FileSystemException("vfs.provider.ftp/get-type.error", getName());
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19

private FTPFile getChildFile(final String name, final boolean flush) throws IOException {
    /*
    * If we should flush cached children, clear our children map unless we're in the middle of a refresh in which
    * case we've just recently refreshed our children. No need to do it again when our children are refresh()ed,
    * calling getChildFile() for themselves from within getInfo(). See getChildren().
    */
    if (flush && !inRefresh) {
        children = null;
    }
    // List the children of this file
    doGetChildren();
    // VFS-210
    if (children == null) {
        return null;
    }
    // Look for the requested child
    final FTPFile ftpFile = children.get(name);
    return ftpFile;
}

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257

import java.io.IOException;
import java.util.Arrays;
import java.util.Map;
import java.util.concurrent.atomic.AtomicReference;

import org.apache.commons.net.ftp.FTPClient;
import org.apache.commons.net.ftp.FTPFile;
import org.apache.commons.vfs2.FileSystemException;
import org.apache.commons.vfs2.FileSystemOptions;
import org.apache.commons.vfs2.provider.UriParser;
import org.apache.commons.vfs2.provider.ftp.FtpClientFactory;
import org.apache.commons.vfs2.provider.ftp.FtpFileSystemConfigBuilder;
import org.apache.commons.vfs2.util.Cryptor;
import org.apache.commons.vfs2.util.CryptorFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.google.common.collect.Maps;

public class FtpUtil {
    private static Logger logger = LoggerFactory.getLogger(FtpUtil.class);
    
    private final static Map<Auth, AtomicReference<FTPClient>> clients = Maps.newConcurrentMap();
    
    public static boolean move(String src, String tar) throws IOException {
        FtpPath srcFtpPath = parse(src);
        FtpPath tarFtpPath = parse(tar);
        if (!srcFtpPath.auth.equals(tarFtpPath.auth)) {
            throw new UnsupportedOperationException("源目录和目标目录的ftp服务器连接信息不一致");
        }
        FTPClient ftpClient = getFTPClient(srcFtpPath.auth);
        try {
            return ftpClient.rename(srcFtpPath.path, tarFtpPath.path);
        } catch (IOException e) {
            closeConnection(ftpClient);
            throw e;
        } finally {
            putFTPClient(srcFtpPath.auth, ftpClient);
        }
    }
    
    public static FtpPath parse(String uri) throws FileSystemException {
        FtpPath ftpPath = new FtpPath();
        StringBuilder name = new StringBuilder();
        UriParser.extractScheme(uri, name);
        // Expecting "//"
        if (name.length() < 2 || name.charAt(0) != '/' || name.charAt(1) != '/') {
            throw new FileSystemException("vfs.provider/missing-double-slashes.error", uri);
        }
        name.delete(0, 2);
     // Extract userinfo, and split into username and password
        final String userInfo = extractUserInfo(name);
        final String userName;
        final String password;
        if (userInfo != null) {
            final int idx = userInfo.indexOf(':');
            if (idx == -1) {
                userName = userInfo;
                password = null;
            } else {
                userName = userInfo.substring(0, idx);
                password = userInfo.substring(idx + 1);
            }
        } else {
            userName = null;
            password = null;
        }
        
        String u = UriParser.decode(userName);
        String p = UriParser.decode(password);
    
        if (p != null && p.startsWith("{") && p.endsWith("}")) {
            try {
                final Cryptor cryptor = CryptorFactory.getCryptor();
                p = cryptor.decrypt(p.substring(1, p.length() - 1));
            } catch (final Exception ex) {
                throw new FileSystemException("Unable to decrypt password", ex);
            }
        }
        
        ftpPath.auth.username = u == null ? null : u.toCharArray();
        ftpPath.auth.password = p == null ? null : p.toCharArray();
        
        // Extract hostname, and normalise (lowercase)
        final String hostName = extractHostName(name);
        if (hostName == null) {
            throw new FileSystemException("vfs.provider/missing-hostname.error", uri);
        }
        ftpPath.auth.host = hostName.toLowerCase();
    
        // Extract port
        ftpPath.auth.port = extractPort(name, uri);
    
        // Expecting '/' or empty name
        if (name.length() > 0 && name.charAt(0) != '/') {
            throw new FileSystemException("vfs.provider/missing-hostname-path-sep.error", uri);
        }
        
        ftpPath.path = name.toString();
        return ftpPath;
    }
    
    /**
     * Extracts the user info from a URI.
     *
     * @param name string buffer with the "scheme://" part has been removed already. Will be modified.
     * @return the user information up to the '@' or null.
     */
    private static String extractUserInfo(final StringBuilder name) {
        final int maxlen = name.length();
        for (int pos = 0; pos < maxlen; pos++) {
            final char ch = name.charAt(pos);
            if (ch == '@') {
                // Found the end of the user info
                final String userInfo = name.substring(0, pos);
                name.delete(0, pos + 1);
                return userInfo;
            }
            if (ch == '/' || ch == '?') {
                // Not allowed in user info
                break;
            }
        }
    
        // Not found
        return null;
    }
    
    /**
     * Extracts the hostname from a URI.
     *
     * @param name string buffer with the "scheme://[userinfo@]" part has been removed already. Will be modified.
     * @return the host name or null.
     */
    private static String extractHostName(final StringBuilder name) {
        final int maxlen = name.length();
        int pos = 0;
        for (; pos < maxlen; pos++) {
            final char ch = name.charAt(pos);
            if (ch == '/' || ch == ';' || ch == '?' || ch == ':' || ch == '@' || ch == '&' || ch == '=' || ch == '+'
                    || ch == '$' || ch == ',') {
                break;
            }
        }
        if (pos == 0) {
            return null;
        }
    
        final String hostname = name.substring(0, pos);
        name.delete(0, pos);
        return hostname;
    }
    
    /**
     * Extracts the port from a URI.
     *
     * @param name string buffer with the "scheme://[userinfo@]hostname" part has been removed already. Will be
     *            modified.
     * @param uri full URI for error reporting.
     * @return The port, or -1 if the URI does not contain a port.
     * @throws FileSystemException if URI is malformed.
     * @throws NumberFormatException if port number cannot be parsed.
     */
    private static int extractPort(final StringBuilder name, final String uri) throws FileSystemException {
        if (name.length() < 1 || name.charAt(0) != ':') {
            return -1;
        }
    
        final int maxlen = name.length();
        int pos = 1;
        for (; pos < maxlen; pos++) {
            final char ch = name.charAt(pos);
            if (ch < '0' || ch > '9') {
                break;
            }
        }
    
        final String port = name.substring(1, pos);
        name.delete(0, pos);
        if (port.length() == 0) {
            throw new FileSystemException("vfs.provider/missing-port.error", uri);
        }
    
        return Integer.parseInt(port);
    }
    
    private static FTPClient getFTPClient(Auth key) throws IOException {
        AtomicReference<FTPClient> refClient = clients.getOrDefault(key, new AtomicReference<FTPClient>(null));
        
        FTPClient client = refClient.getAndSet(null);
        if (client == null || !client.isConnected()) {
            client = createClient(key);
        }
        return client;
    }
    
    private static FTPClient createClient(Auth key) throws IOException {
        FtpFileSystemConfigBuilder builder = FtpFileSystemConfigBuilder.getInstance();
        FileSystemOptions options = new FileSystemOptions();
        builder.setControlEncoding(options, "UTF-8");
        builder.setServerLanguageCode(options, "zh");
        builder.setPassiveMode(options, true);
        return FtpClientFactory.createConnection(key.host, key.port, key.username, key.password, null, options);
    }
    
    private static void putFTPClient(Auth key, FTPClient client) {
        AtomicReference<FTPClient> refClient = clients.getOrDefault(key, new AtomicReference<FTPClient>(null));
        
        if (!refClient.compareAndSet(null, client)) {
            closeConnection(client);
        }
    }
    
    private static void closeConnection(FTPClient client) {
        try {
            if (client.isConnected()) {
                client.disconnect();
            }
        } catch (final IOException e) {
            logger.error(e.getMessage(), e);
        }
    }
    
    private static class Auth {
        String host;
        int port;
        char[] username;
        char[] password;
        
        @Override
        public boolean equals(Object obj) {
            if (this == obj) {
                return true;
            }
            if (obj instanceof Auth) {
                Auth k = (Auth) obj;
                return this.host.equals(k.host) && this.port == k.port && Arrays.equals(this.username, k.username)
                        && Arrays.equals(this.password, k.password);
            }
            return false;
        }
        
        @Override
        public int hashCode() {
            int h = host.hashCode();
            h = 31 * h + port;
            h = 31 * h + username.hashCode();
            h = 31 * h + password.hashCode();
            return h;
        }
    }
    
    private static class FtpPath {
        Auth auth = new Auth();
        String path;
    }
}