commit 606c7aeff77ed85d0c8374e5ef56a9fc92549687 from: leo date: Thu Feb 22 10:36:00 2007 UTC changed layout and packages added rw capabilities with caching (primitive) commit - 5ef073501ae508b2c79b5ea9d4369bef6f380c51 commit + 606c7aeff77ed85d0c8374e5ef56a9fc92549687 blob - 0e0225a25f66bc7037f99212080eb30f5bf7e944 blob + 15364b45c9595d79770be30f05df170501a74e69 --- src/main/java/com/thinkberg/moxo/ResourceManager.java +++ src/main/java/com/thinkberg/moxo/ResourceManager.java @@ -39,7 +39,7 @@ public class ResourceManager { root = fsm.createVirtualFileSystem(rootObject); System.err.println("Created virtual file system: " + rootObject); } catch (FileSystemException e) { - System.err.println("Can't create virtual filesystem: " + e.getMessage()); + System.err.println("Can't create virtual file system: " + e.getMessage()); e.printStackTrace(); } } blob - 0299d52204489160b361b989b9f0e74e73180407 blob + 7a8e901869c402cbbb6c27b78bb16ebe5cec3292 --- src/main/java/com/thinkberg/moxo/dav/PutHandler.java +++ src/main/java/com/thinkberg/moxo/dav/PutHandler.java @@ -66,8 +66,10 @@ public class PutHandler extends WebdavHandler { InputStream is = request.getInputStream(); OutputStream os = object.getContent().getOutputStream(); - log("copied " + Util.copyStream(is, os) + " bytes"); - os.close(); + log("PUT sends " + request.getHeader("Content-length") + " bytes"); + log("PUT copied " + Util.copyStream(is, os) + " bytes"); + os.flush(); + object.close(); response.setStatus(HttpServletResponse.SC_CREATED); } blob - 1729f3d7d238c8ce6d25dae22834c23c3f734ea3 blob + 635e944e7bab4630c048790a9115f02f07481a7c --- src/main/java/com/thinkberg/moxo/dav/Util.java +++ src/main/java/com/thinkberg/moxo/dav/Util.java @@ -41,7 +41,7 @@ public class Util { public static int copyStream(InputStream is, OutputStream os) throws IOException { - byte[] buffer = new byte[4096]; + byte[] buffer = new byte[8192]; int bytesRead, bytesCount = 0; while ((bytesRead = is.read(buffer)) != -1) { os.write(buffer, 0, bytesRead); blob - c9780e5ce9c328ffad20e67ea417ac5ce93ae6e7 blob + a74e1a9ff4bbc10d2647532ca2252c2a48acd7c8 --- src/main/java/com/thinkberg/moxo/dav/WebdavHandler.java +++ src/main/java/com/thinkberg/moxo/dav/WebdavHandler.java @@ -120,7 +120,7 @@ public abstract class WebdavHandler { * * @param request the servlet request * @return the file object of the destination - * @throws FileSystemException if the filesystem cannot create a file object + * @throws FileSystemException if the file system cannot create a file object * @throws MalformedURLException if the url is misformatted */ FileObject getDestination(HttpServletRequest request) throws FileSystemException, MalformedURLException { blob - 5d8f96a1cb04ca9f1dfd9b2d00daa1eb695d0e06 (mode 644) blob + /dev/null --- src/main/java/com/thinkberg/moxo/s3/S3Connector.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright 2007 Matthias L. Jugel. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.thinkberg.moxo.s3; - -import org.apache.commons.vfs.FileSystemException; -import org.jets3t.service.Jets3tProperties; -import org.jets3t.service.S3Service; -import org.jets3t.service.S3ServiceException; -import org.jets3t.service.impl.rest.httpclient.RestS3Service; -import org.jets3t.service.model.S3Bucket; -import org.jets3t.service.security.AWSCredentials; - -/** - * @author Matthias L. Jugel - */ -public class S3Connector { - private static final String APPLICATION_DESCRIPTION = "S3 VFS Connector/1.0"; - - private static S3Connector instance; - - /** - * Get an instance of the S3Connector which is initialized and authenticated to the - * Amazon S3 Service. - * - * @return an S3 connector - * @throws FileSystemException if connection or authentication fails - */ - public static S3Connector getInstance() throws FileSystemException { - if (null == instance) { - instance = new S3Connector(); - } - return instance; - } - - private S3Service service; - - /** - * Initialize Amazon S3. - * - * @throws FileSystemException if S3 can't be initialized - */ - private S3Connector() throws FileSystemException { - String propertiesFileName = System.getProperty("moxo.properties", "moxo.properties"); - Jets3tProperties properties = Jets3tProperties.getInstance(propertiesFileName); - - if (!properties.isLoaded()) { - throw new FileSystemException("can't find S3 configuration: " + propertiesFileName); - } - - AWSCredentials awsCredentials = new AWSCredentials( - properties.getStringProperty("accesskey", null), - properties.getStringProperty("secretkey", null)); - - - try { - service = new RestS3Service(awsCredentials, APPLICATION_DESCRIPTION, null); - } catch (S3ServiceException e) { - throw new FileSystemException("can't initialize S3 Service", e); - } - } - - /** - * Get a virtual file system root corresponding to a bucket. - * - * @param bucket the bucket that contains the filesystem - * @return the S3 root - * @throws FileSystemException if the bucket is not found - */ - public S3VfsRoot getRoot(String bucket) throws FileSystemException { - try { - if (service.isBucketAccessible(bucket)) { - return new S3VfsRootImpl(service, new S3Bucket(bucket)); - } - throw new FileSystemException("vsf.provider.vfs/cant-access.error"); - } catch (S3ServiceException e) { - throw new FileSystemException(e); - } - } -} blob - b647ccf5ddd6dc7bd616ac1e91ecafb4480ede53 (mode 644) blob + /dev/null --- src/main/java/com/thinkberg/moxo/s3/S3VfsObject.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Copyright 2007 Matthias L. Jugel. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.thinkberg.moxo.s3; - -import org.apache.commons.vfs.FileSystemException; -import org.apache.commons.vfs.FileType; - -import java.io.InputStream; - -/** - * This interface describes methods to access S3 Objects whether files or folders. - * - * @author Matthias L. Jugel - */ -public interface S3VfsObject { - /** - * The base name of the S3 Object. - * - * @return the base name without path - */ - public String getName(); - - /** - * The absolute path of the S3 Object. - * - * @return the absolute path - */ - public String getPath(); - - /** - * The type of the S3 Object. May be file, folder or imaginary. - * - * @return the file type - * @see org.apache.commons.vfs.FileType#FILE - * @see org.apache.commons.vfs.FileType#FOLDER - * @see org.apache.commons.vfs.FileType#IMAGINARY - */ - public FileType getType(); - - /** - * Get the last modified time. The value is undefined if this is an imaginary file. - * - * @return the last modified time in milliseconds - */ - public long getLastModified(); - - /** - * Get the content length. The value may be 0 for imaginary files. - * - * @return the content length in bytes - */ - public long getContentLength(); - - /** - * Get the actual content MIME type. May be null for imaginary files. - * - * @return the MIME type - */ - public String getContentType(); - - /** - * Get the input stream to read data from the object or null if this file is imaginary - * - * @return the input stream - * @throws FileSystemException if the S3 object cannot be accessed - */ - public InputStream getInputStream() throws FileSystemException; - - /** - * List all children names relative to this S3 Object. It is assumed that this is - * only called on folders. Non-Folders may return results but there is no defined - * behaviour in such a case. - * - * @return the list of children names (without path) - * @throws FileSystemException if the object cannot be accessed - */ - public String[] getChildren() throws FileSystemException; -} blob - 413839eee68c4c031e4a9e55a9ad6c96d64f13c8 (mode 644) blob + /dev/null --- src/main/java/com/thinkberg/moxo/s3/S3VfsObjectImpl.java +++ /dev/null @@ -1,222 +0,0 @@ -/* - * Copyright 2007 Matthias L. Jugel. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.thinkberg.moxo.s3; - -import org.apache.commons.vfs.FileSystemException; -import org.apache.commons.vfs.FileType; -import org.jets3t.service.S3Service; -import org.jets3t.service.S3ServiceException; -import org.jets3t.service.model.S3Bucket; -import org.jets3t.service.model.S3Object; -import org.jets3t.service.utils.Mimetypes; - -import java.io.InputStream; - -/** - * Implementation of the virtual S3 file system object using the Jets3t library. - * - * @author Matthias L. Jugel - */ -public class S3VfsObjectImpl implements S3VfsObject { - private final S3Service service; - private final S3Bucket bucket; - private S3Object object; - - /** - * Create a new S3 Object wrapper for the virtual filesystem. If an object exists its details - * are loaded and a virtual object is created for non-existing files. If data is requested that - * is not part of the details (HEAD) the real object is loaded on demand. - * - * @param service the S3 service used for retrieving the object - * @param bucket the bucket the object is located in - * @param path the full path to the S3 object (the key) - * @throws FileSystemException if there is a problem accessing the object - */ - @SuppressWarnings({"RedundantThrows"}) - public S3VfsObjectImpl(S3Service service, S3Bucket bucket, String path) throws FileSystemException { - this.service = service; - this.bucket = bucket; - // check object and load details or create a virtual object - String s3Path = makeS3Path(path); - try { - object = service.getObjectDetails(bucket, s3Path); - } catch (S3ServiceException e) { - object = new S3Object(bucket, s3Path); - } - } - - /** - * Get the name of this object. - * - * @return the base name of the object - */ - public String getName() { - String key = object.getKey(); - if ("".equals(key)) { - return "/"; - } - - int lastSlash = key.lastIndexOf("/"); - if (lastSlash == -1) { - return key; - } else { - return key.substring(lastSlash + 1); - } - } - - /** - * The file type as provided by commons-vfs. - * - * @return the file type - * @see org.apache.commons.vfs.FileType#FILE - * @see org.apache.commons.vfs.FileType#FOLDER - * @see org.apache.commons.vfs.FileType#IMAGINARY - */ - public FileType getType() { - if (null == object.getContentType()) { - return FileType.IMAGINARY; - } - - String key = object.getKey(); - String contentType = object.getContentType(); - if ("".equals(key) || "/".equals(key) || Mimetypes.MIMETYPE_JETS3T_DIRECTORY.equals(contentType)) { - return FileType.FOLDER; - } - - return FileType.FILE; - } - - /** - * Get the full path of the object as an absolute path (including heading /). - * - * @return the full path - */ - public String getPath() { - return makeAbsolutePath(object.getKey()); - } - - /** - * Get the last modified time of the object. The result is undefined if this object does not exist. - * - * @return the last modified time - */ - public long getLastModified() { - return object.getLastModifiedDate().getTime(); - } - - /** - * Get the content length of the object. The result is 0 for non-existing objects. - * - * @return the content length - */ - public long getContentLength() { - return object.getContentLength(); - } - - /** - * Get the actual content type as a MIME type. The result is undefined if this object does not exist. - * - * @return the mime type - */ - public String getContentType() { - return object.getContentType(); - } - - /** - * Get all children names relative to this object. It is assumed that - * this method is called for directories only. The returned names are - * not absolute names, but rather relative base names. - * - * @return the list of children names - * @throws FileSystemException if the object cannot be accessed - */ - public String[] getChildren() throws FileSystemException { - String path = object.getKey(); - // make sure we add a '/' slash at the end to find children - if (!"".equals(path)) { - path = path + "/"; - } - - try { - S3Object[] children = service.listObjects(bucket, path, "/"); - String[] childrenNames = new String[children.length]; - for (int i = 0; i < children.length; i++) { - if (!children[i].getKey().equals(path)) { - // strip path from name (leave only base name) - childrenNames[i] = children[i].getKey().replaceAll("[^/]*//*", ""); - } - } - return childrenNames; - } catch (S3ServiceException e) { - throw new FileSystemException(e); - } - } - - /** - * Get the input stream to read data from the object. Returns null for non-existing objects. - * Calling this method causes the object to be reloaded if it has not yet been fully - * retrieved from S3. - * - * @return the input stream - * @throws FileSystemException if the input stream cannot be accessed - */ - public InputStream getInputStream() throws FileSystemException { - try { - InputStream is = object.getDataInputStream(); - if (null == is) { - object = service.getObject(bucket, object.getKey()); - is = object.getDataInputStream(); - } - return is; - } catch (S3ServiceException e) { - throw new FileSystemException(e); - } - } - - // Utility methods - - /** - * Make an absolute path out of an S3 (Jets3t path) which does not contain - * a slash. If the key is "SomeDirectory/afile.txt" it will become - * "/SomeDirectory/afile.txt". All files will be handled with their full - * key name so this cannot be confused with relative file names which are - * handled by commons-vfs. - * - * @param key the objects key - * @return the absolute path name - */ - private String makeAbsolutePath(String key) { - return "/" + key; - } - - /** - * Create an S3 path (the key) from a commons-vfs path. This simply - * strips the slash from the beginning if it exists. We assume that the - * path is always absolute, so a missing slash at the beginning is - * simply ignored. - * - * @param path the absolute file path (with or without heading slash) - * @return the S3 object key - */ - private String makeS3Path(String path) { - if ("".equals(path)) { - return path; - } else { - return path.substring(1); - } - } -} blob - 6594366a8e657a91f2c102756d9a35b42e138610 (mode 644) blob + /dev/null --- src/main/java/com/thinkberg/moxo/s3/S3VfsRoot.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright 2007 Matthias L. Jugel. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.thinkberg.moxo.s3; - -import org.apache.commons.vfs.FileSystemException; - -/** - * Virtual Root of an S3 file system. Implement this interface to be able to get - * objects from Amazon S3. The root should be bound to a bucket that contains the - * virtual file system. All calls to S3VfsRoot#getObject should return a virtual - * object within this file system. - * - * @author Matthias L. Jugel - */ -public interface S3VfsRoot { - /** - * Get a file object relative to this S3 root. A roo - * - * @param path the path of the file (the absolute path) - * @return the virtual S3 file object - * @throws FileSystemException if the object cannot be accessed - */ - public S3VfsObject getObject(String path) throws FileSystemException; -} blob - 435dc12b13ce00f347f7278cb86e82f7d44dc975 (mode 644) blob + /dev/null --- src/main/java/com/thinkberg/moxo/s3/S3VfsRootImpl.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright 2007 Matthias L. Jugel. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.thinkberg.moxo.s3; - -import org.apache.commons.vfs.FileSystemException; -import org.jets3t.service.S3Service; -import org.jets3t.service.model.S3Bucket; - -/** - * Virtual Root of an S3 file system using the Jets3t library. The root - * stores the service and bucket to create file objects found in the - * bucket. - * - * @author Matthias L. Jugel - */ -public class S3VfsRootImpl implements S3VfsRoot { - private final S3Service service; - private final S3Bucket bucket; - - /** - * Create a new bucket file system root. - * - * @param service the S3 service to use - * @param bucket the S3 bucket this root is bound to - */ - public S3VfsRootImpl(S3Service service, S3Bucket bucket) { - this.service = service; - this.bucket = bucket; - } - - /** - * Create a new S3VfsObject using the given path. The object may not exist - * in the bucket and will then return an object that has an imaginary file type. - * - * @param path the absolute path to the file - * @return the virtual S3 object representing the file - * @throws FileSystemException if the file cannot be accessed - * @see com.thinkberg.moxo.s3.S3VfsObject - */ - public S3VfsObject getObject(String path) throws FileSystemException { - return new S3VfsObjectImpl(service, bucket, path); - } -} blob - 7e14e4183a36cc376545891551023afd1a7e67ba (mode 644) blob + /dev/null --- src/main/java/com/thinkberg/moxo/vfs/S3FileObject.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright 2007 Matthias L. Jugel. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.thinkberg.moxo.vfs; - -import com.thinkberg.moxo.s3.S3VfsObject; -import org.apache.commons.vfs.FileName; -import org.apache.commons.vfs.FileType; -import org.apache.commons.vfs.provider.AbstractFileObject; - -import java.io.InputStream; - -/** - * A VFS wrapper for the S3 file object. All requests are proxied through to the actual - * data object that contains an implementation of the virtual S3 Object. - * - * @author Matthias L. Jugel - * @see com.thinkberg.moxo.s3.S3VfsObject - */ -public class S3FileObject extends AbstractFileObject { - - private final S3VfsObject s3Object; - - /** - * Create a new S3 file object with the given virtual S3 object as data backend. - * - * @param fileName the file name of the current file in the virtual filesystem - * @param fileSystem the filesystem used (@see S3FileSystem) - * @param s3Object the actual data object - */ - @SuppressWarnings({"WeakerAccess"}) - protected S3FileObject(FileName fileName, S3FileSystem fileSystem, S3VfsObject s3Object) { - super(fileName, fileSystem); - this.s3Object = s3Object; - } - - protected FileType doGetType() throws Exception { - return s3Object.getType(); - } - - protected String[] doListChildren() throws Exception { - return s3Object.getChildren(); - } - - protected long doGetContentSize() throws Exception { - return s3Object.getContentLength(); - } - - - protected long doGetLastModifiedTime() throws Exception { - return s3Object.getLastModified(); - } - - protected InputStream doGetInputStream() throws Exception { - return s3Object.getInputStream(); - } - -} blob - c0fa0a50a3b224fdf9528bfdc917c8ad756523de blob + 1f34a86342e7cb4329618df20eb9bd5c024e96ca --- src/main/java/com/thinkberg/moxo/vfs/S3FileProvider.java +++ src/main/java/com/thinkberg/moxo/vfs/S3FileProvider.java @@ -16,8 +16,6 @@ package com.thinkberg.moxo.vfs; -import com.thinkberg.moxo.s3.S3Connector; -import com.thinkberg.moxo.s3.S3VfsRoot; import org.apache.commons.vfs.Capability; import org.apache.commons.vfs.FileName; import org.apache.commons.vfs.FileSystem; @@ -30,31 +28,29 @@ import java.util.Collection; import java.util.Collections; /** + * An S3 file provider. Create an S3 file system out of an S3 file name. + * Also defines the capabilities of the file system. + * * @author Matthias L. Jugel */ public class S3FileProvider extends AbstractOriginatingFileProvider { public final static Collection capabilities = Collections.unmodifiableCollection(Arrays.asList( -/* Capability.CREATE, Capability.DELETE, Capability.RENAME, -*/ -Capability.GET_TYPE, -Capability.GET_LAST_MODIFIED, -/* + Capability.GET_TYPE, + Capability.GET_LAST_MODIFIED, Capability.SET_LAST_MODIFIED_FILE, Capability.SET_LAST_MODIFIED_FOLDER, -*/ -Capability.LIST_CHILDREN, -Capability.READ_CONTENT, -Capability.URI/*, - + Capability.LIST_CHILDREN, + Capability.READ_CONTENT, + Capability.URI, Capability.WRITE_CONTENT, - Capability.APPEND_CONTENT, + Capability.APPEND_CONTENT/*, Capability.RANDOM_ACCESS_READ, - Capability.RANDOM_ACCESS_WRITE -*/ + Capability.RANDOM_ACCESS_WRITE*/ + )); @@ -63,13 +59,23 @@ Capability.URI/*, setFileNameParser(S3FileNameParser.getInstance()); } + /** + * Create a file system with the S3 root provided. + * + * @param fileName the S3 file name that defines the root (bucket) + * @param fileSystemOptions file system options + * @return an S3 file system + * @throws FileSystemException if te file system cannot be created + */ protected FileSystem doCreateFileSystem(FileName fileName, FileSystemOptions fileSystemOptions) throws FileSystemException { - S3FileName s3FileName = (S3FileName) fileName; - String s3BucketId = s3FileName.getRootFile(); - S3VfsRoot s3VfsRoot = S3Connector.getInstance().getRoot(s3BucketId); - return new S3FileSystem(s3FileName, fileSystemOptions, s3VfsRoot); + return S3FileSystemFactory.getFileSystem((S3FileName) fileName, fileSystemOptions); } + /** + * Get the capabilities of the file system provider. + * + * @return the file system capabilities + */ public Collection getCapabilities() { return capabilities; } blob - /dev/null blob + 27fc92eae78c75b11491a79b35cad2fdb8444ed9 (mode 644) --- /dev/null +++ src/main/java/com/thinkberg/moxo/vfs/S3FileSystemFactory.java @@ -0,0 +1,32 @@ +/* + * Copyright 2007 Matthias L. Jugel. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.thinkberg.moxo.vfs; + +import com.thinkberg.moxo.vfs.jets3t.Jets3tFileSystem; +import org.apache.commons.vfs.FileSystem; +import org.apache.commons.vfs.FileSystemException; +import org.apache.commons.vfs.FileSystemOptions; + +/** + * @author Matthias L. Jugel + */ +public class S3FileSystemFactory { + public static FileSystem getFileSystem(S3FileName fileName, FileSystemOptions fileSystemOptions) throws FileSystemException { + // TODO load dynamically + return new Jets3tFileSystem(fileName, fileSystemOptions); + } +} blob - b93b1b2b2e973615eca2965d99c3ab752bfd73c3 (mode 644) blob + /dev/null --- src/main/java/com/thinkberg/moxo/vfs/S3FileSystem.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright 2007 Matthias L. Jugel. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.thinkberg.moxo.vfs; - -import com.thinkberg.moxo.s3.S3VfsRoot; -import org.apache.commons.vfs.FileName; -import org.apache.commons.vfs.FileObject; -import org.apache.commons.vfs.FileSystem; -import org.apache.commons.vfs.FileSystemOptions; -import org.apache.commons.vfs.provider.AbstractFileSystem; - -import java.util.Collection; - - -/** - * @author Matthias L. Jugel - */ -public class S3FileSystem extends AbstractFileSystem implements FileSystem { - private final S3VfsRoot s3VfsRoot; - - @SuppressWarnings({"WeakerAccess"}) - protected S3FileSystem(S3FileName fileName, FileSystemOptions fileSystemOptions, S3VfsRoot s3VfsRoot) { - super(fileName, null, fileSystemOptions); - this.s3VfsRoot = s3VfsRoot; - } - - @SuppressWarnings({"unchecked"}) - protected void addCapabilities(Collection caps) { - caps.addAll(S3FileProvider.capabilities); - } - - protected FileObject createFile(FileName fileName) throws Exception { - return new S3FileObject(fileName, this, s3VfsRoot.getObject(fileName.getPathDecoded())); - } - -} blob - /dev/null blob + 8352c585cb21de0f0c73b65170f9e14120ae23b0 (mode 644) --- /dev/null +++ src/main/java/com/thinkberg/moxo/vfs/jets3t/Jets3tConnector.java @@ -0,0 +1,77 @@ +/* + * Copyright 2007 Matthias L. Jugel. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.thinkberg.moxo.vfs.jets3t; + +import org.jets3t.service.Jets3tProperties; +import org.jets3t.service.S3Service; +import org.jets3t.service.S3ServiceException; +import org.jets3t.service.impl.rest.httpclient.RestS3Service; +import org.jets3t.service.security.AWSCredentials; + +/** + * @author Matthias L. Jugel + */ +public class Jets3tConnector { + private static final String APPLICATION_DESCRIPTION = "S3 VFS Connector/1.0"; + + private static Jets3tConnector instance; + + /** + * Get an instance of the Jets3tConnector which is initialized and authenticated to the + * Amazon S3 Service. + * + * @return a Jets3t S3 connector + * @throws org.jets3t.service.S3ServiceException + * if connection or authentication fails + */ + public static Jets3tConnector getInstance() throws S3ServiceException { + if (null == instance) { + instance = new Jets3tConnector(); + } + return instance; + } + + private S3Service service; + + /** + * Initialize Amazon S3. + * + * @throws org.jets3t.service.S3ServiceException + * if the service cannot be accessed + */ + private Jets3tConnector() throws S3ServiceException { + System.err.print("Authenticated to Amazon S3: "); + String propertiesFileName = System.getProperty("moxo.properties", "moxo.properties"); + Jets3tProperties properties = Jets3tProperties.getInstance(propertiesFileName); + + if (!properties.isLoaded()) { + throw new S3ServiceException("can't find S3 configuration: " + propertiesFileName); + } + + AWSCredentials awsCredentials = new AWSCredentials( + properties.getStringProperty("accesskey", null), + properties.getStringProperty("secretkey", null)); + + + service = new RestS3Service(awsCredentials, APPLICATION_DESCRIPTION, null); + System.err.println("OK"); + } + + public S3Service getService() { + return service; + } +} blob - /dev/null blob + f5fd3a29646ca8af31028d47e7e86e0dec740e6d (mode 644) --- /dev/null +++ src/main/java/com/thinkberg/moxo/vfs/jets3t/Jets3tFileObject.java @@ -0,0 +1,193 @@ +/* + * Copyright 2007 Matthias L. Jugel. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.thinkberg.moxo.vfs.jets3t; + +import org.apache.commons.vfs.FileName; +import org.apache.commons.vfs.FileObject; +import org.apache.commons.vfs.FileSystemException; +import org.apache.commons.vfs.FileType; +import org.apache.commons.vfs.provider.AbstractFileObject; +import org.jets3t.service.S3Service; +import org.jets3t.service.S3ServiceException; +import org.jets3t.service.model.S3Bucket; +import org.jets3t.service.model.S3Object; +import org.jets3t.service.utils.Mimetypes; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.RandomAccessFile; +import java.nio.channels.Channels; +import java.nio.channels.FileChannel; +import java.nio.channels.ReadableByteChannel; +import java.util.Date; + +/** + * Implementation of the virtual S3 file system object using the Jets3t library. + * + * @author Matthias L. Jugel + */ +public class Jets3tFileObject extends AbstractFileObject { + private final S3Service service; + private final S3Bucket bucket; + + private boolean attached = false; + private boolean changed = false; + private S3Object object; + private File cacheFile; + + public Jets3tFileObject(FileName fileName, + Jets3tFileSystem fileSystem, + S3Service service, S3Bucket bucket) + throws FileSystemException { + super(fileName, fileSystem); + this.service = service; + this.bucket = bucket; + } + + protected void doAttach() throws Exception { + if (!attached) { + try { + object = service.getObject(bucket, getS3Key()); + System.err.println("Attached file to S3 Object: " + object); + InputStream is = object.getDataInputStream(); + if (object.getContentLength() > 0) { + ReadableByteChannel rbc = Channels.newChannel(is); + FileChannel cacheFc = getCacheFileChannel(); + cacheFc.transferFrom(rbc, 0, object.getContentLength()); + cacheFc.close(); + rbc.close(); + } else { + is.close(); + } + } catch (S3ServiceException e) { + object = new S3Object(bucket, getS3Key()); + object.setLastModifiedDate(new Date()); + System.err.println("Attached file to new S3 Object: " + object); + } + attached = true; + } + } + + protected void doDetach() throws Exception { + // TODO do not send immediately but put in some kind of upload queue + if (attached && changed) { + System.err.println("Detaching changed object: " + object); + if (cacheFile != null) { + FileChannel cacheFc = getCacheFileChannel(); + object.setContentLength(cacheFc.size()); + object.setDataInputStream(getInputStream()); + } + System.err.println(object); + service.putObject(bucket, object); + attached = false; + } + } + + protected void doDelete() throws Exception { + service.deleteObject(bucket, object.getKey()); + } + + protected void doRename(FileObject newfile) throws Exception { + super.doRename(newfile); + } + + protected void doCreateFolder() throws Exception { + if (!Mimetypes.MIMETYPE_JETS3T_DIRECTORY.equals(object.getContentType())) { + object.setContentType(Mimetypes.MIMETYPE_JETS3T_DIRECTORY); + service.putObject(bucket, object); + changed = false; + } + } + + protected long doGetLastModifiedTime() throws Exception { + return object.getLastModifiedDate().getTime(); + } + + protected void doSetLastModifiedTime(final long modtime) throws Exception { + changed = true; + object.setLastModifiedDate(new Date(modtime)); + } + + protected InputStream doGetInputStream() throws Exception { + return Channels.newInputStream(getCacheFileChannel()); + } + + protected OutputStream doGetOutputStream(boolean bAppend) throws Exception { + changed = true; + return Channels.newOutputStream(getCacheFileChannel()); + } + + protected FileType doGetType() throws Exception { + if (null == object.getContentType()) { + return FileType.IMAGINARY; + } + + String contentType = object.getContentType(); + if ("".equals(object.getKey()) || Mimetypes.MIMETYPE_JETS3T_DIRECTORY.equals(contentType)) { + return FileType.FOLDER; + } + + return FileType.FILE; + } + + protected String[] doListChildren() throws Exception { + String path = object.getKey(); + // make sure we add a '/' slash at the end to find children + if (!"".equals(path)) { + path = path + "/"; + } + + S3Object[] children = service.listObjects(bucket, path, "/"); + String[] childrenNames = new String[children.length]; + for (int i = 0; i < children.length; i++) { + if (!children[i].getKey().equals(path)) { + // strip path from name (leave only base name) + childrenNames[i] = children[i].getKey().replaceAll("[^/]*//*", ""); + } + } + return childrenNames; + } + + protected long doGetContentSize() throws Exception { + return object.getContentLength(); + } + + // Utility methods + /** + * Create an S3 key from a commons-vfs path. This simply + * strips the slash from the beginning if it exists. + * + * @return the S3 object key + */ + private String getS3Key() { + String path = getName().getPath(); + if ("".equals(path)) { + return path; + } else { + return path.substring(1); + } + } + + private FileChannel getCacheFileChannel() throws IOException { + if (cacheFile == null) { + cacheFile = File.createTempFile("moxo.", ".s3"); + } + return new RandomAccessFile(cacheFile, "rw").getChannel(); + } +} blob - /dev/null blob + e78aada5f0071545a49f52fb1f86aa003f038a87 (mode 644) --- /dev/null +++ src/main/java/com/thinkberg/moxo/vfs/jets3t/Jets3tFileSystem.java @@ -0,0 +1,64 @@ +/* + * Copyright 2007 Matthias L. Jugel. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.thinkberg.moxo.vfs.jets3t; + +import com.thinkberg.moxo.vfs.S3FileName; +import com.thinkberg.moxo.vfs.S3FileProvider; +import org.apache.commons.vfs.FileName; +import org.apache.commons.vfs.FileObject; +import org.apache.commons.vfs.FileSystemException; +import org.apache.commons.vfs.FileSystemOptions; +import org.apache.commons.vfs.provider.AbstractFileSystem; +import org.jets3t.service.S3Service; +import org.jets3t.service.S3ServiceException; +import org.jets3t.service.model.S3Bucket; + +import java.util.Collection; + +/** + * An S3 file system. + * + * @author Matthias L. Jugel + */ +public class Jets3tFileSystem extends AbstractFileSystem { + private S3Service service; + private S3Bucket bucket; + + public Jets3tFileSystem(S3FileName fileName, FileSystemOptions fileSystemOptions) throws FileSystemException { + super(fileName, null, fileSystemOptions); + String bucketId = fileName.getRootFile(); + try { + service = Jets3tConnector.getInstance().getService(); + bucket = new S3Bucket(bucketId); + if (!service.isBucketAccessible(bucketId)) { + bucket = service.createBucket(bucketId); + } + System.err.println("Created new S3 FileSystem: " + bucketId); + } catch (S3ServiceException e) { + throw new FileSystemException(e); + } + } + + @SuppressWarnings({"unchecked"}) + protected void addCapabilities(Collection caps) { + caps.addAll(S3FileProvider.capabilities); + } + + protected FileObject createFile(FileName fileName) throws Exception { + return new Jets3tFileObject(fileName, this, service, bucket); + } +} blob - /dev/null blob + ee8611fc7310c54462bf8672a9ff9845cbab912a (mode 644) --- /dev/null +++ src/main/resources/commons-logging.properties @@ -0,0 +1,17 @@ +# +# Copyright 2007 Matthias L. Jugel. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +#org.apache.commons.logging.Log=org.apache.commons.logging.impl.Jdk14Logger +org.apache.commons.logging.Log=org.apache.commons.logging.impl.SimpleLog \ No newline at end of file blob - 063bbb02000bd53c285e6e92b4a9669d148ed99d blob + cf3d423ee160e82b17ab56a44e55faddac0f384d --- src/main/resources/moxo.template.properties +++ src/main/resources/moxo.template.properties @@ -35,4 +35,4 @@ acl=PRIVATE password= # the bucket that contains the file system -bucket= \ No newline at end of file +bucket= \ No newline at end of file blob - /dev/null blob + 253190f1ea5939f131d17688ef3a6e947791f8eb (mode 644) --- /dev/null +++ src/main/resources/simplelog.properties @@ -0,0 +1,17 @@ +# +# Copyright 2007 Matthias L. Jugel. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +org.apache.commons.logging.simplelog.defaultlog=info \ No newline at end of file blob - d30eb9df05dee7f0339cc282835d00c601283bd2 blob + 75d3440d7341f1159ea710440ef5a4d9ab1c7e4a --- src/test/java/com/thinkberg/moxo/MoxoTest.java +++ src/test/java/com/thinkberg/moxo/MoxoTest.java @@ -2,7 +2,6 @@ package com.thinkberg.moxo; import com.thinkberg.moxo.dav.DavLockManagerTest; import com.thinkberg.moxo.dav.DavResourceTest; -import com.thinkberg.moxo.s3.S3WrapperTest; import com.thinkberg.moxo.vfs.S3FileNameTest; import com.thinkberg.moxo.vfs.S3FileProviderTest; import junit.framework.Test; @@ -41,8 +40,6 @@ public class MoxoTest extends TestCase { String bucketId = properties.getStringProperty("bucket", null); if (null != bucketId) { - s.addTestSuite(S3WrapperTest.class); - s.addTestSuite(S3FileNameTest.class); s.addTestSuite(S3FileProviderTest.class); } blob - 4daf32ab4820a8f9afbb63a5bb033383d7855fbe (mode 644) blob + /dev/null --- src/test/java/com/thinkberg/moxo/s3/S3WrapperTest.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Copyright 2007 Matthias L. Jugel. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.thinkberg.moxo.s3; - -import junit.framework.TestCase; -import org.apache.commons.vfs.FileSystemException; -import org.apache.commons.vfs.FileType; -import org.jets3t.service.Jets3tProperties; - -import java.io.InputStream; - -import com.thinkberg.moxo.S3TestCase; - -/** - * @author Matthias L. Jugel - */ -public class S3WrapperTest extends S3TestCase { - public void testCreateConnection() throws FileSystemException { - assertNotNull(S3Connector.getInstance()); - } - - public void testGetS3BucketRoot() throws FileSystemException { - assertNotNull(S3Connector.getInstance().getRoot(BUCKETID)); - } - - public void testGetS3BucketRootMissing() throws FileSystemException { - try { - assertNull(S3Connector.getInstance().getRoot(BUCKETID+".NOTEXISTING")); - } catch (FileSystemException e) { - assertNotNull(e); - } - } - - public void testGetRootFolder() throws FileSystemException { - S3VfsRoot root = S3Connector.getInstance().getRoot(BUCKETID); - S3VfsObject slashObject = root.getObject("/"); - assertEquals("/", slashObject.getName()); - } - - public void testRootFolderTypeIsCorrect() throws FileSystemException { - S3VfsRoot root = S3Connector.getInstance().getRoot(BUCKETID); - S3VfsObject slashObject = root.getObject("/"); - assertEquals(FileType.FOLDER, slashObject.getType()); - } - - public void testRootFolderListing() throws FileSystemException { - S3VfsRoot root = S3Connector.getInstance().getRoot(BUCKETID); - S3VfsObject slashObject = root.getObject("/"); - assertEquals(1, slashObject.getChildren().length); - } - - public void testFolderTypeIsCorrect() throws FileSystemException { - S3VfsRoot root = S3Connector.getInstance().getRoot(BUCKETID); - S3VfsObject folderObject = root.getObject("/Sites"); - assertEquals(FileType.FOLDER, folderObject.getType()); - } - - public void testFileTypeIsCorrect() throws FileSystemException { - S3VfsRoot root = S3Connector.getInstance().getRoot(BUCKETID); - S3VfsObject fileObject = root.getObject("/Sites/Sites/index.html"); - assertEquals(FileType.FILE, fileObject.getType()); - } - - public void testFolderSize() throws FileSystemException { - S3VfsRoot root = S3Connector.getInstance().getRoot(BUCKETID); - S3VfsObject folderObject = root.getObject("/Sites/Sites/images"); - assertEquals(3, folderObject.getChildren().length); - } - - public void testGetFile() throws FileSystemException { - S3VfsRoot root = S3Connector.getInstance().getRoot(BUCKETID); - S3VfsObject fileObject = root.getObject("/Sites/Sites/images/macosxlogo.gif"); - assertNotNull(fileObject.getInputStream()); - } - - public void testGetMissingFileIsImaginary() throws FileSystemException { - S3VfsRoot root = S3Connector.getInstance().getRoot(BUCKETID); - S3VfsObject fileObject = root.getObject("/notexisting.txt"); - assertEquals(FileType.IMAGINARY, fileObject.getType()); - } -} blob - 6fddf9948fa98da70f3c6f8d3e6268191d6ee83d blob + 9f9b360ec733337b7a07da2071e596cd97c791fb --- src/test/java/com/thinkberg/moxo/vfs/S3FileProviderTest.java +++ src/test/java/com/thinkberg/moxo/vfs/S3FileProviderTest.java @@ -16,24 +16,24 @@ package com.thinkberg.moxo.vfs; +import com.thinkberg.moxo.S3TestCase; import org.apache.commons.vfs.FileObject; +import org.apache.commons.vfs.FileSelectInfo; +import org.apache.commons.vfs.FileSelector; +import org.apache.commons.vfs.FileSystem; import org.apache.commons.vfs.FileSystemException; -import org.apache.commons.vfs.FileSystemOptions; -import org.apache.commons.vfs.FileSystemManager; -import org.apache.commons.vfs.VFS; import org.apache.commons.vfs.FileType; -import org.apache.commons.vfs.impl.DefaultFileSystemManager; -import org.jets3t.service.Jets3tProperties; -import junit.framework.TestCase; -import com.thinkberg.moxo.S3TestCase; +import org.apache.commons.vfs.VFS; +import java.io.IOException; + /** * @author Matthias L. Jugel */ public class S3FileProviderTest extends S3TestCase { public void testDoCreateFileSystem() throws FileSystemException { FileObject object = VFS.getManager().resolveFile(ROOT); - assertEquals(BUCKETID, ((S3FileName)object.getName()).getRootFile()); + assertEquals(BUCKETID, ((S3FileName) object.getName()).getRootFile()); } public void testRootDirectoryIsFolder() throws FileSystemException { @@ -49,6 +49,9 @@ public class S3FileProviderTest extends S3TestCase { public void testGetDirectoryListing() throws FileSystemException { FileObject object = VFS.getManager().resolveFile(ROOT + "/Sites/Sites/images"); FileObject[] files = object.findFiles(new DepthFileSelector(1)); + for (FileObject file : files) { + System.out.println("Found file: " + file.getName().getPath()); + } assertEquals(4, files.length); } @@ -56,4 +59,39 @@ public class S3FileProviderTest extends S3TestCase { FileObject object = VFS.getManager().resolveFile(ROOT + "/nonexisting.txt"); assertFalse(object.exists()); } + + public void testDeleteFile() throws FileSystemException { + FileObject object = VFS.getManager().resolveFile(ROOT + "/newfile.txt"); + object.delete(); + } + + public void testDeleteFolder() throws FileSystemException { + FileObject object = VFS.getManager().resolveFile(ROOT + "/newfolder"); + object.delete(new FileSelector() { + + public boolean includeFile(FileSelectInfo fileInfo) throws Exception { + return true; + } + + public boolean traverseDescendents(FileSelectInfo fileInfo) throws Exception { + return true; + } + }); + } + + public void testCreateFolder() throws FileSystemException { + FileObject object = VFS.getManager().resolveFile(ROOT + "/newfolder"); + object.createFolder(); + } + + public void testCreateFile() throws IOException { + FileObject object = VFS.getManager().resolveFile(ROOT + "/newfile.txt"); + object.getContent().getOutputStream().write(0xfc); + object.close(); + } + + public void testCloseFileSystem() throws FileSystemException { + FileSystem fs = VFS.getManager().resolveFile(ROOT).getFileSystem(); + VFS.getManager().closeFileSystem(fs); + } }