Index: hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/kfs/package.html =================================================================== --- hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/kfs/package.html (revision 1395699) +++ hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/kfs/package.html (working copy) @@ -1,98 +0,0 @@ - - - - -
- --<property> - <name>fs.kfs.impl</name> - <value>org.apache.hadoop.fs.kfs.KosmosFileSystem</value> - <description>The FileSystem for kfs: uris.</description> -</property> -- -
-<property> - <name>fs.default.name</name> - <value>kfs://<server:port></value> -</property> - -<property> - <name>fs.kfs.metaServerHost</name> - <value><server></value> - <description>The location of the KFS meta server.</description> -</property> - -<property> - <name>fs.kfs.metaServerPort</name> - <value><port></value> - <description>The location of the meta server's port.</description> -</property> - --
-export LD_LIBRARY_PATH=<path> -- - -
kfs
- */
- @Override
- public String getScheme() {
- return "kfs";
- }
-
- @Override
- public URI getUri() {
- return uri;
- }
-
- @Override
- public void initialize(URI uri, Configuration conf) throws IOException {
- super.initialize(uri, conf);
- try {
- if (kfsImpl == null) {
- if (uri.getHost() == null) {
- kfsImpl = new KFSImpl(conf.get("fs.kfs.metaServerHost", ""),
- conf.getInt("fs.kfs.metaServerPort", -1),
- statistics);
- } else {
- kfsImpl = new KFSImpl(uri.getHost(), uri.getPort(), statistics);
- }
- }
-
- this.localFs = FileSystem.getLocal(conf);
- this.uri = URI.create(uri.getScheme() + "://" + uri.getAuthority());
- this.workingDir = new Path("/user", System.getProperty("user.name")
- ).makeQualified(this);
- setConf(conf);
-
- } catch (Exception e) {
- e.printStackTrace();
- System.out.println("Unable to initialize KFS");
- System.exit(-1);
- }
- }
-
- @Override
- public Path getWorkingDirectory() {
- return workingDir;
- }
-
- @Override
- public void setWorkingDirectory(Path dir) {
- workingDir = makeAbsolute(dir);
- }
-
- private Path makeAbsolute(Path path) {
- if (path.isAbsolute()) {
- return path;
- }
- return new Path(workingDir, path);
- }
-
- @Override
- public boolean mkdirs(Path path, FsPermission permission
- ) throws IOException {
- Path absolute = makeAbsolute(path);
- String srep = absolute.toUri().getPath();
-
- int res;
-
- // System.out.println("Calling mkdirs on: " + srep);
-
- res = kfsImpl.mkdirs(srep);
-
- return res == 0;
- }
-
- @Override
- public boolean isDirectory(Path path) throws IOException {
- Path absolute = makeAbsolute(path);
- String srep = absolute.toUri().getPath();
-
- // System.out.println("Calling isdir on: " + srep);
-
- return kfsImpl.isDirectory(srep);
- }
-
- @Override
- public boolean isFile(Path path) throws IOException {
- Path absolute = makeAbsolute(path);
- String srep = absolute.toUri().getPath();
- return kfsImpl.isFile(srep);
- }
-
- @Override
- public FileStatus[] listStatus(Path path) throws IOException {
- Path absolute = makeAbsolute(path);
- String srep = absolute.toUri().getPath();
-
- if(!kfsImpl.exists(srep))
- throw new FileNotFoundException("File " + path + " does not exist.");
-
- if (kfsImpl.isFile(srep))
- return new FileStatus[] { getFileStatus(path) } ;
-
- return kfsImpl.readdirplus(absolute);
- }
-
- @Override
- public FileStatus getFileStatus(Path path) throws IOException {
- Path absolute = makeAbsolute(path);
- String srep = absolute.toUri().getPath();
- if (!kfsImpl.exists(srep)) {
- throw new FileNotFoundException("File " + path + " does not exist.");
- }
- if (kfsImpl.isDirectory(srep)) {
- // System.out.println("Status of path: " + path + " is dir");
- return new FileStatus(0, true, 1, 0, kfsImpl.getModificationTime(srep),
- path.makeQualified(this));
- } else {
- // System.out.println("Status of path: " + path + " is file");
- return new FileStatus(kfsImpl.filesize(srep), false,
- kfsImpl.getReplication(srep),
- getDefaultBlockSize(),
- kfsImpl.getModificationTime(srep),
- path.makeQualified(this));
- }
- }
-
- @Override
- public FSDataOutputStream append(Path f, int bufferSize,
- Progressable progress) throws IOException {
- Path parent = f.getParent();
- if (parent != null && !mkdirs(parent)) {
- throw new IOException("Mkdirs failed to create " + parent);
- }
-
- Path absolute = makeAbsolute(f);
- String srep = absolute.toUri().getPath();
-
- return kfsImpl.append(srep, bufferSize, progress);
- }
-
- @Override
- public FSDataOutputStream create(Path file, FsPermission permission,
- boolean overwrite, int bufferSize,
- short replication, long blockSize, Progressable progress)
- throws IOException {
-
- if (exists(file)) {
- if (overwrite) {
- delete(file, true);
- } else {
- throw new IOException("File already exists: " + file);
- }
- }
-
- Path parent = file.getParent();
- if (parent != null && !mkdirs(parent)) {
- throw new IOException("Mkdirs failed to create " + parent);
- }
-
- Path absolute = makeAbsolute(file);
- String srep = absolute.toUri().getPath();
-
- return kfsImpl.create(srep, replication, bufferSize, progress);
- }
-
- @Override
- public FSDataInputStream open(Path path, int bufferSize) throws IOException {
- if (!exists(path))
- throw new IOException("File does not exist: " + path);
-
- Path absolute = makeAbsolute(path);
- String srep = absolute.toUri().getPath();
-
- return kfsImpl.open(srep, bufferSize);
- }
-
- @Override
- public boolean rename(Path src, Path dst) throws IOException {
- Path absoluteS = makeAbsolute(src);
- String srepS = absoluteS.toUri().getPath();
- Path absoluteD = makeAbsolute(dst);
- String srepD = absoluteD.toUri().getPath();
-
- // System.out.println("Calling rename on: " + srepS + " -> " + srepD);
-
- return kfsImpl.rename(srepS, srepD) == 0;
- }
-
- // recursively delete the directory and its contents
- @Override
- public boolean delete(Path path, boolean recursive) throws IOException {
- Path absolute = makeAbsolute(path);
- String srep = absolute.toUri().getPath();
- if (kfsImpl.isFile(srep))
- return kfsImpl.remove(srep) == 0;
-
- FileStatus[] dirEntries = listStatus(absolute);
- if (!recursive && (dirEntries.length != 0)) {
- throw new IOException("Directory " + path.toString() +
- " is not empty.");
- }
-
- for (int i = 0; i < dirEntries.length; i++) {
- delete(new Path(absolute, dirEntries[i].getPath()), recursive);
- }
- return kfsImpl.rmdir(srep) == 0;
- }
-
- @Override
- public short getDefaultReplication() {
- return 3;
- }
-
- @Override
- public boolean setReplication(Path path, short replication)
- throws IOException {
-
- Path absolute = makeAbsolute(path);
- String srep = absolute.toUri().getPath();
-
- int res = kfsImpl.setReplication(srep, replication);
- return res >= 0;
- }
-
- // 64MB is the KFS block size
-
- @Override
- public long getDefaultBlockSize() {
- return 1 << 26;
- }
-
- @Deprecated
- public void lock(Path path, boolean shared) throws IOException {
-
- }
-
- @Deprecated
- public void release(Path path) throws IOException {
-
- }
-
- /**
- * Return null if the file doesn't exist; otherwise, get the
- * locations of the various chunks of the file file from KFS.
- */
- @Override
- public BlockLocation[] getFileBlockLocations(FileStatus file, long start,
- long len) throws IOException {
-
- if (file == null) {
- return null;
- }
- String srep = makeAbsolute(file.getPath()).toUri().getPath();
- String[][] hints = kfsImpl.getDataLocation(srep, start, len);
- if (hints == null) {
- return null;
- }
- BlockLocation[] result = new BlockLocation[hints.length];
- long blockSize = getDefaultBlockSize();
- long length = len;
- long blockStart = start;
- for(int i=0; i < result.length; ++i) {
- result[i] = new BlockLocation(null, hints[i], blockStart,
- length < blockSize ? length : blockSize);
- blockStart += blockSize;
- length -= blockSize;
- }
- return result;
- }
-
- @Override
- public void copyFromLocalFile(boolean delSrc, Path src, Path dst) throws IOException {
- FileUtil.copy(localFs, src, this, dst, delSrc, getConf());
- }
-
- @Override
- public void copyToLocalFile(boolean delSrc, Path src, Path dst) throws IOException {
- FileUtil.copy(this, src, localFs, dst, delSrc, getConf());
- }
-
- @Override
- public Path startLocalOutput(Path fsOutputFile, Path tmpLocalFile)
- throws IOException {
- return tmpLocalFile;
- }
-
- @Override
- public void completeLocalOutput(Path fsOutputFile, Path tmpLocalFile)
- throws IOException {
- moveFromLocalFile(tmpLocalFile, fsOutputFile);
- }
-}
Index: hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/kfs/IFSImpl.java
===================================================================
--- hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/kfs/IFSImpl.java (revision 1395699)
+++ hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/kfs/IFSImpl.java (working copy)
@@ -1,59 +0,0 @@
-/**
- *
- * Licensed under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
- * implied. See the License for the specific language governing
- * permissions and limitations under the License.
- *
- *
- * We need to provide the ability to the code in fs/kfs without really
- * having a KFS deployment. In particular, the glue code that wraps
- * around calls to KfsAccess object. This is accomplished by defining a
- * filesystem implementation interface:
- * -- for testing purposes, a dummy implementation of this interface
- * will suffice; as long as the dummy implementation is close enough
- * to doing what KFS does, we are good.
- * -- for deployment purposes with KFS, this interface is
- * implemented by the KfsImpl object.
- */
-
-package org.apache.hadoop.fs.kfs;
-
-import java.io.*;
-
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.util.Progressable;
-
-interface IFSImpl {
- public boolean exists(String path) throws IOException;
- public boolean isDirectory(String path) throws IOException;
- public boolean isFile(String path) throws IOException;
- public String[] readdir(String path) throws IOException;
- public FileStatus[] readdirplus(Path path) throws IOException;
-
- public int mkdirs(String path) throws IOException;
- public int rename(String source, String dest) throws IOException;
-
- public int rmdir(String path) throws IOException;
- public int remove(String path) throws IOException;
- public long filesize(String path) throws IOException;
- public short getReplication(String path) throws IOException;
- public short setReplication(String path, short replication) throws IOException;
- public String[][] getDataLocation(String path, long start, long len) throws IOException;
-
- public long getModificationTime(String path) throws IOException;
- public FSDataOutputStream create(String path, short replication, int bufferSize, Progressable progress) throws IOException;
- public FSDataInputStream open(String path, int bufferSize) throws IOException;
- public FSDataOutputStream append(String path, int bufferSize, Progressable progress) throws IOException;
-
-};
Index: hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/kfs/KFSConfigKeys.java
===================================================================
--- hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/kfs/KFSConfigKeys.java (revision 1395699)
+++ hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/kfs/KFSConfigKeys.java (working copy)
@@ -1,47 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.kfs;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.fs.CommonConfigurationKeys;
-
-/**
- * This class contains constants for configuration keys used
- * in the kfs file system.
- *
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-public class KFSConfigKeys extends CommonConfigurationKeys {
- public static final String KFS_BLOCK_SIZE_KEY = "kfs.blocksize";
- public static final long KFS_BLOCK_SIZE_DEFAULT = 64*1024*1024;
- public static final String KFS_REPLICATION_KEY = "kfs.replication";
- public static final short KFS_REPLICATION_DEFAULT = 1;
- public static final String KFS_STREAM_BUFFER_SIZE_KEY =
- "kfs.stream-buffer-size";
- public static final int KFS_STREAM_BUFFER_SIZE_DEFAULT = 4096;
- public static final String KFS_BYTES_PER_CHECKSUM_KEY =
- "kfs.bytes-per-checksum";
- public static final int KFS_BYTES_PER_CHECKSUM_DEFAULT = 512;
- public static final String KFS_CLIENT_WRITE_PACKET_SIZE_KEY =
- "kfs.client-write-packet-size";
- public static final int KFS_CLIENT_WRITE_PACKET_SIZE_DEFAULT = 64*1024;
-}
-
Index: hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/kfs/KFSImpl.java
===================================================================
--- hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/kfs/KFSImpl.java (revision 1395699)
+++ hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/kfs/KFSImpl.java (working copy)
@@ -1,171 +0,0 @@
-/**
- *
- * Licensed under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
- * implied. See the License for the specific language governing
- * permissions and limitations under the License.
- *
- *
- * Provide the implementation of KFS which turn into calls to KfsAccess.
- */
-
-package org.apache.hadoop.fs.kfs;
-
-import java.io.*;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.Path;
-
-import org.kosmix.kosmosfs.access.KfsAccess;
-import org.kosmix.kosmosfs.access.KfsFileAttr;
-import org.apache.hadoop.util.Progressable;
-
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-class KFSImpl implements IFSImpl {
- private KfsAccess kfsAccess = null;
- private FileSystem.Statistics statistics;
-
- @Deprecated
- public KFSImpl(String metaServerHost, int metaServerPort
- ) throws IOException {
- this(metaServerHost, metaServerPort, null);
- }
-
- public KFSImpl(String metaServerHost, int metaServerPort,
- FileSystem.Statistics stats) throws IOException {
- kfsAccess = new KfsAccess(metaServerHost, metaServerPort);
- statistics = stats;
- }
-
- @Override
- public boolean exists(String path) throws IOException {
- return kfsAccess.kfs_exists(path);
- }
-
- @Override
- public boolean isDirectory(String path) throws IOException {
- return kfsAccess.kfs_isDirectory(path);
- }
-
- @Override
- public boolean isFile(String path) throws IOException {
- return kfsAccess.kfs_isFile(path);
- }
-
- @Override
- public String[] readdir(String path) throws IOException {
- return kfsAccess.kfs_readdir(path);
- }
-
- @Override
- public FileStatus[] readdirplus(Path path) throws IOException {
- String srep = path.toUri().getPath();
- KfsFileAttr[] fattr = kfsAccess.kfs_readdirplus(srep);
- if (fattr == null)
- return null;
- int numEntries = 0;
- for (int i = 0; i < fattr.length; i++) {
- if ((fattr[i].filename.compareTo(".") == 0) || (fattr[i].filename.compareTo("target/generated-sources") == 0))
- continue;
- numEntries++;
- }
- FileStatus[] fstatus = new FileStatus[numEntries];
- int j = 0;
- for (int i = 0; i < fattr.length; i++) {
- if ((fattr[i].filename.compareTo(".") == 0) || (fattr[i].filename.compareTo("target/generated-sources") == 0))
- continue;
- Path fn = new Path(path, fattr[i].filename);
-
- if (fattr[i].isDirectory)
- fstatus[j] = new FileStatus(0, true, 1, 0, fattr[i].modificationTime, fn);
- else
- fstatus[j] = new FileStatus(fattr[i].filesize, fattr[i].isDirectory,
- fattr[i].replication,
- (long)
- (1 << 26),
- fattr[i].modificationTime,
- fn);
-
- j++;
- }
- return fstatus;
- }
-
-
- @Override
- public int mkdirs(String path) throws IOException {
- return kfsAccess.kfs_mkdirs(path);
- }
-
- @Override
- public int rename(String source, String dest) throws IOException {
- return kfsAccess.kfs_rename(source, dest);
- }
-
- @Override
- public int rmdir(String path) throws IOException {
- return kfsAccess.kfs_rmdir(path);
- }
-
- @Override
- public int remove(String path) throws IOException {
- return kfsAccess.kfs_remove(path);
- }
-
- @Override
- public long filesize(String path) throws IOException {
- return kfsAccess.kfs_filesize(path);
- }
-
- @Override
- public short getReplication(String path) throws IOException {
- return kfsAccess.kfs_getReplication(path);
- }
-
- @Override
- public short setReplication(String path, short replication) throws IOException {
- return kfsAccess.kfs_setReplication(path, replication);
- }
-
- @Override
- public String[][] getDataLocation(String path, long start, long len) throws IOException {
- return kfsAccess.kfs_getDataLocation(path, start, len);
- }
-
- @Override
- public long getModificationTime(String path) throws IOException {
- return kfsAccess.kfs_getModificationTime(path);
- }
-
- @Override
- public FSDataInputStream open(String path, int bufferSize) throws IOException {
- return new FSDataInputStream(new KFSInputStream(kfsAccess, path,
- statistics));
- }
-
- @Override
- public FSDataOutputStream create(String path, short replication, int bufferSize, Progressable progress) throws IOException {
- return new FSDataOutputStream(new KFSOutputStream(kfsAccess, path, replication, false, progress),
- statistics);
- }
-
- @Override
- public FSDataOutputStream append(String path, int bufferSize, Progressable progress) throws IOException {
- // when opening for append, # of replicas is ignored
- return new FSDataOutputStream(new KFSOutputStream(kfsAccess, path, (short) 1, true, progress),
- statistics);
- }
-}
Index: hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem
===================================================================
--- hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem (revision 1395699)
+++ hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem (working copy)
@@ -17,6 +17,5 @@
org.apache.hadoop.fs.viewfs.ViewFileSystem
org.apache.hadoop.fs.s3.S3FileSystem
org.apache.hadoop.fs.s3native.NativeS3FileSystem
-org.apache.hadoop.fs.kfs.KosmosFileSystem
org.apache.hadoop.fs.ftp.FTPFileSystem
org.apache.hadoop.fs.HarFileSystem
Index: hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
===================================================================
--- hadoop-common-project/hadoop-common/src/main/resources/core-default.xml (revision 1395699)
+++ hadoop-common-project/hadoop-common/src/main/resources/core-default.xml (working copy)
@@ -774,42 +774,6 @@