diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..87ee19c --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +/accumulo-1.6.0-0616258.tar.gz +/accumulo-1.6.1-src.tar.gz +/accumulo-1.6.2-src.tar.gz +/accumulo-1.6.4-src.tar.gz +/accumulo-1.6.6-src.tar.gz diff --git a/ACCUMULO-3470.patch b/ACCUMULO-3470.patch new file mode 100644 index 0000000..b618479 --- /dev/null +++ b/ACCUMULO-3470.patch @@ -0,0 +1,1164 @@ +From b363e88772ccda2a3bdeddd9378c89266c386cd3 Mon Sep 17 00:00:00 2001 +From: Dave Marion +Date: Fri, 20 May 2016 12:35:55 -0400 +Subject: [PATCH 1/1] ACCUMULO-3470: Set commons-vfs2 version to 2.1, removed + VFS related classes, updated imports + +--- + pom.xml | 2 +- + .../classloader/vfs/AccumuloVFSClassLoader.java | 4 +- + .../vfs/providers/HdfsFileAttributes.java | 59 ---- + .../vfs/providers/HdfsFileContentInfoFactory.java | 49 --- + .../classloader/vfs/providers/HdfsFileObject.java | 303 ------------------- + .../vfs/providers/HdfsFileProvider.java | 75 ----- + .../classloader/vfs/providers/HdfsFileSystem.java | 137 --------- + .../vfs/providers/HdfsFileSystemConfigBuilder.java | 45 --- + .../vfs/providers/HdfsRandomAccessContent.java | 329 --------------------- + .../start/classloader/vfs/providers/package.html | 19 -- + .../providers/ReadOnlyHdfsFileProviderTest.java | 2 + + .../org/apache/accumulo/test/AccumuloDFSBase.java | 2 +- + 12 files changed, 6 insertions(+), 1020 deletions(-) + delete mode 100644 start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsFileAttributes.java + delete mode 100644 start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsFileContentInfoFactory.java + delete mode 100644 start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsFileObject.java + delete mode 100644 start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsFileProvider.java + delete mode 100644 start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsFileSystem.java + delete mode 100644 start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsFileSystemConfigBuilder.java + delete mode 100644 start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsRandomAccessContent.java + delete mode 100644 start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/package.html + +diff --git a/pom.xml b/pom.xml +index d4b3867..2b48125 100644 +--- a/pom.xml ++++ b/pom.xml +@@ -336,7 +336,7 @@ + + org.apache.commons + commons-vfs2 +- 2.0 ++ 2.1 + + + org.apache.hadoop +diff --git a/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java b/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java +index 38d7469..98b3b54 100644 +--- a/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java ++++ b/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java +@@ -27,8 +27,6 @@ import java.util.Collections; + import java.util.List; + + import org.apache.accumulo.start.classloader.AccumuloClassLoader; +-import org.apache.accumulo.start.classloader.vfs.providers.HdfsFileObject; +-import org.apache.accumulo.start.classloader.vfs.providers.HdfsFileProvider; + import org.apache.commons.io.FileUtils; + import org.apache.commons.vfs2.CacheStrategy; + import org.apache.commons.vfs2.FileObject; +@@ -40,6 +38,8 @@ import org.apache.commons.vfs2.impl.DefaultFileSystemManager; + import org.apache.commons.vfs2.impl.FileContentInfoFilenameFactory; + import org.apache.commons.vfs2.impl.VFSClassLoader; + import org.apache.commons.vfs2.provider.FileReplicator; ++import org.apache.commons.vfs2.provider.hdfs.HdfsFileObject; ++import org.apache.commons.vfs2.provider.hdfs.HdfsFileProvider; + import org.apache.hadoop.conf.Configuration; + import org.apache.hadoop.fs.FileSystem; + import org.apache.log4j.Logger; +diff --git a/start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsFileAttributes.java b/start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsFileAttributes.java +deleted file mode 100644 +index bfdd561..0000000 +--- a/start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsFileAttributes.java ++++ /dev/null +@@ -1,59 +0,0 @@ +-/* +- * Licensed to the Apache Software Foundation (ASF) under one or more +- * contributor license agreements. See the NOTICE file distributed with +- * this work for additional information regarding copyright ownership. +- * The ASF licenses this file to You under the Apache License, Version 2.0 +- * (the "License"); you may not use this file except in compliance with +- * the License. You may obtain a copy of the License at +- * +- * http://www.apache.org/licenses/LICENSE-2.0 +- * +- * Unless required by applicable law or agreed to in writing, software +- * distributed under the License is distributed on an "AS IS" BASIS, +- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +- * See the License for the specific language governing permissions and +- * limitations under the License. +- */ +-package org.apache.accumulo.start.classloader.vfs.providers; +- +-/** +- * HDFS file content attributes. +- * +- * @since 2.1 +- */ +-public enum HdfsFileAttributes { +- /** +- * Last access time. +- */ +- LAST_ACCESS_TIME, +- +- /** +- * Block size. +- */ +- BLOCK_SIZE, +- +- /** +- * Group. +- */ +- GROUP, +- +- /** +- * Owner. +- */ +- OWNER, +- +- /** +- * Permissions. +- */ +- PERMISSIONS, +- +- /** +- * Length. +- */ +- LENGTH, +- +- /** +- * Modification time. +- */ +- MODIFICATION_TIME; +-} +diff --git a/start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsFileContentInfoFactory.java b/start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsFileContentInfoFactory.java +deleted file mode 100644 +index b1a4abe..0000000 +--- a/start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsFileContentInfoFactory.java ++++ /dev/null +@@ -1,49 +0,0 @@ +-/* +- * Licensed to the Apache Software Foundation (ASF) under one or more +- * contributor license agreements. See the NOTICE file distributed with +- * this work for additional information regarding copyright ownership. +- * The ASF licenses this file to You under the Apache License, Version 2.0 +- * (the "License"); you may not use this file except in compliance with +- * the License. You may obtain a copy of the License at +- * +- * http://www.apache.org/licenses/LICENSE-2.0 +- * +- * Unless required by applicable law or agreed to in writing, software +- * distributed under the License is distributed on an "AS IS" BASIS, +- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +- * See the License for the specific language governing permissions and +- * limitations under the License. +- */ +-package org.apache.accumulo.start.classloader.vfs.providers; +- +-import org.apache.commons.vfs2.FileContent; +-import org.apache.commons.vfs2.FileContentInfo; +-import org.apache.commons.vfs2.FileContentInfoFactory; +-import org.apache.commons.vfs2.FileSystemException; +-import org.apache.commons.vfs2.impl.DefaultFileContentInfo; +- +-/** +- * Creates FileContentInfo instances for HDFS. +- * +- * @since 2.1 +- */ +-public class HdfsFileContentInfoFactory implements FileContentInfoFactory { +- private static final java.nio.charset.Charset UTF_8 = java.nio.charset.Charset.forName("UTF-8"); +- private static final String CONTENT = "text/plain"; +- private static final String ENCODING = UTF_8.name(); +- +- /** +- * Creates a FileContentInfo for a the given FileContent. +- * +- * @param fileContent +- * Use this FileContent to create a matching FileContentInfo +- * @return a FileContentInfo for the given FileContent with content set to "text/plain" and encoding set to "UTF-8" +- * @throws FileSystemException +- * when a problem occurs creating the FileContentInfo. +- */ +- @Override +- public FileContentInfo create(final FileContent fileContent) throws FileSystemException { +- return new DefaultFileContentInfo(CONTENT, ENCODING); +- } +- +-} +diff --git a/start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsFileObject.java b/start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsFileObject.java +deleted file mode 100644 +index 6849073..0000000 +--- a/start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsFileObject.java ++++ /dev/null +@@ -1,303 +0,0 @@ +-/* +- * Licensed to the Apache Software Foundation (ASF) under one or more +- * contributor license agreements. See the NOTICE file distributed with +- * this work for additional information regarding copyright ownership. +- * The ASF licenses this file to You under the Apache License, Version 2.0 +- * (the "License"); you may not use this file except in compliance with +- * the License. You may obtain a copy of the License at +- * +- * http://www.apache.org/licenses/LICENSE-2.0 +- * +- * Unless required by applicable law or agreed to in writing, software +- * distributed under the License is distributed on an "AS IS" BASIS, +- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +- * See the License for the specific language governing permissions and +- * limitations under the License. +- */ +-package org.apache.accumulo.start.classloader.vfs.providers; +- +-import java.io.FileNotFoundException; +-import java.io.InputStream; +-import java.util.HashMap; +-import java.util.Map; +- +-import org.apache.commons.vfs2.FileNotFolderException; +-import org.apache.commons.vfs2.FileObject; +-import org.apache.commons.vfs2.FileSystemException; +-import org.apache.commons.vfs2.FileType; +-import org.apache.commons.vfs2.RandomAccessContent; +-import org.apache.commons.vfs2.provider.AbstractFileName; +-import org.apache.commons.vfs2.provider.AbstractFileObject; +-import org.apache.commons.vfs2.util.RandomAccessMode; +-import org.apache.hadoop.fs.FileStatus; +-import org.apache.hadoop.fs.FileSystem; +-import org.apache.hadoop.fs.Path; +- +-/** +- * A VFS representation of an HDFS file. +- * +- * @since 2.1 +- */ +-public class HdfsFileObject extends AbstractFileObject { +- private final HdfsFileSystem fs; +- private final FileSystem hdfs; +- private final Path path; +- private FileStatus stat; +- +- /** +- * Constructs a new HDFS FileObject +- * +- * @param name +- * FileName +- * @param fs +- * HdfsFileSystem instance +- * @param hdfs +- * Hadoop FileSystem instance +- * @param p +- * Path to the file in HDFS +- */ +- protected HdfsFileObject(final AbstractFileName name, final HdfsFileSystem fs, final FileSystem hdfs, final Path p) { +- super(name, fs); +- this.fs = fs; +- this.hdfs = hdfs; +- this.path = p; +- } +- +- /** +- * @see org.apache.commons.vfs2.provider.AbstractFileObject#canRenameTo(org.apache.commons.vfs2.FileObject) +- */ +- @Override +- public boolean canRenameTo(final FileObject newfile) { +- throw new UnsupportedOperationException(); +- } +- +- /** +- * @see org.apache.commons.vfs2.provider.AbstractFileObject#doAttach() +- */ +- @Override +- protected void doAttach() throws Exception { +- try { +- this.stat = this.hdfs.getFileStatus(this.path); +- } catch (final FileNotFoundException e) { +- this.stat = null; +- return; +- } +- } +- +- /** +- * @see org.apache.commons.vfs2.provider.AbstractFileObject#doGetAttributes() +- */ +- @Override +- protected Map doGetAttributes() throws Exception { +- if (null == this.stat) { +- return super.doGetAttributes(); +- } else { +- final Map attrs = new HashMap(); +- attrs.put(HdfsFileAttributes.LAST_ACCESS_TIME.toString(), this.stat.getAccessTime()); +- attrs.put(HdfsFileAttributes.BLOCK_SIZE.toString(), this.stat.getBlockSize()); +- attrs.put(HdfsFileAttributes.GROUP.toString(), this.stat.getGroup()); +- attrs.put(HdfsFileAttributes.OWNER.toString(), this.stat.getOwner()); +- attrs.put(HdfsFileAttributes.PERMISSIONS.toString(), this.stat.getPermission().toString()); +- attrs.put(HdfsFileAttributes.LENGTH.toString(), this.stat.getLen()); +- attrs.put(HdfsFileAttributes.MODIFICATION_TIME.toString(), this.stat.getModificationTime()); +- return attrs; +- } +- } +- +- /** +- * @see org.apache.commons.vfs2.provider.AbstractFileObject#doGetContentSize() +- */ +- @Override +- protected long doGetContentSize() throws Exception { +- return stat.getLen(); +- } +- +- /** +- * @see org.apache.commons.vfs2.provider.AbstractFileObject#doGetInputStream() +- */ +- @Override +- protected InputStream doGetInputStream() throws Exception { +- return this.hdfs.open(this.path); +- } +- +- /** +- * @see org.apache.commons.vfs2.provider.AbstractFileObject#doGetLastModifiedTime() +- */ +- @Override +- protected long doGetLastModifiedTime() throws Exception { +- if (null != this.stat) { +- return this.stat.getModificationTime(); +- } else { +- return -1; +- } +- } +- +- /** +- * @see org.apache.commons.vfs2.provider.AbstractFileObject#doGetRandomAccessContent (org.apache.commons.vfs2.util.RandomAccessMode) +- */ +- @Override +- protected RandomAccessContent doGetRandomAccessContent(final RandomAccessMode mode) throws Exception { +- if (mode.equals(RandomAccessMode.READWRITE)) { +- throw new UnsupportedOperationException(); +- } +- return new HdfsRandomAccessContent(this.path, this.hdfs); +- } +- +- /** +- * @see org.apache.commons.vfs2.provider.AbstractFileObject#doGetType() +- */ +- @Override +- // TODO Remove deprecation warning suppression when Hadoop1 support is dropped +- @SuppressWarnings("deprecation") +- protected FileType doGetType() throws Exception { +- try { +- doAttach(); +- if (null == stat) { +- return FileType.IMAGINARY; +- } +- if (stat.isDir()) { +- return FileType.FOLDER; +- } else { +- return FileType.FILE; +- } +- } catch (final FileNotFoundException fnfe) { +- return FileType.IMAGINARY; +- } +- } +- +- /** +- * @see org.apache.commons.vfs2.provider.AbstractFileObject#doIsHidden() +- */ +- @Override +- protected boolean doIsHidden() throws Exception { +- return false; +- } +- +- /** +- * @see org.apache.commons.vfs2.provider.AbstractFileObject#doIsReadable() +- */ +- @Override +- protected boolean doIsReadable() throws Exception { +- return true; +- } +- +- /** +- * @see org.apache.commons.vfs2.provider.AbstractFileObject#doIsSameFile(org.apache.commons.vfs2.FileObject) +- */ +- @Override +- protected boolean doIsSameFile(final FileObject destFile) throws FileSystemException { +- throw new UnsupportedOperationException(); +- } +- +- /** +- * @see org.apache.commons.vfs2.provider.AbstractFileObject#doIsWriteable() +- */ +- @Override +- protected boolean doIsWriteable() throws Exception { +- return false; +- } +- +- /** +- * @see org.apache.commons.vfs2.provider.AbstractFileObject#doListChildren() +- */ +- @Override +- protected String[] doListChildren() throws Exception { +- if (this.doGetType() != FileType.FOLDER) { +- throw new FileNotFolderException(this); +- } +- +- final FileStatus[] files = this.hdfs.listStatus(this.path); +- final String[] children = new String[files.length]; +- int i = 0; +- for (final FileStatus status : files) { +- children[i++] = status.getPath().getName(); +- } +- return children; +- } +- +- /** +- * @see org.apache.commons.vfs2.provider.AbstractFileObject#doListChildrenResolved() +- */ +- @Override +- protected FileObject[] doListChildrenResolved() throws Exception { +- if (this.doGetType() != FileType.FOLDER) { +- return null; +- } +- final String[] children = doListChildren(); +- final FileObject[] fo = new FileObject[children.length]; +- for (int i = 0; i < children.length; i++) { +- final Path p = new Path(this.path, children[i]); +- fo[i] = this.fs.resolveFile(p.toUri().toString()); +- } +- return fo; +- } +- +- /** +- * @see org.apache.commons.vfs2.provider.AbstractFileObject#doRemoveAttribute(java.lang.String) +- */ +- @Override +- protected void doRemoveAttribute(final String attrName) throws Exception { +- throw new UnsupportedOperationException(); +- } +- +- /** +- * @see org.apache.commons.vfs2.provider.AbstractFileObject#doSetAttribute(java.lang.String, java.lang.Object) +- */ +- @Override +- protected void doSetAttribute(final String attrName, final Object value) throws Exception { +- throw new UnsupportedOperationException(); +- } +- +- /** +- * @see org.apache.commons.vfs2.provider.AbstractFileObject#doSetLastModifiedTime(long) +- */ +- @Override +- protected boolean doSetLastModifiedTime(final long modtime) throws Exception { +- throw new UnsupportedOperationException(); +- } +- +- /** +- * @see java.lang.Object#equals(java.lang.Object) +- */ +- @Override +- public boolean equals(final Object o) { +- if (null == o) { +- return false; +- } +- if (o == this) { +- return true; +- } +- if (o instanceof HdfsFileObject) { +- final HdfsFileObject other = (HdfsFileObject) o; +- if (other.path.equals(this.path)) { +- return true; +- } +- } +- return false; +- } +- +- /** +- * @see org.apache.commons.vfs2.provider.AbstractFileObject#exists() +- * @return boolean true if file exists, false if not +- */ +- @Override +- public boolean exists() throws FileSystemException { +- try { +- doAttach(); +- return this.stat != null; +- } catch (final FileNotFoundException fne) { +- return false; +- } catch (final Exception e) { +- throw new FileSystemException("Unable to check existance ", e); +- } +- } +- +- /** +- * @see java.lang.Object#hashCode() +- */ +- @Override +- public int hashCode() { +- return this.path.getName().toString().hashCode(); +- } +- +-} +diff --git a/start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsFileProvider.java b/start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsFileProvider.java +deleted file mode 100644 +index 9ddfab5..0000000 +--- a/start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsFileProvider.java ++++ /dev/null +@@ -1,75 +0,0 @@ +-/* +- * Licensed to the Apache Software Foundation (ASF) under one or more +- * contributor license agreements. See the NOTICE file distributed with +- * this work for additional information regarding copyright ownership. +- * The ASF licenses this file to You under the Apache License, Version 2.0 +- * (the "License"); you may not use this file except in compliance with +- * the License. You may obtain a copy of the License at +- * +- * http://www.apache.org/licenses/LICENSE-2.0 +- * +- * Unless required by applicable law or agreed to in writing, software +- * distributed under the License is distributed on an "AS IS" BASIS, +- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +- * See the License for the specific language governing permissions and +- * limitations under the License. +- */ +-package org.apache.accumulo.start.classloader.vfs.providers; +- +-import java.util.Arrays; +-import java.util.Collection; +-import java.util.Collections; +- +-import org.apache.commons.vfs2.Capability; +-import org.apache.commons.vfs2.FileName; +-import org.apache.commons.vfs2.FileSystem; +-import org.apache.commons.vfs2.FileSystemConfigBuilder; +-import org.apache.commons.vfs2.FileSystemException; +-import org.apache.commons.vfs2.FileSystemOptions; +-import org.apache.commons.vfs2.provider.AbstractOriginatingFileProvider; +-import org.apache.commons.vfs2.provider.http.HttpFileNameParser; +- +-/** +- * FileProvider for HDFS files. +- * +- * @since 2.1 +- */ +-public class HdfsFileProvider extends AbstractOriginatingFileProvider { +- protected static final Collection CAPABILITIES = Collections.unmodifiableCollection(Arrays.asList(new Capability[] {Capability.GET_TYPE, +- Capability.READ_CONTENT, Capability.URI, Capability.GET_LAST_MODIFIED, Capability.ATTRIBUTES, Capability.RANDOM_ACCESS_READ, +- Capability.DIRECTORY_READ_CONTENT, Capability.LIST_CHILDREN})); +- +- /** +- * Constructs a new HdfsFileProvider +- */ +- public HdfsFileProvider() { +- super(); +- this.setFileNameParser(HttpFileNameParser.getInstance()); +- } +- +- /** +- * @see org.apache.commons.vfs2.provider.AbstractOriginatingFileProvider#doCreateFileSystem(org.apache.commons.vfs2.FileName, +- * org.apache.commons.vfs2.FileSystemOptions) +- */ +- @Override +- protected FileSystem doCreateFileSystem(final FileName rootName, final FileSystemOptions fileSystemOptions) throws FileSystemException { +- return new HdfsFileSystem(rootName, fileSystemOptions); +- } +- +- /** +- * @see org.apache.commons.vfs2.provider.FileProvider#getCapabilities() +- */ +- @Override +- public Collection getCapabilities() { +- return CAPABILITIES; +- } +- +- /** +- * @see org.apache.commons.vfs2.provider.AbstractFileProvider#getConfigBuilder() +- */ +- @Override +- public FileSystemConfigBuilder getConfigBuilder() { +- return HdfsFileSystemConfigBuilder.getInstance(); +- } +- +-} +diff --git a/start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsFileSystem.java b/start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsFileSystem.java +deleted file mode 100644 +index 8d45555..0000000 +--- a/start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsFileSystem.java ++++ /dev/null +@@ -1,137 +0,0 @@ +-/* +- * Licensed to the Apache Software Foundation (ASF) under one or more +- * contributor license agreements. See the NOTICE file distributed with +- * this work for additional information regarding copyright ownership. +- * The ASF licenses this file to You under the Apache License, Version 2.0 +- * (the "License"); you may not use this file except in compliance with +- * the License. You may obtain a copy of the License at +- * +- * http://www.apache.org/licenses/LICENSE-2.0 +- * +- * Unless required by applicable law or agreed to in writing, software +- * distributed under the License is distributed on an "AS IS" BASIS, +- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +- * See the License for the specific language governing permissions and +- * limitations under the License. +- */ +-package org.apache.accumulo.start.classloader.vfs.providers; +- +-import java.io.IOException; +-import java.io.UnsupportedEncodingException; +-import java.net.URLDecoder; +-import java.util.Collection; +- +-import org.apache.commons.logging.Log; +-import org.apache.commons.logging.LogFactory; +-import org.apache.commons.vfs2.CacheStrategy; +-import org.apache.commons.vfs2.Capability; +-import org.apache.commons.vfs2.FileName; +-import org.apache.commons.vfs2.FileObject; +-import org.apache.commons.vfs2.FileSystemException; +-import org.apache.commons.vfs2.FileSystemOptions; +-import org.apache.commons.vfs2.provider.AbstractFileName; +-import org.apache.commons.vfs2.provider.AbstractFileSystem; +-import org.apache.hadoop.conf.Configuration; +-import org.apache.hadoop.fs.FileSystem; +-import org.apache.hadoop.fs.Path; +- +-/** +- * A VFS FileSystem that interacts with HDFS. +- * +- * @since 2.1 +- */ +-public class HdfsFileSystem extends AbstractFileSystem { +- private static final java.nio.charset.Charset UTF_8 = java.nio.charset.Charset.forName("UTF-8"); +- private static final Log log = LogFactory.getLog(HdfsFileSystem.class); +- +- private FileSystem fs; +- +- protected HdfsFileSystem(final FileName rootName, final FileSystemOptions fileSystemOptions) { +- super(rootName, null, fileSystemOptions); +- } +- +- /** +- * @see org.apache.commons.vfs2.provider.AbstractFileSystem#addCapabilities(java.util.Collection) +- */ +- @Override +- protected void addCapabilities(final Collection capabilities) { +- capabilities.addAll(HdfsFileProvider.CAPABILITIES); +- } +- +- /** +- * @see org.apache.commons.vfs2.provider.AbstractFileSystem#close() +- */ +- @Override +- synchronized public void close() { +- try { +- if (null != fs) { +- fs.close(); +- } +- } catch (final IOException e) { +- throw new RuntimeException("Error closing HDFS client", e); +- } +- super.close(); +- } +- +- /** +- * @see org.apache.commons.vfs2.provider.AbstractFileSystem#createFile(org.apache.commons.vfs2.provider.AbstractFileName) +- */ +- @Override +- protected FileObject createFile(final AbstractFileName name) throws Exception { +- throw new FileSystemException("Operation not supported"); +- } +- +- /** +- * @see org.apache.commons.vfs2.provider.AbstractFileSystem#resolveFile(org.apache.commons.vfs2.FileName) +- */ +- @Override +- public FileObject resolveFile(final FileName name) throws FileSystemException { +- +- synchronized (this) { +- if (null == this.fs) { +- final String hdfsUri = name.getRootURI(); +- final Configuration conf = new Configuration(true); +- conf.set(org.apache.hadoop.fs.FileSystem.FS_DEFAULT_NAME_KEY, hdfsUri); +- this.fs = null; +- try { +- fs = org.apache.hadoop.fs.FileSystem.get(conf); +- } catch (final IOException e) { +- log.error("Error connecting to filesystem " + hdfsUri, e); +- throw new FileSystemException("Error connecting to filesystem " + hdfsUri, e); +- } +- } +- } +- +- boolean useCache = (null != getContext().getFileSystemManager().getFilesCache()); +- FileObject file; +- if (useCache) { +- file = this.getFileFromCache(name); +- } else { +- file = null; +- } +- if (null == file) { +- String path = null; +- try { +- path = URLDecoder.decode(name.getPath(), UTF_8.name()); +- } catch (final UnsupportedEncodingException e) { +- path = name.getPath(); +- } +- final Path filePath = new Path(path); +- file = new HdfsFileObject((AbstractFileName) name, this, fs, filePath); +- if (useCache) { +- this.putFileToCache(file); +- } +- +- } +- +- /** +- * resync the file information if requested +- */ +- if (getFileSystemManager().getCacheStrategy().equals(CacheStrategy.ON_RESOLVE)) { +- file.refresh(); +- } +- +- return file; +- } +- +-} +diff --git a/start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsFileSystemConfigBuilder.java b/start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsFileSystemConfigBuilder.java +deleted file mode 100644 +index 0defa56..0000000 +--- a/start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsFileSystemConfigBuilder.java ++++ /dev/null +@@ -1,45 +0,0 @@ +-/* +- * Licensed to the Apache Software Foundation (ASF) under one or more +- * contributor license agreements. See the NOTICE file distributed with +- * this work for additional information regarding copyright ownership. +- * The ASF licenses this file to You under the Apache License, Version 2.0 +- * (the "License"); you may not use this file except in compliance with +- * the License. You may obtain a copy of the License at +- * +- * http://www.apache.org/licenses/LICENSE-2.0 +- * +- * Unless required by applicable law or agreed to in writing, software +- * distributed under the License is distributed on an "AS IS" BASIS, +- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +- * See the License for the specific language governing permissions and +- * limitations under the License. +- */ +-package org.apache.accumulo.start.classloader.vfs.providers; +- +-import org.apache.commons.vfs2.FileSystem; +-import org.apache.commons.vfs2.FileSystemConfigBuilder; +- +-/** +- * Configuration settings for the HdfsFileSystem. +- * +- * @since 2.1 +- */ +-public class HdfsFileSystemConfigBuilder extends FileSystemConfigBuilder { +- private static final HdfsFileSystemConfigBuilder BUILDER = new HdfsFileSystemConfigBuilder(); +- +- /** +- * @return HdfsFileSystemConfigBuilder instance +- */ +- public static HdfsFileSystemConfigBuilder getInstance() { +- return BUILDER; +- } +- +- /** +- * @return HDFSFileSystem +- */ +- @Override +- protected Class getConfigClass() { +- return HdfsFileSystem.class; +- } +- +-} +diff --git a/start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsRandomAccessContent.java b/start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsRandomAccessContent.java +deleted file mode 100644 +index 30140ba..0000000 +--- a/start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsRandomAccessContent.java ++++ /dev/null +@@ -1,329 +0,0 @@ +-/* +- * Licensed to the Apache Software Foundation (ASF) under one or more +- * contributor license agreements. See the NOTICE file distributed with +- * this work for additional information regarding copyright ownership. +- * The ASF licenses this file to You under the Apache License, Version 2.0 +- * (the "License"); you may not use this file except in compliance with +- * the License. You may obtain a copy of the License at +- * +- * http://www.apache.org/licenses/LICENSE-2.0 +- * +- * Unless required by applicable law or agreed to in writing, software +- * distributed under the License is distributed on an "AS IS" BASIS, +- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +- * See the License for the specific language governing permissions and +- * limitations under the License. +- */ +-package org.apache.accumulo.start.classloader.vfs.providers; +- +-import java.io.BufferedReader; +-import java.io.IOException; +-import java.io.InputStream; +-import java.io.InputStreamReader; +- +-import org.apache.commons.vfs2.RandomAccessContent; +-import org.apache.hadoop.fs.FSDataInputStream; +-import org.apache.hadoop.fs.FileSystem; +-import org.apache.hadoop.fs.Path; +- +-/** +- * Provides random access to content in an HdfsFileObject. Currently this only supports read operations. All write operations throw an +- * {@link UnsupportedOperationException}. +- * +- * @since 2.1 +- */ +-public class HdfsRandomAccessContent implements RandomAccessContent { +- private static final java.nio.charset.Charset UTF_8 = java.nio.charset.Charset.forName("UTF-8"); +- private final FileSystem fs; +- private final Path path; +- private final FSDataInputStream fis; +- +- /** +- * +- * @param path +- * A Hadoop Path +- * @param fs +- * A Hadoop FileSystem +- * @throws IOException +- * when the path cannot be processed. +- */ +- public HdfsRandomAccessContent(final Path path, final FileSystem fs) throws IOException { +- this.fs = fs; +- this.path = path; +- this.fis = this.fs.open(this.path); +- } +- +- /** +- * @see org.apache.commons.vfs2.RandomAccessContent#close() +- */ +- @Override +- public void close() throws IOException { +- this.fis.close(); +- } +- +- /** +- * @see org.apache.commons.vfs2.RandomAccessContent#getFilePointer() +- */ +- @Override +- public long getFilePointer() throws IOException { +- return this.fis.getPos(); +- } +- +- /** +- * @see org.apache.commons.vfs2.RandomAccessContent#getInputStream() +- */ +- @Override +- public InputStream getInputStream() throws IOException { +- return this.fis; +- } +- +- /** +- * @see org.apache.commons.vfs2.RandomAccessContent#length() +- */ +- @Override +- public long length() throws IOException { +- return this.fs.getFileStatus(this.path).getLen(); +- } +- +- /** +- * @see java.io.DataInput#readBoolean() +- */ +- @Override +- public boolean readBoolean() throws IOException { +- return this.fis.readBoolean(); +- } +- +- /** +- * @see java.io.DataInput#readByte() +- */ +- @Override +- public byte readByte() throws IOException { +- return this.fis.readByte(); +- } +- +- /** +- * @see java.io.DataInput#readChar() +- */ +- @Override +- public char readChar() throws IOException { +- return this.fis.readChar(); +- } +- +- /** +- * @see java.io.DataInput#readDouble() +- */ +- @Override +- public double readDouble() throws IOException { +- return this.fis.readDouble(); +- } +- +- /** +- * @see java.io.DataInput#readFloat() +- */ +- @Override +- public float readFloat() throws IOException { +- return this.fis.readFloat(); +- } +- +- /** +- * @see java.io.DataInput#readFully(byte[]) +- */ +- @Override +- public void readFully(final byte[] b) throws IOException { +- throw new UnsupportedOperationException(); +- } +- +- /** +- * @see java.io.DataInput#readFully(byte[], int, int) +- */ +- @Override +- public void readFully(final byte[] b, final int off, final int len) throws IOException { +- throw new UnsupportedOperationException(); +- } +- +- /** +- * @see java.io.DataInput#readInt() +- */ +- @Override +- public int readInt() throws IOException { +- return this.fis.readInt(); +- } +- +- /** +- * @see java.io.DataInput#readLine() +- */ +- @Override +- public String readLine() throws IOException { +- BufferedReader d = new BufferedReader(new InputStreamReader(this.fis, UTF_8)); +- return d.readLine(); +- } +- +- /** +- * @see java.io.DataInput#readLong() +- */ +- @Override +- public long readLong() throws IOException { +- return this.fis.readLong(); +- } +- +- /** +- * @see java.io.DataInput#readShort() +- */ +- @Override +- public short readShort() throws IOException { +- return this.fis.readShort(); +- } +- +- /** +- * @see java.io.DataInput#readUnsignedByte() +- */ +- @Override +- public int readUnsignedByte() throws IOException { +- return this.fis.readUnsignedByte(); +- } +- +- /** +- * @see java.io.DataInput#readUnsignedShort() +- */ +- @Override +- public int readUnsignedShort() throws IOException { +- return this.fis.readUnsignedShort(); +- } +- +- /** +- * @see java.io.DataInput#readUTF() +- */ +- @Override +- public String readUTF() throws IOException { +- return this.fis.readUTF(); +- } +- +- /** +- * @see org.apache.commons.vfs2.RandomAccessContent#seek(long) +- */ +- @Override +- public void seek(final long pos) throws IOException { +- this.fis.seek(pos); +- } +- +- /** +- * @see java.io.DataInput#skipBytes(int) +- */ +- @Override +- public int skipBytes(final int n) throws IOException { +- throw new UnsupportedOperationException(); +- } +- +- /** +- * @see java.io.DataOutput#write(byte[]) +- */ +- @Override +- public void write(final byte[] b) throws IOException { +- throw new UnsupportedOperationException(); +- } +- +- /** +- * @see java.io.DataOutput#write(byte[], int, int) +- */ +- @Override +- public void write(final byte[] b, final int off, final int len) throws IOException { +- throw new UnsupportedOperationException(); +- } +- +- /** +- * @see java.io.DataOutput#write(int) +- */ +- @Override +- public void write(final int b) throws IOException { +- throw new UnsupportedOperationException(); +- } +- +- /** +- * @see java.io.DataOutput#writeBoolean(boolean) +- */ +- @Override +- public void writeBoolean(final boolean v) throws IOException { +- throw new UnsupportedOperationException(); +- } +- +- /** +- * @see java.io.DataOutput#writeByte(int) +- */ +- @Override +- public void writeByte(final int v) throws IOException { +- throw new UnsupportedOperationException(); +- } +- +- /** +- * @see java.io.DataOutput#writeBytes(java.lang.String) +- */ +- @Override +- public void writeBytes(final String s) throws IOException { +- throw new UnsupportedOperationException(); +- } +- +- /** +- * @see java.io.DataOutput#writeChar(int) +- */ +- @Override +- public void writeChar(final int v) throws IOException { +- throw new UnsupportedOperationException(); +- } +- +- /** +- * @see java.io.DataOutput#writeChars(java.lang.String) +- */ +- @Override +- public void writeChars(final String s) throws IOException { +- throw new UnsupportedOperationException(); +- } +- +- /** +- * @see java.io.DataOutput#writeDouble(double) +- */ +- @Override +- public void writeDouble(final double v) throws IOException { +- throw new UnsupportedOperationException(); +- } +- +- /** +- * @see java.io.DataOutput#writeFloat(float) +- */ +- @Override +- public void writeFloat(final float v) throws IOException { +- throw new UnsupportedOperationException(); +- } +- +- /** +- * @see java.io.DataOutput#writeInt(int) +- */ +- @Override +- public void writeInt(final int v) throws IOException { +- throw new UnsupportedOperationException(); +- } +- +- /** +- * @see java.io.DataOutput#writeLong(long) +- */ +- @Override +- public void writeLong(final long v) throws IOException { +- throw new UnsupportedOperationException(); +- } +- +- /** +- * @see java.io.DataOutput#writeShort(int) +- */ +- @Override +- public void writeShort(final int v) throws IOException { +- throw new UnsupportedOperationException(); +- } +- +- /** +- * @see java.io.DataOutput#writeUTF(java.lang.String) +- */ +- @Override +- public void writeUTF(final String s) throws IOException { +- throw new UnsupportedOperationException(); +- } +- +-} +diff --git a/start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/package.html b/start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/package.html +deleted file mode 100644 +index 8cbb34e..0000000 +--- a/start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/package.html ++++ /dev/null +@@ -1,19 +0,0 @@ +- +- +-

The HDFS File Provider

+- +diff --git a/start/src/test/java/org/apache/accumulo/start/classloader/vfs/providers/ReadOnlyHdfsFileProviderTest.java b/start/src/test/java/org/apache/accumulo/start/classloader/vfs/providers/ReadOnlyHdfsFileProviderTest.java +index b5cec83..5b69df9 100644 +--- a/start/src/test/java/org/apache/accumulo/start/classloader/vfs/providers/ReadOnlyHdfsFileProviderTest.java ++++ b/start/src/test/java/org/apache/accumulo/start/classloader/vfs/providers/ReadOnlyHdfsFileProviderTest.java +@@ -24,6 +24,8 @@ import org.apache.commons.vfs2.FileObject; + import org.apache.commons.vfs2.FileSystemException; + import org.apache.commons.vfs2.FileType; + import org.apache.commons.vfs2.impl.DefaultFileSystemManager; ++import org.apache.commons.vfs2.provider.hdfs.HdfsFileAttributes; ++import org.apache.commons.vfs2.provider.hdfs.HdfsFileProvider; + import org.apache.commons.vfs2.util.RandomAccessMode; + import org.apache.hadoop.fs.FileSystem; + import org.apache.hadoop.fs.Path; +diff --git a/start/src/test/java/org/apache/accumulo/test/AccumuloDFSBase.java b/start/src/test/java/org/apache/accumulo/test/AccumuloDFSBase.java +index feab493..343a3ee 100644 +--- a/start/src/test/java/org/apache/accumulo/test/AccumuloDFSBase.java ++++ b/start/src/test/java/org/apache/accumulo/test/AccumuloDFSBase.java +@@ -21,7 +21,6 @@ import java.io.IOException; + import java.net.URI; + + import org.apache.accumulo.start.classloader.vfs.MiniDFSUtil; +-import org.apache.accumulo.start.classloader.vfs.providers.HdfsFileProvider; + import org.apache.commons.vfs2.CacheStrategy; + import org.apache.commons.vfs2.FileSystemException; + import org.apache.commons.vfs2.cache.DefaultFilesCache; +@@ -29,6 +28,7 @@ import org.apache.commons.vfs2.cache.SoftRefFilesCache; + import org.apache.commons.vfs2.impl.DefaultFileReplicator; + import org.apache.commons.vfs2.impl.DefaultFileSystemManager; + import org.apache.commons.vfs2.impl.FileContentInfoFilenameFactory; ++import org.apache.commons.vfs2.provider.hdfs.HdfsFileProvider; + import org.apache.hadoop.conf.Configuration; + import org.apache.hadoop.hdfs.DFSConfigKeys; + import org.apache.hadoop.hdfs.MiniDFSCluster; +-- +2.7.4 + diff --git a/accumulo-gc.service b/accumulo-gc.service new file mode 100644 index 0000000..e9135b3 --- /dev/null +++ b/accumulo-gc.service @@ -0,0 +1,12 @@ +[Unit] +Description=Apache Accumulo Garbage Collector service +After=syslog.target network.target + +[Service] +User=accumulo +Group=accumulo +ExecStart=/usr/bin/accumulo gc +SuccessExitStatus=143 + +[Install] +WantedBy=multi-user.target diff --git a/accumulo-master.service b/accumulo-master.service new file mode 100644 index 0000000..0695f04 --- /dev/null +++ b/accumulo-master.service @@ -0,0 +1,12 @@ +[Unit] +Description=Apache Accumulo Master service +After=syslog.target network.target + +[Service] +User=accumulo +Group=accumulo +ExecStart=/usr/bin/accumulo master +SuccessExitStatus=143 + +[Install] +WantedBy=multi-user.target diff --git a/accumulo-monitor.service b/accumulo-monitor.service new file mode 100644 index 0000000..a291928 --- /dev/null +++ b/accumulo-monitor.service @@ -0,0 +1,12 @@ +[Unit] +Description=Apache Accumulo Monitor service +After=syslog.target network.target + +[Service] +User=accumulo +Group=accumulo +ExecStart=/usr/bin/accumulo monitor +SuccessExitStatus=143 + +[Install] +WantedBy=multi-user.target diff --git a/accumulo-tracer.service b/accumulo-tracer.service new file mode 100644 index 0000000..8d9caad --- /dev/null +++ b/accumulo-tracer.service @@ -0,0 +1,12 @@ +[Unit] +Description=Apache Accumulo Tracer service +After=syslog.target network.target + +[Service] +User=accumulo +Group=accumulo +ExecStart=/usr/bin/accumulo tracer +SuccessExitStatus=143 + +[Install] +WantedBy=multi-user.target diff --git a/accumulo-tserver.service b/accumulo-tserver.service new file mode 100644 index 0000000..a69c5b2 --- /dev/null +++ b/accumulo-tserver.service @@ -0,0 +1,12 @@ +[Unit] +Description=Apache Accumulo TServer service +After=syslog.target network.target + +[Service] +User=accumulo +Group=accumulo +ExecStart=/usr/bin/accumulo tserver +SuccessExitStatus=143 + +[Install] +WantedBy=multi-user.target diff --git a/accumulo.conf b/accumulo.conf new file mode 100644 index 0000000..191fe35 --- /dev/null +++ b/accumulo.conf @@ -0,0 +1,34 @@ +#!/usr/bin/bash + +# This file is sourced by /usr/bin/accumulo before launching java. It will use +# the $ACCUMULO_OPTS environment variable created here to add options to the +# java command line. This file can be overridden per-user by creating a +# $HOME/.accumulorc to be sourced after this file. + +# The $1 parameter is passed with the first argument provided to +# /usr/bin/accumulo, which is usually the name of the accumulo service or +# function to run. + +# Append some common arguments +# +# Note: app isn't used for anything, but makes it easier to locate services +# quickly with ps/top/etc output +ACCUMULO_OPTS=("-Dapp=$1" '-XX:+UseConcMarkSweepGC' '-XX:CMSInitiatingOccupancyFraction=75' '-Djava.net.preferIPv4Stack=true' '-XX:-OmitStackTraceInFastThrow' '-Djavax.xml.parsers.DocumentBuilderFactory=com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl' '-XX:OnOutOfMemoryError=kill -9 %p') + +# Append some service-specific arguments +case "$1" in + gc) ACCUMULO_OPTS=("${ACCUMULO_OPTS[@]}" '-Xmx256m' '-Xms256m') ;; + master) ACCUMULO_OPTS=("${ACCUMULO_OPTS[@]}" '-Xmx1g' '-Xms1g') ;; + monitor) ACCUMULO_OPTS=("${ACCUMULO_OPTS[@]}" '-Xmx1g' '-Xms256m') ;; + tserver) ACCUMULO_OPTS=("${ACCUMULO_OPTS[@]}" '-Xmx1g' '-Xms1g' '-XX:NewSize=500m' '-XX:MaxNewSize=500m') ;; + *) ACCUMULO_OPTS=("${ACCUMULO_OPTS[@]}" '-Xmx1g' '-Xms256m') ;; +esac + +# Set this because it's read by the VFS classloader +export ACCUMULO_HOME=/etc/accumulo +# Set these because they could be referenced by logger configuration +export ACCUMULO_CONF_DIR=/etc/accumulo +export ACCUMULO_LOG_DIR=/var/log/accumulo + +# See HADOOP-7154 and ACCUMULO-847 +export MALLOC_ARENA_MAX=1 diff --git a/accumulo.spec b/accumulo.spec new file mode 100644 index 0000000..ba2547c --- /dev/null +++ b/accumulo.spec @@ -0,0 +1,614 @@ +%global _hardened_build 1 +%global longproj Apache Accumulo + +# jpackage main class +%global main_class org.apache.%{name}.start.Main + +Name: accumulo +Version: 1.6.6 +Release: 13%{?dist} +Summary: A software platform for processing vast amounts of data +License: ASL 2.0 +Group: Development/Libraries +URL: https://%{name}.apache.org +Source0: https://www.apache.org/dist/%{name}/%{version}/%{name}-%{version}-src.tar.gz + +# systemd service files +Source1: %{name}-master.service +Source2: %{name}-tserver.service +Source3: %{name}-gc.service +Source4: %{name}-tracer.service +Source5: %{name}-monitor.service + +# Java configuration file for Fedora +Source6: %{name}.conf + +# Use Jetty version 9 instead of 8 +Patch0: jetty9.patch +# Use current version of commons-configuration +Patch1: commons-configuration.patch +# Use current version of commons-math +Patch2: commons-math.patch +# Apply Fedora JNI conventions +Patch3: native-code.patch +# Disable broken tests +Patch4: disabled-tests.patch +# Patch upstream-provided example configuration for Fedora +Patch5: default-conf.patch +%if 0%{?fedora} > 24 +# Patch upstream ACCUMULO-3470 (update to commons-vfs 2.1) +Patch6: ACCUMULO-3470.patch +%endif +# Fix for Shell erroneously reading accumulo-site.xml +Patch7: shell-not-read-conf.patch +# Fix for updating to flot 0.8 from 0.7 (adds flot.time) +Patch8: flot8.patch +# Workaround for https://github.com/jline/jline2/issues/205 +Patch9: jline-shell-workaround.patch + +BuildRequires: apache-commons-cli +BuildRequires: apache-commons-codec +BuildRequires: apache-commons-collections +BuildRequires: apache-commons-configuration +BuildRequires: apache-commons-io +BuildRequires: apache-commons-lang +BuildRequires: apache-commons-logging +BuildRequires: apache-commons-math +%if 0%{?fedora} > 24 +BuildRequires: apache-commons-vfs >= 2.1-7 +%else +BuildRequires: apache-commons-vfs < 2.1 +%endif +BuildRequires: beust-jcommander +BuildRequires: bouncycastle +BuildRequires: exec-maven-plugin +BuildRequires: google-gson +BuildRequires: guava +BuildRequires: hadoop-client +BuildRequires: hadoop-tests +BuildRequires: java-devel +BuildRequires: jetty-security +BuildRequires: jetty-server +BuildRequires: jetty-servlet +BuildRequires: jetty-util +BuildRequires: jline2 +BuildRequires: jpackage-utils +BuildRequires: libthrift-java +BuildRequires: log4j12 +BuildRequires: maven-local +BuildRequires: mvn(javax.servlet:javax.servlet-api) +BuildRequires: native-maven-plugin +BuildRequires: powermock-api-easymock +BuildRequires: powermock-core +BuildRequires: powermock-junit4 +BuildRequires: slf4j +BuildRequires: systemd-units +BuildRequires: zookeeper-java + +Requires: %{name}-core = %{version}-%{release} +Requires: %{name}-master = %{version}-%{release} +Requires: %{name}-tserver = %{version}-%{release} +Requires: %{name}-gc = %{version}-%{release} +Requires: %{name}-monitor = %{version}-%{release} +Requires: %{name}-tracer = %{version}-%{release} +Requires: %{name}-examples = %{version}-%{release} +Requires: %{name}-native%{?_isa} = %{version}-%{release} + +%description + %{longproj} is a sorted, distributed key/value store based on Google's +BigTable design. It is built on top of Apache Hadoop, Zookeeper, and Thrift. It +features a few novel improvements on the BigTable design in the form of +cell-level access labels and a server-side programming mechanism that can +modify key/value pairs at various points in the data management process. + +%package core +Summary: Libraries for %{longproj} Java clients +# the bloom filter code is BSD licensed, everything else is ASL 2.0 +License: ASL 2.0 and BSD +Group: Applications/System +BuildArch: noarch +Requires(pre): /usr/sbin/useradd +Obsoletes: %{name}-javadoc < 1.6.0-5%{?dist} + +%description core + %{longproj} is a sorted, distributed key/value store based on Google's +BigTable design. It is built on top of Apache Hadoop, Zookeeper, and Thrift. It +features a few novel improvements on the BigTable design in the form of +cell-level access labels and a server-side programming mechanism that can +modify key/value pairs at various points in the data management process. + +This package provides libraries for %{longproj} clients. + +%package server-base +Summary: The %{longproj} Server Base libraries +License: ASL 2.0 +Group: Applications/System +BuildArch: noarch +Requires: %{name}-core = %{version}-%{release} + +%description server-base + %{longproj} is a sorted, distributed key/value store based on Google's +BigTable design. It is built on top of Apache Hadoop, Zookeeper, and Thrift. It +features a few novel improvements on the BigTable design in the form of +cell-level access labels and a server-side programming mechanism that can +modify key/value pairs at various points in the data management process. + +This package provides jars for other %{longproj} services. + +%package master +Summary: The %{longproj} Master service +License: ASL 2.0 +Group: Applications/System +BuildArch: noarch +Requires: %{name}-core = %{version}-%{release} +Requires: %{name}-server-base = %{version}-%{release} +Requires(post): systemd-units +Requires(preun): systemd-units +Requires(postun): systemd-units + +%description master + %{longproj} is a sorted, distributed key/value store based on Google's +BigTable design. It is built on top of Apache Hadoop, Zookeeper, and Thrift. It +features a few novel improvements on the BigTable design in the form of +cell-level access labels and a server-side programming mechanism that can +modify key/value pairs at various points in the data management process. + +This package provides the master service for %{longproj}. + +%package tserver +Summary: The %{longproj} TServer service +License: ASL 2.0 +Group: Applications/System +BuildArch: noarch +Requires: %{name}-core = %{version}-%{release} +Requires: %{name}-server-base = %{version}-%{release} +Requires(post): systemd-units +Requires(preun): systemd-units +Requires(postun): systemd-units + +%description tserver + %{longproj} is a sorted, distributed key/value store based on Google's +BigTable design. It is built on top of Apache Hadoop, Zookeeper, and Thrift. It +features a few novel improvements on the BigTable design in the form of +cell-level access labels and a server-side programming mechanism that can +modify key/value pairs at various points in the data management process. + +This package provides the tserver service for %{longproj}. + +%package gc +Summary: The %{longproj} Garbage Collector service +License: ASL 2.0 +Group: Applications/System +BuildArch: noarch +Requires: %{name}-core = %{version}-%{release} +Requires: %{name}-server-base = %{version}-%{release} +Requires(post): systemd-units +Requires(preun): systemd-units +Requires(postun): systemd-units + +%description gc + %{longproj} is a sorted, distributed key/value store based on Google's +BigTable design. It is built on top of Apache Hadoop, Zookeeper, and Thrift. It +features a few novel improvements on the BigTable design in the form of +cell-level access labels and a server-side programming mechanism that can +modify key/value pairs at various points in the data management process. + +This package provides the gc service for %{longproj}. + +%package monitor +Summary: The %{longproj} Monitor service +License: ASL 2.0 +Group: Applications/System +BuildArch: noarch +Requires: %{name}-core = %{version}-%{release} +Requires: %{name}-server-base = %{version}-%{release} +Requires(post): systemd-units +Requires(preun): systemd-units +Requires(postun): systemd-units +Requires: nodejs-flot +Requires: js-jquery1 + +%description monitor + %{longproj} is a sorted, distributed key/value store based on Google's +BigTable design. It is built on top of Apache Hadoop, Zookeeper, and Thrift. It +features a few novel improvements on the BigTable design in the form of +cell-level access labels and a server-side programming mechanism that can +modify key/value pairs at various points in the data management process. + +This package provides the monitor service for %{longproj}. + +%package tracer +Summary: The %{longproj} Tracer service +License: ASL 2.0 +Group: Applications/System +BuildArch: noarch +Requires: %{name}-core = %{version}-%{release} +Requires: %{name}-server-base = %{version}-%{release} +Requires(post): systemd-units +Requires(preun): systemd-units +Requires(postun): systemd-units + +%description tracer + %{longproj} is a sorted, distributed key/value store based on Google's +BigTable design. It is built on top of Apache Hadoop, Zookeeper, and Thrift. It +features a few novel improvements on the BigTable design in the form of +cell-level access labels and a server-side programming mechanism that can +modify key/value pairs at various points in the data management process. + +This package provides the tracer service for %{longproj}. + +%package examples +Summary: Examples for %{longproj} +License: ASL 2.0 +Group: Applications/System +BuildArch: noarch +Requires: %{name}-core = %{version}-%{release} + +%description examples + %{longproj} is a sorted, distributed key/value store based on Google's +BigTable design. It is built on top of Apache Hadoop, Zookeeper, and Thrift. It +features a few novel improvements on the BigTable design in the form of +cell-level access labels and a server-side programming mechanism that can +modify key/value pairs at various points in the data management process. + +This package provides examples for %{longproj}. + +%package native +Summary: Native libraries for %{longproj} +License: ASL 2.0 +Group: Development/Libraries +Requires: %{name}-tserver = %{version}-%{release} + +%description native + %{longproj} is a sorted, distributed key/value store based on Google's +BigTable design. It is built on top of Apache Hadoop, Zookeeper, and Thrift. It +features a few novel improvements on the BigTable design in the form of +cell-level access labels and a server-side programming mechanism that can +modify key/value pairs at various points in the data management process. + +This package provides native code for %{longproj}'s TServer. + +%prep +%autosetup -p1 +# Remove flot and jquery bundling from upstream tarball +rm -rf server/monitor/src/main/resources/web/flot/ + +# Update dependency versions +%pom_xpath_set "pom:project/pom:dependencyManagement/pom:dependencies/pom:dependency[pom:artifactId='jline']/pom:version" "2.10" +%pom_xpath_set "pom:project/pom:dependencyManagement/pom:dependencies/pom:dependency[pom:artifactId='zookeeper']/pom:version" "3.4.5" +%pom_xpath_set "pom:project/pom:dependencyManagement/pom:dependencies/pom:dependency[pom:artifactId='libthrift']/pom:version" "0.9.1" +%pom_xpath_set "pom:project/pom:dependencyManagement/pom:dependencies/pom:dependency[pom:artifactId='log4j']/pom:version" "1.2.17" +%pom_xpath_set "pom:project/pom:dependencyManagement/pom:dependencies/pom:dependency[pom:artifactId='commons-math']/pom:version" "3.2" +%pom_xpath_set "pom:project/pom:dependencyManagement/pom:dependencies/pom:dependency[pom:artifactId='commons-math']/pom:artifactId" "commons-math3" +%pom_xpath_set "pom:project/pom:dependencies/pom:dependency[pom:artifactId='commons-math']/pom:artifactId" "commons-math3" core +%pom_xpath_set "pom:project/pom:dependencyManagement/pom:dependencies/pom:dependency[pom:artifactId='bcprov-jdk15on']/pom:artifactId" "bcprov-jdk16" + +# Remove enforcer animal-sniffer rule +%pom_xpath_remove "pom:project/pom:build/pom:pluginManagement/pom:plugins/pom:plugin[pom:artifactId='maven-enforcer-plugin']/pom:dependencies" +%pom_xpath_remove "pom:project/pom:build/pom:plugins/pom:plugin[pom:artifactId='maven-enforcer-plugin']/pom:executions/pom:execution[pom:id='enforce-java-signatures']" + +# Disable unneeded/unused modules +%pom_disable_module test +%pom_disable_module proxy +%pom_disable_module maven-plugin +%pom_disable_module docs +%pom_disable_module assemble +# Mini isn't needed +%pom_disable_module minicluster + +# Remove unneeded plugins +%pom_remove_plugin :maven-checkstyle-plugin +%pom_remove_plugin :maven-site-plugin +%pom_remove_plugin :maven-failsafe-plugin +%pom_remove_plugin :apache-rat-plugin +%pom_remove_plugin :sortpom-maven-plugin +%pom_remove_plugin :mavanagaiata +%pom_remove_plugin :findbugs-maven-plugin +%pom_remove_plugin :maven-java-formatter-plugin +%pom_remove_plugin :modernizer-maven-plugin + +# Mini isn't needed +#%%mvn_package ":%%{name}-minicluster" __noinstall +%mvn_package ":%{name}-{project,core,fate,trace,start}" core +%mvn_package ":%{name}-examples-simple" examples +%mvn_package ":%{name}-gc" gc +%mvn_package ":%{name}-master" master +%mvn_package ":%{name}-monitor" monitor +%mvn_package ":%{name}-server-base" server-base +%mvn_package ":%{name}-tracer" tracer +%mvn_package ":%{name}-tserver" tserver + +# build native, but skip install; JNI *.so is copied manually +%mvn_package ":%{name}-native" __noinstall + +%build +# TODO Unit tests are skipped, because upstream tries to do some integration +# testing in the unit tests, and they expect certain resources and dependencies +# that are not typically available, or are too complicated to configure, +# especially in the start jar. These should be enabled when possible. +# ITs are skipped, because they time out frequently and take too many resources +# to run reliably. Failures do not reliably indicate meaningful issues. +%mvn_build -j -- -DforkCount=1C -DskipTests -DskipITs + +%install +%mvn_install + +# create symlink for system-provided web assets to be added to classpath +install -d -m 755 %{buildroot}%{_datadir}/%{name}/lib/web +rm -f %{buildroot}%{_datadir}/%{name}/lib/web/flot +ln -s %{_usr}/lib/node_modules/flot %{buildroot}%{_datadir}/%{name}/lib/web/flot + +# native libs +install -d -m 755 %{buildroot}%{_libdir}/%{name} +install -d -m 755 %{buildroot}%{_var}/cache/%{name} +install -p -m 755 server/native/target/%{name}-native-%{version}/%{name}-native-%{version}/lib%{name}.so %{buildroot}%{_libdir}/%{name} + +# generate default config for Fedora from upstream examples +install -d -m 755 %{buildroot}%{_sysconfdir}/%{name} +install -d -m 755 %{buildroot}%{_sysconfdir}/%{name}/lib +install -d -m 755 %{buildroot}%{_sysconfdir}/%{name}/lib/ext +assemble/bin/bootstrap_config.sh -o -d %{buildroot}%{_sysconfdir}/%{name} -s 3GB -n -v 2 +for x in gc masters monitor slaves tracers %{name}-env.sh generic_logger.xml generic_logger.properties monitor_logger.xml monitor_logger.properties %{name}.policy.example; do rm -f %{buildroot}%{_sysconfdir}/%{name}/$x; done +cp %{buildroot}%{_sysconfdir}/%{name}/log4j.properties %{buildroot}%{_sysconfdir}/%{name}/generic_logger.properties +cp %{buildroot}%{_sysconfdir}/%{name}/log4j.properties %{buildroot}%{_sysconfdir}/%{name}/monitor_logger.properties + +# main launcher +%jpackage_script %{main_class} "" "" %{name}:%{name}/%{name}-tserver:jetty:servlet:avro/avro:apache-commons-io:apache-commons-cli:apache-commons-codec:apache-commons-collections:apache-commons-configuration:apache-commons-lang:apache-commons-logging:apache-commons-math:apache-commons-vfs:beust-jcommander:google-gson:guava:hadoop/hadoop-auth:hadoop/hadoop-common:hadoop/hadoop-hdfs:jansi/jansi:jline/jline:libthrift:log4j-1.2.17:slf4j/slf4j-api:slf4j/slf4j-log4j12:zookeeper/zookeeper:protobuf-java %{name} true +# fixup the generated jpackage script +sed -i -e 's/^#!\/bin\/sh$/#!\/usr\/bin\/bash/' %{buildroot}%{_bindir}/%{name} +# ensure the java configuration options know which service is being called +sed -i -e 's/^\s*\.\s\s*\/etc\/java\/'%{name}'\.conf/& "\$1"/' %{buildroot}%{_bindir}/%{name} +sed -i -e 's/^\s*\.\s\s*\$HOME\/\.'%{name}'rc$/& "\$1"/' %{buildroot}%{_bindir}/%{name} +# options may have spaces in them, so replace run with an exec that properly +# parses arguments as arrays. +sed -i -e '/^run .*$/d' %{buildroot}%{_bindir}/%{name} +sed -i -e '/^set_flags .*$/d' %{buildroot}%{_bindir}/%{name} +sed -i -e '/^set_options .*$/d' %{buildroot}%{_bindir}/%{name} +cat <>%{buildroot}%{_bindir}/%{name} +CLASSPATH="%{_sysconfdir}/%{name}:%{_datadir}/%{name}/lib/:\${CLASSPATH}" +set_javacmd + +if [ -n "\${VERBOSE}" ]; then + echo "Java virtual machine used: \${JAVACMD}" + echo "classpath used: \${CLASSPATH}" + echo "main class used: \${MAIN_CLASS}" + echo "flags used: \${FLAGS[*]}" + echo "options used: \${ACCUMULO_OPTS[*]}" + echo "arguments used: \${*}" +fi + +export CLASSPATH +exec "\${JAVACMD}" "\${FLAGS[@]}" "\${ACCUMULO_OPTS[@]}" "\${MAIN_CLASS}" "\${@}" +EOF + +# scripts for services/utilities +for service in master tserver shell init admin gc tracer classpath version rfile-info login-info zookeeper create-token info jar; do + cat <"%{name}-$service" +#!/usr/bin/bash +echo "%{name}-$service script is deprecated. Use '%{name} $service' instead." 1>&2 +%{_bindir}/%{name} $service "\$@" +EOF + install -p -m 755 %{name}-$service %{buildroot}%{_bindir} +done + +# systemd services +install -d -m 755 %{buildroot}%{_unitdir} +install -p -m 644 %{SOURCE1} %{buildroot}%{_unitdir}/%{name}-master.service +install -p -m 644 %{SOURCE2} %{buildroot}%{_unitdir}/%{name}-tserver.service +install -p -m 644 %{SOURCE3} %{buildroot}%{_unitdir}/%{name}-gc.service +install -p -m 644 %{SOURCE4} %{buildroot}%{_unitdir}/%{name}-tracer.service +install -p -m 644 %{SOURCE5} %{buildroot}%{_unitdir}/%{name}-monitor.service + +# java configuration file for Fedora +install -d -m 755 %{buildroot}%{_javaconfdir} +install -p -m 755 %{SOURCE6} %{buildroot}%{_javaconfdir}/%{name}.conf + +%files + +%files core -f .mfiles-core +%doc LICENSE +%doc README +%doc NOTICE +%dir %{_javadir}/%{name} +%dir %{_mavenpomdir}/%{name} +%dir %{_datadir}/%{name} +%dir %{_datadir}/%{name}/lib +%{_bindir}/%{name} +%{_bindir}/%{name}-shell +%{_bindir}/%{name}-classpath +%{_bindir}/%{name}-version +%{_bindir}/%{name}-rfile-info +%{_bindir}/%{name}-login-info +%{_bindir}/%{name}-zookeeper +%{_bindir}/%{name}-create-token +%{_bindir}/%{name}-info +%{_bindir}/%{name}-jar +%attr(0750, %{name}, -) %dir %{_var}/cache/%{name} +%attr(0755, %{name}, -) %dir %{_sysconfdir}/%{name} +%attr(0755, %{name}, -) %dir %{_sysconfdir}/%{name}/lib +%attr(0755, %{name}, -) %dir %{_sysconfdir}/%{name}/lib/ext +%attr(0755, %{name}, -) %config(noreplace) %{_javaconfdir}/%{name}.conf +%attr(0640, %{name}, -) %config(noreplace) %{_sysconfdir}/%{name}/%{name}-metrics.xml +%attr(0640, %{name}, -) %config(noreplace) %{_sysconfdir}/%{name}/%{name}-site.xml +%attr(0640, %{name}, -) %config(noreplace) %{_sysconfdir}/%{name}/auditLog.xml +%attr(0640, %{name}, -) %config(noreplace) %{_sysconfdir}/%{name}/generic_logger.properties +%attr(0644, %{name}, -) %config(noreplace) %{_sysconfdir}/%{name}/log4j.properties +%attr(0640, %{name}, -) %config(noreplace) %{_sysconfdir}/%{name}/monitor_logger.properties + +%files server-base -f .mfiles-server-base +%{_bindir}/%{name}-init +%{_bindir}/%{name}-admin + +%files master -f .mfiles-master +%{_bindir}/%{name}-master +%{_unitdir}/%{name}-master.service + +%files tserver -f .mfiles-tserver +%dir %{_jnidir}/%{name} +%{_bindir}/%{name}-tserver +%{_unitdir}/%{name}-tserver.service + +%files gc -f .mfiles-gc +%{_bindir}/%{name}-gc +%{_unitdir}/%{name}-gc.service + +%files monitor -f .mfiles-monitor +%dir %{_datadir}/%{name}/lib/web +%{_datadir}/%{name}/lib/web/flot +%{_unitdir}/%{name}-monitor.service + +%files tracer -f .mfiles-tracer +%{_bindir}/%{name}-tracer +%{_unitdir}/%{name}-tracer.service + +%files examples -f .mfiles-examples + +%files native +%dir %{_libdir}/%{name} +%{_libdir}/%{name}/lib%{name}.so + +%preun master +%systemd_preun %{name}-master.service + +%preun tserver +%systemd_preun %{name}-tserver.service + +%preun gc +%systemd_preun %{name}-gc.service + +%preun tracer +%systemd_preun %{name}-tracer.service + +%preun monitor +%systemd_preun %{name}-monitor.service + +%postun master +%systemd_postun_with_restart %{name}-master.service + +%postun tserver +%systemd_postun_with_restart %{name}-tserver.service + +%postun gc +%systemd_postun_with_restart %{name}-gc.service + +%postun tracer +%systemd_postun_with_restart %{name}-tracer.service + +%postun monitor +%systemd_postun_with_restart %{name}-monitor.service + +%pre core +getent group %{name} >/dev/null || /usr/sbin/groupadd -r %{name} +getent passwd %{name} >/dev/null || /usr/sbin/useradd --comment "%{longproj}" --shell /sbin/nologin -M -r -g %{name} --home %{_var}/cache/%{name} %{name} + +%post master +%systemd_post %{name}-master.service + +%post tserver +%systemd_post %{name}-tserver.service + +%post gc +%systemd_post %{name}-gc.service + +%post tracer +%systemd_post %{name}-tracer.service + +%post monitor +%systemd_post %{name}-monitor.service + +%changelog +* Tue Dec 06 2016 Christopher Tubbs - 1.6.6-13 +- Fix missing protobuf-java and hadoop config classpath bug, and systemd exit + code problems + +* Mon Dec 05 2016 Mike Miller - 1.6.6-12 +- Another fix for Shell erroneously reading accumulo-site.xml + +* Fri Dec 02 2016 Christopher Tubbs - 1.6.6-11 +- Vastly simplify out-of-box configuration and fix missing avro jar + +* Fri Dec 02 2016 Christopher Tubbs - 1.6.6-10 +- Add google-gson to classpath for monitor REST service + +* Fri Dec 02 2016 Christopher Tubbs - 1.6.6-9 +- Include Monitor (bz#1132725) + +* Thu Nov 03 2016 Christopher Tubbs - 1.6.6-8 +- Re-enable VFS 2.1 HDFS Provider (bz#1387110) + +* Wed Nov 02 2016 Mike Miller - 1.6.6-7 +- Fix for Shell erroneously reading accumulo-site.xml + +* Fri Oct 28 2016 Christopher Tubbs - 1.6.6-6 +- fix classpath (bz#1389325) and log to console + +* Thu Oct 20 2016 Christopher Tubbs - 1.6.6-5 +- Use commons-vfs 2.1 patch from upstream for f25+ + +* Thu Oct 20 2016 Christopher Tubbs - 1.6.6-4 +- Fix whitespace in native patch for fuzz=0 opt in f25+ + +* Thu Oct 20 2016 Christopher Tubbs - 1.6.6-3 +- Remove animal sniffer enforcer rule + +* Thu Oct 20 2016 Christopher Tubbs - 1.6.6-2 +- Remove modernizer plugin + +* Wed Oct 19 2016 Christopher Tubbs - 1.6.6-1 +- Update to 1.6.6 + +* Wed Feb 03 2016 Fedora Release Engineering - 1.6.4-5 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_24_Mass_Rebuild + +* Thu Nov 12 2015 Christopher Tubbs - 1.6.4-4 +- Use autosetup macro to apply patches + +* Thu Nov 05 2015 Christopher Tubbs - 1.6.4-3 +- Remove unnecessary checkstyle plugin + +* Thu Nov 05 2015 Christopher Tubbs - 1.6.4-2 +- Fix patches for 1.6.4 + +* Thu Nov 05 2015 Christopher Tubbs - 1.6.4-1 +- Update to 1.6.4 + +* Thu Jun 25 2015 Christopher Tubbs - 1.6.2-1 +- Update to 1.6.2 bugfix release + +* Tue Jun 16 2015 Fedora Release Engineering - 1.6.1-5 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_23_Mass_Rebuild + +* Mon May 04 2015 Kalev Lember - 1.6.1-4 +- Rebuilt for GCC 5 C++11 ABI change + +* Wed Apr 22 2015 Peter Robinson 1.6.1-3 +- ARMv7 now has hadoop + +* Tue Dec 16 2014 Christopher Tubbs - 1.6.1-2 +- Remove mortbay Jetty deps + +* Tue Dec 16 2014 Christopher Tubbs - 1.6.1-1 +- Update to 1.6.1 + +* Sun Sep 7 2014 Ville Skyttä - 1.6.0-7 +- Fix -debuginfo + +* Thu Aug 21 2014 Christopher Tubbs - 1.6.0-6 +- Skip javadoc generation in mvn_build when not used + +* Wed Aug 20 2014 Christopher Tubbs - 1.6.0-5 +- Use jpackage_script macro, standard java env, and working example config + +* Fri Aug 15 2014 Fedora Release Engineering - 1.6.0-4 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_21_22_Mass_Rebuild + +* Wed Jul 16 2014 Christopher Tubbs - 1.6.0-3 +- Fix broken service launch scripts +- Add conditional for lib directory to build for f20 + +* Wed Jul 9 2014 Christopher Tubbs - 1.6.0-2 +- Add conditional for pom directory to build for f20 +- Remove fno-strict-aliasing flag based on upstream ACCUMULO-2762 + +* Wed Apr 30 2014 Christopher Tubbs - 1.6.0-1 +- Initial packaging diff --git a/commons-configuration.patch b/commons-configuration.patch new file mode 100644 index 0000000..84707fc --- /dev/null +++ b/commons-configuration.patch @@ -0,0 +1,27 @@ +diff --git a/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java b/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java +index e512936..b78e958 100644 +--- a/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java ++++ b/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java +@@ -713,7 +713,8 @@ public class MiniAccumuloClusterImpl implements AccumuloCluster { + + @Override + public ClientConfiguration getClientConfig() { +- return new ClientConfiguration(Arrays.asList(new MapConfiguration(config.getSiteConfig()))).withInstance(this.getInstanceName()).withZkHosts( ++ MapConfiguration array = new MapConfiguration((Map)(Map)config.getSiteConfig()); ++ return new ClientConfiguration(Arrays.asList(array)).withInstance(this.getInstanceName()).withZkHosts( + this.getZooKeepers()); + } + +diff --git a/pom.xml b/pom.xml +index 3d0d903..8c3d880 100644 +--- a/pom.xml ++++ b/pom.xml +@@ -179,7 +179,7 @@ + + commons-configuration + commons-configuration +- 1.6 ++ 1.9 + + + commons-httpclient diff --git a/commons-math.patch b/commons-math.patch new file mode 100644 index 0000000..a743931 --- /dev/null +++ b/commons-math.patch @@ -0,0 +1,36 @@ +diff --git a/core/src/main/java/org/apache/accumulo/core/util/Stat.java b/core/src/main/java/org/apache/accumulo/core/util/Stat.java +index d2d560e..2fdfd62 100644 +--- a/core/src/main/java/org/apache/accumulo/core/util/Stat.java ++++ b/core/src/main/java/org/apache/accumulo/core/util/Stat.java +@@ -16,12 +16,12 @@ + */ + package org.apache.accumulo.core.util; + +-import org.apache.commons.math.stat.descriptive.StorelessUnivariateStatistic; +-import org.apache.commons.math.stat.descriptive.moment.Mean; +-import org.apache.commons.math.stat.descriptive.moment.StandardDeviation; +-import org.apache.commons.math.stat.descriptive.rank.Max; +-import org.apache.commons.math.stat.descriptive.rank.Min; +-import org.apache.commons.math.stat.descriptive.summary.Sum; ++import org.apache.commons.math3.stat.descriptive.StorelessUnivariateStatistic; ++import org.apache.commons.math3.stat.descriptive.moment.Mean; ++import org.apache.commons.math3.stat.descriptive.moment.StandardDeviation; ++import org.apache.commons.math3.stat.descriptive.rank.Max; ++import org.apache.commons.math3.stat.descriptive.rank.Min; ++import org.apache.commons.math3.stat.descriptive.summary.Sum; + + public class Stat { + Min min; +diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java +index d2d560e..2fdfd62 100644 +--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java ++++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java +@@ -64,7 +64,7 @@ + import org.apache.accumulo.core.iterators.system.LocalityGroupIterator.LocalityGroup; + import org.apache.accumulo.core.util.MutableByteSequence; + import org.apache.commons.lang.mutable.MutableLong; +-import org.apache.commons.math.stat.descriptive.SummaryStatistics; ++import org.apache.commons.math3.stat.descriptive.SummaryStatistics; + import org.apache.hadoop.io.Writable; + import org.apache.log4j.Logger; + diff --git a/dead.package b/dead.package deleted file mode 100644 index 5204a84..0000000 --- a/dead.package +++ /dev/null @@ -1 +0,0 @@ -Orphaned for 6+ weeks diff --git a/default-conf.patch b/default-conf.patch new file mode 100644 index 0000000..1bc7f28 --- /dev/null +++ b/default-conf.patch @@ -0,0 +1,100 @@ +diff --git a/assemble/conf/templates/accumulo-metrics.xml b/assemble/conf/templates/accumulo-metrics.xml +index ec54994..04edb84 100644 +--- a/assemble/conf/templates/accumulo-metrics.xml ++++ b/assemble/conf/templates/accumulo-metrics.xml +@@ -23,7 +23,8 @@ + Metrics log directory + --> + +- ${ACCUMULO_HOME}/metrics ++ ++ /var/log/accumulo/metrics + + + + +- instance.zookeeper.host +- localhost:2181 +- comma separated list of zookeeper servers ++ instance.volumes ++ file:///tmp/accumulo ++ comma separated list of dfs URIs + + + +- logger.dir.walog +- walogs +- The property only needs to be set if upgrading from 1.4 which used to store write-ahead logs on the local +- filesystem. In 1.5 write-ahead logs are stored in DFS. When 1.5 is started for the first time it will copy any 1.4 +- write ahead logs into DFS. It is possible to specify a comma-separated list of directories. +- ++ instance.zookeeper.host ++ localhost:2181 ++ comma separated list of zookeeper servers + + + +@@ -89,49 +86,13 @@ + ${mvnProjBaseDir} + + general.classpaths +- +- +- +- $ACCUMULO_HOME/lib/accumulo-server.jar, +- $ACCUMULO_HOME/lib/accumulo-core.jar, +- $ACCUMULO_HOME/lib/accumulo-start.jar, +- $ACCUMULO_HOME/lib/accumulo-fate.jar, +- $ACCUMULO_HOME/lib/accumulo-proxy.jar, +- $ACCUMULO_HOME/lib/[^.].*.jar, +- +- $ZOOKEEPER_HOME/zookeeper[^.].*.jar, +- +- $HADOOP_CONF_DIR, +- +- $HADOOP_PREFIX/share/hadoop/common/[^.].*.jar, +- $HADOOP_PREFIX/share/hadoop/common/lib/(?!slf4j)[^.].*.jar, +- $HADOOP_PREFIX/share/hadoop/hdfs/[^.].*.jar, +- $HADOOP_PREFIX/share/hadoop/mapreduce/[^.].*.jar, +- $HADOOP_PREFIX/share/hadoop/yarn/[^.].*.jar, +- $HADOOP_PREFIX/share/hadoop/yarn/lib/jersey.*.jar, +- +- +- /usr/lib/hadoop/[^.].*.jar, +- /usr/lib/hadoop/lib/[^.].*.jar, +- /usr/lib/hadoop-hdfs/[^.].*.jar, +- /usr/lib/hadoop-mapreduce/[^.].*.jar, +- /usr/lib/hadoop-yarn/[^.].*.jar, +- /usr/lib/hadoop-yarn/lib/jersey.*.jar, +- +- +- /usr/hdp/current/hadoop-client/[^.].*.jar, +- /usr/hdp/current/hadoop-client/lib/(?!slf4j)[^.].*.jar, +- /usr/hdp/current/hadoop-hdfs-client/[^.].*.jar, +- /usr/hdp/current/hadoop-mapreduce-client/[^.].*.jar, +- /usr/hdp/current/hadoop-yarn-client/[^.].*.jar, +- /usr/hdp/current/hadoop-yarn-client/lib/jersey.*.jar, +- /usr/hdp/current/hive-client/lib/hive-accumulo-handler.jar +- +- +- $HADOOP_PREFIX/[^.].*.jar, +- $HADOOP_PREFIX/lib/(?!slf4j)[^.].*.jar, +- +- ++ /etc/accumulo/lib,/etc/hadoop + Classpaths that accumulo checks for updates and class files. + ++ ++ ++ general.dynamic.classpaths ++ /etc/accumulo/lib/ext ++ Classpaths that accumulo checks for updates and class files to dynamically load. ++ + diff --git a/disabled-tests.patch b/disabled-tests.patch new file mode 100644 index 0000000..fe25c8f --- /dev/null +++ b/disabled-tests.patch @@ -0,0 +1,40 @@ +diff --git a/core/src/test/java/org/apache/accumulo/core/util/shell/ShellSetInstanceTest.java b/core/src/test/java/org/apache/accumulo/core/util/shell/ShellSetInstanceTest.java +index 0453beb..84e4ad9 100644 +--- a/core/src/test/java/org/apache/accumulo/core/util/shell/ShellSetInstanceTest.java ++++ b/core/src/test/java/org/apache/accumulo/core/util/shell/ShellSetInstanceTest.java +@@ -53,6 +53,7 @@ import org.junit.After; + import org.junit.AfterClass; + import org.junit.Before; + import org.junit.BeforeClass; ++import org.junit.Ignore; + import org.junit.Test; + import org.junit.runner.RunWith; + import org.powermock.core.classloader.annotations.PrepareForTest; +@@ -60,6 +61,7 @@ import org.powermock.modules.junit4.PowerMockRunner; + + @RunWith(PowerMockRunner.class) + @PrepareForTest({Shell.class, ZooUtil.class, ConfigSanityCheck.class}) ++@Ignore // test skipped due to https://bugzilla.redhat.com/show_bug.cgi?id=1096992 + public class ShellSetInstanceTest { + public static class TestOutputStream extends OutputStream { + StringBuilder sb = new StringBuilder(); +diff --git a/core/src/test/java/org/apache/accumulo/core/util/shell/commands/HistoryCommandTest.java b/core/src/test/java/org/apache/accumulo/core/util/shell/commands/HistoryCommandTest.java +index 9b98e4a..696a090 100644 +--- a/core/src/test/java/org/apache/accumulo/core/util/shell/commands/HistoryCommandTest.java ++++ b/core/src/test/java/org/apache/accumulo/core/util/shell/commands/HistoryCommandTest.java +@@ -33,6 +33,7 @@ import org.apache.accumulo.core.util.shell.Shell; + import org.apache.commons.cli.CommandLine; + import org.junit.Assume; + import org.junit.Before; ++import org.junit.Ignore; + import org.junit.Test; + + public class HistoryCommandTest { +@@ -76,6 +77,7 @@ public class HistoryCommandTest { + } + + @Test ++ @Ignore // this test doesn't work in rpmbuild environment for some yet unknown reason + public void testEventExpansion() throws IOException { + // If we use an unsupported terminal, then history expansion doesn't work because JLine can't do magic buffer manipulations. + // This has been observed to be the case on certain versions of Eclipse. However, mvn is usually fine. diff --git a/flot8.patch b/flot8.patch new file mode 100644 index 0000000..b78f15e --- /dev/null +++ b/flot8.patch @@ -0,0 +1,12 @@ +diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/BasicServlet.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/BasicServlet.java +index 75d5436..9660340 100644 +--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/BasicServlet.java ++++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/BasicServlet.java +@@ -156,6 +156,7 @@ abstract public class BasicServlet extends HttpServlet { + sb.append("\n"); + sb.append("\n"); + sb.append("\n"); ++ sb.append("\n"); + + sb.append("\n"); + diff --git a/jetty9.patch b/jetty9.patch new file mode 100644 index 0000000..1ecb1c7 --- /dev/null +++ b/jetty9.patch @@ -0,0 +1,194 @@ +diff --git a/assemble/pom.xml b/assemble/pom.xml +index db26388..49d127f 100644 +--- a/assemble/pom.xml ++++ b/assemble/pom.xml +@@ -126,10 +126,6 @@ + + + org.eclipse.jetty +- jetty-continuation +- +- +- org.eclipse.jetty + jetty-http + + +diff --git a/assemble/src/main/assemblies/component.xml b/assemble/src/main/assemblies/component.xml +index 96e1fbe..a0fd410 100644 +--- a/assemble/src/main/assemblies/component.xml ++++ b/assemble/src/main/assemblies/component.xml +@@ -35,7 +35,6 @@ + org.apache.commons:commons-math + org.apache.commons:commons-vfs2 + org.apache.thrift:libthrift +- org.eclipse.jetty:jetty-continuation + org.eclipse.jetty:jetty-http + org.eclipse.jetty:jetty-io + org.eclipse.jetty:jetty-security +diff --git a/pom.xml b/pom.xml +index bd206e3..3d0d903 100644 +--- a/pom.xml ++++ b/pom.xml +@@ -131,7 +131,7 @@ + 2.2.0 + 3.1 + false +- 8.1.15.v20140411 ++ 9.1.5.v20140505 + 1.6 + 1.6 + 3.0.5 +@@ -204,7 +204,7 @@ + + javax.servlet + javax.servlet-api +- 3.0.1 ++ 3.1.0 + + + javax.ws.rs +@@ -421,11 +421,6 @@ + + + org.eclipse.jetty +- jetty-continuation +- ${jetty.version} +- +- +- org.eclipse.jetty + jetty-http + ${jetty.version} + +@@ -454,12 +449,6 @@ + jetty-util + ${jetty.version} + +- +- +- org.mortbay.jetty +- jetty +- 6.1.26 +- + + org.powermock + powermock-api-easymock +diff --git a/server/monitor/pom.xml b/server/monitor/pom.xml +index f3dcd4a..e8146be 100644 +--- a/server/monitor/pom.xml ++++ b/server/monitor/pom.xml +@@ -81,10 +81,6 @@ + + + org.eclipse.jetty +- jetty-http +- +- +- org.eclipse.jetty + jetty-security + + +@@ -101,7 +97,7 @@ + + + org.eclipse.jetty +- jetty-continuation ++ jetty-http + runtime + + +diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/EmbeddedWebServer.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/EmbeddedWebServer.java +index 41890e8..dea263a 100644 +--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/EmbeddedWebServer.java ++++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/EmbeddedWebServer.java +@@ -23,10 +23,10 @@ import org.apache.accumulo.core.conf.Property; + import org.apache.commons.lang.StringUtils; + import org.eclipse.jetty.security.ConstraintMapping; + import org.eclipse.jetty.security.ConstraintSecurityHandler; ++import org.eclipse.jetty.server.HttpConnectionFactory; + import org.eclipse.jetty.server.Server; +-import org.eclipse.jetty.server.nio.SelectChannelConnector; ++import org.eclipse.jetty.server.ServerConnector; + import org.eclipse.jetty.server.session.SessionHandler; +-import org.eclipse.jetty.server.ssl.SslSelectChannelConnector; + import org.eclipse.jetty.servlet.ServletContextHandler; + import org.eclipse.jetty.util.security.Constraint; + import org.eclipse.jetty.util.ssl.SslContextFactory; +@@ -35,7 +35,7 @@ public class EmbeddedWebServer { + private static String EMPTY = ""; + + Server server = null; +- SelectChannelConnector connector = null; ++ ServerConnector connector = null; + ServletContextHandler handler; + boolean usingSsl; + +@@ -48,13 +48,13 @@ public class EmbeddedWebServer { + final AccumuloConfiguration conf = Monitor.getSystemConfiguration(); + if (EMPTY.equals(conf.get(Property.MONITOR_SSL_KEYSTORE)) || EMPTY.equals(conf.get(Property.MONITOR_SSL_KEYSTOREPASS)) + || EMPTY.equals(conf.get(Property.MONITOR_SSL_TRUSTSTORE)) || EMPTY.equals(conf.get(Property.MONITOR_SSL_TRUSTSTOREPASS))) { +- connector = new SelectChannelConnector(); ++ connector = new ServerConnector(server, new HttpConnectionFactory()); + usingSsl = false; + } else { + SslContextFactory sslContextFactory = new SslContextFactory(); + sslContextFactory.setKeyStorePath(conf.get(Property.MONITOR_SSL_KEYSTORE)); + sslContextFactory.setKeyStorePassword(conf.get(Property.MONITOR_SSL_KEYSTOREPASS)); +- sslContextFactory.setTrustStore(conf.get(Property.MONITOR_SSL_TRUSTSTORE)); ++ sslContextFactory.setTrustStorePath(conf.get(Property.MONITOR_SSL_TRUSTSTORE)); + sslContextFactory.setTrustStorePassword(conf.get(Property.MONITOR_SSL_TRUSTSTOREPASS)); + + final String includedCiphers = conf.get(Property.MONITOR_SSL_INCLUDE_CIPHERS); +@@ -72,7 +72,7 @@ public class EmbeddedWebServer { + sslContextFactory.setIncludeProtocols(StringUtils.split(includeProtocols, ',')); + } + +- connector = new SslSelectChannelConnector(sslContextFactory); ++ connector = new ServerConnector(server, sslContextFactory); + usingSsl = true; + } + +diff --git a/start/pom.xml b/start/pom.xml +index 9f74aff..bd36449 100644 +--- a/start/pom.xml ++++ b/start/pom.xml +@@ -109,24 +109,4 @@ + + + +- +- +- +- hadoop-1 +- +- +- hadoop.profile +- 1 +- +- +- +- +- org.mortbay.jetty +- jetty +- test +- +- +- +- + +diff --git a/test/pom.xml b/test/pom.xml +index 8d8b838..13998c3 100644 +--- a/test/pom.xml ++++ b/test/pom.xml +@@ -307,11 +307,6 @@ + hadoop-tools + test + +- +- org.mortbay.jetty +- jetty +- test +- + + +