Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -290,32 +290,31 @@ public boolean equals(Mutation m) {
public static class Partitioner {

private Map<ByteSequence,Integer> colfamToLgidMap;
private PreAllocatedArray<Map<ByteSequence,MutableLong>> groups;
private List<Map<ByteSequence,MutableLong>> groups;

public Partitioner(PreAllocatedArray<Map<ByteSequence,MutableLong>> groups) {
public Partitioner(List<Map<ByteSequence,MutableLong>> groups) {
this.groups = groups;
this.colfamToLgidMap = new HashMap<>();

for (int i = 0; i < groups.length; i++) {
for (int i = 0; i < groups.size(); i++) {
for (ByteSequence cf : groups.get(i).keySet()) {
colfamToLgidMap.put(cf, i);
}
}
}

public void partition(List<Mutation> mutations,
PreAllocatedArray<List<Mutation>> partitionedMutations) {
public void partition(List<Mutation> mutations, List<List<Mutation>> partitionedMutations) {

final var mbs = new ArrayByteSequence(new byte[0], 0, 0);

PreAllocatedArray<List<ColumnUpdate>> parts = new PreAllocatedArray<>(groups.length + 1);
List<List<ColumnUpdate>> parts = PreallocatedList.create(groups.size() + 1);

for (Mutation mutation : mutations) {
if (mutation.getUpdates().size() == 1) {
int lgid = getLgid(mbs, mutation.getUpdates().get(0));
partitionedMutations.get(lgid).add(mutation);
} else {
for (int i = 0; i < parts.length; i++) {
for (int i = 0; i < parts.size(); i++) {
parts.set(i, null);
}

Expand All @@ -333,14 +332,14 @@ public void partition(List<Mutation> mutations,
}

if (lgcount == 1) {
for (int i = 0; i < parts.length; i++) {
for (int i = 0; i < parts.size(); i++) {
if (parts.get(i) != null) {
partitionedMutations.get(i).add(mutation);
break;
}
}
} else {
for (int i = 0; i < parts.length; i++) {
for (int i = 0; i < parts.size(); i++) {
if (parts.get(i) != null) {
partitionedMutations.get(i)
.add(new PartitionedMutation(mutation.getRow(), parts.get(i)));
Expand All @@ -355,7 +354,7 @@ private Integer getLgid(ArrayByteSequence mbs, ColumnUpdate cu) {
mbs.reset(cu.getColumnFamily(), 0, cu.getColumnFamily().length);
Integer lgid = colfamToLgidMap.get(mbs);
if (lgid == null) {
lgid = groups.length;
lgid = groups.size();
}
return lgid;
}
Expand Down

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.accumulo.core.util;

import java.util.Arrays;
import java.util.List;

public class PreallocatedList {
@SuppressWarnings("unchecked")
public static <T> List<T> create(int size) {
return Arrays.asList((T[]) new Object[size]);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ public class PartitionerTest {
@Test
public void test1() {

PreAllocatedArray<Map<ByteSequence,MutableLong>> groups = new PreAllocatedArray<>(2);
List<Map<ByteSequence,MutableLong>> groups = PreallocatedList.create(2);

groups.set(0, new HashMap<>());
groups.get(0).put(new ArrayByteSequence("cf1"), new MutableLong(1));
Expand Down Expand Up @@ -72,9 +72,9 @@ public void test1() {
m5.put("cf5", "cq3", "v9");

List<Mutation> mutations = Arrays.asList(m1, m2, m3, m4, m5);
PreAllocatedArray<List<Mutation>> partitioned = new PreAllocatedArray<>(3);
List<List<Mutation>> partitioned = PreallocatedList.create(3);

for (int i = 0; i < partitioned.length; i++) {
for (int i = 0; i < partitioned.size(); i++) {
partitioned.set(i, new ArrayList<>());
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,56 +25,56 @@
import static org.junit.jupiter.api.Assertions.assertTrue;

import java.util.Iterator;
import java.util.List;

import org.junit.jupiter.api.Test;

import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;

@SuppressFBWarnings(value = "RV_RETURN_VALUE_IGNORED_NO_SIDE_EFFECT",
justification = "lambda assertThrows testing exception thrown")
public class PreAllocatedArrayTest {
public class PreAllocatedListTest {

/**
* Test method for {@link org.apache.accumulo.core.util.PreAllocatedArray#PreAllocatedArray(int)}.
* Test method for {@link org.apache.accumulo.core.util.PreallocatedList#create(int)}.
*/
@Test
public void testPreAllocatedArray() {
PreAllocatedArray<String> strings = new PreAllocatedArray<>(5);
assertEquals(5, strings.length);
List<String> strings = PreallocatedList.create(5);
assertEquals(5, strings.size());

strings = new PreAllocatedArray<>(3);
assertEquals(3, strings.length);
strings = PreallocatedList.create(3);
assertEquals(3, strings.size());

strings = new PreAllocatedArray<>(0);
assertEquals(0, strings.length);
strings = PreallocatedList.create(0);
assertEquals(0, strings.size());
}

@Test
public void testPreAllocatedArray_Fail() {
assertThrows(NegativeArraySizeException.class, () -> new PreAllocatedArray<String>(-5));
assertThrows(NegativeArraySizeException.class, () -> PreallocatedList.create(-5));
}

/**
* Test method for
* {@link org.apache.accumulo.core.util.PreAllocatedArray#set(int, java.lang.Object)}.<br>
* Test method for {@link org.apache.accumulo.core.util.PreAllocatedArray#get(int)}.<br>
* Test method for {@link org.apache.accumulo.core.util.PreAllocatedArray#iterator()}.
* Test method for {@link List#set(int, java.lang.Object)}.<br>
* Test method for {@link List#get(int)}.<br>
* Test method for {@link List#iterator()}.
*/
@Test
public void testSet() {
int capacity = 5;
PreAllocatedArray<String> strings = new PreAllocatedArray<>(capacity);
assertEquals(capacity, strings.length);
List<String> strings = PreallocatedList.create(capacity);
assertEquals(capacity, strings.size());

// everything else should be null
strings.set(1, "a");
strings.set(4, "b");
assertEquals(capacity, strings.length);
assertEquals(capacity, strings.size());

// overwrite
String b = strings.set(4, "c");
assertEquals("b", b);
assertEquals(capacity, strings.length);
assertEquals(capacity, strings.size());

Iterator<String> iter = strings.iterator();
assertNull(iter.next()); // index 0
Expand All @@ -87,37 +87,37 @@ public void testSet() {

@Test
public void testSetIndexHigh() {
PreAllocatedArray<String> strings = new PreAllocatedArray<>(3);
List<String> strings = PreallocatedList.create(3);
strings.set(2, "in bounds");
assertThrows(IndexOutOfBoundsException.class, () -> strings.set(3, "out of bounds"));
}

@Test
public void testSetIndexNegative() {
PreAllocatedArray<String> strings = new PreAllocatedArray<>(3);
List<String> strings = PreallocatedList.create(3);
strings.set(0, "in bounds");
assertThrows(IndexOutOfBoundsException.class, () -> strings.set(-3, "out of bounds"));
}

@Test
public void testGetIndexHigh() {
PreAllocatedArray<String> strings = new PreAllocatedArray<>(3);
List<String> strings = PreallocatedList.create(3);
assertNull(strings.get(2));
// spotbugs error suppressed at class level for lambda
assertThrows(IndexOutOfBoundsException.class, () -> strings.get(3));
}

@Test
public void testGetIndexNegative() {
PreAllocatedArray<String> strings = new PreAllocatedArray<>(3);
List<String> strings = PreallocatedList.create(3);
assertNull(strings.get(0));
// spotbugs error suppressed at class level for lambda
assertThrows(IndexOutOfBoundsException.class, () -> strings.get(-3));
}

@Test
public void testIteratorRemove() {
PreAllocatedArray<String> strings = new PreAllocatedArray<>(3);
List<String> strings = PreallocatedList.create(3);
strings.set(1, "data");
var iter = strings.iterator();
for (int i = 0; i < 3; i++) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@
import org.apache.accumulo.core.util.LocalityGroupUtil;
import org.apache.accumulo.core.util.LocalityGroupUtil.Partitioner;
import org.apache.accumulo.core.util.Pair;
import org.apache.accumulo.core.util.PreAllocatedArray;
import org.apache.accumulo.core.util.PreallocatedList;
import org.apache.accumulo.server.ServerContext;
import org.apache.accumulo.server.conf.TableConfiguration;
import org.apache.commons.lang3.mutable.MutableLong;
Expand Down Expand Up @@ -280,17 +280,17 @@ public void mutate(List<Mutation> mutations, int kvCount) {

private static class LocalityGroupMap implements SimpleMap {

private final PreAllocatedArray<Map<ByteSequence,MutableLong>> groupFams;
private final List<Map<ByteSequence,MutableLong>> groupFams;

// the last map in the array is the default locality group
private final SimpleMap[] maps;
private final Partitioner partitioner;
private final PreAllocatedArray<List<Mutation>> partitioned;
private final List<List<Mutation>> partitioned;

LocalityGroupMap(Map<String,Set<ByteSequence>> groups, boolean useNativeMap) {
this.groupFams = new PreAllocatedArray<>(groups.size());
this.groupFams = PreallocatedList.create(groups.size());
this.maps = new SimpleMap[groups.size() + 1];
this.partitioned = new PreAllocatedArray<>(groups.size() + 1);
this.partitioned = PreallocatedList.create(groups.size() + 1);

for (int i = 0; i < maps.length; i++) {
maps[i] = newMap(useNativeMap);
Expand All @@ -307,7 +307,7 @@ private static class LocalityGroupMap implements SimpleMap {

partitioner = new LocalityGroupUtil.Partitioner(this.groupFams);

for (int i = 0; i < partitioned.length; i++) {
for (int i = 0; i < partitioned.size(); i++) {
partitioned.set(i, new ArrayList<>());
}
}
Expand All @@ -329,7 +329,7 @@ public InterruptibleIterator skvIterator(SamplerConfigurationImpl samplerConfig)

LocalityGroup[] groups = new LocalityGroup[maps.length];
for (int i = 0; i < groups.length; i++) {
if (i < groupFams.length) {
if (i < groupFams.size()) {
groups[i] = new LocalityGroup(maps[i].skvIterator(null), groupFams.get(i), false);
} else {
groups[i] = new LocalityGroup(maps[i].skvIterator(null), null, true);
Expand Down Expand Up @@ -364,7 +364,7 @@ public synchronized void mutate(List<Mutation> mutations, int kvCount) {
try {
partitioner.partition(mutations, partitioned);

for (int i = 0; i < partitioned.length; i++) {
for (int i = 0; i < partitioned.size(); i++) {
if (!partitioned.get(i).isEmpty()) {
maps[i].mutate(partitioned.get(i), kvCount);
for (Mutation m : partitioned.get(i)) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@
import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
import org.apache.accumulo.core.iteratorsImpl.system.InterruptibleIterator;
import org.apache.accumulo.core.iteratorsImpl.system.IterationInterruptedException;
import org.apache.accumulo.core.util.PreAllocatedArray;
import org.apache.accumulo.core.util.PreallocatedList;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand Down Expand Up @@ -181,7 +181,7 @@ private class ConcurrentIterator implements Iterator<Map.Entry<Key,Value>> {

private NMIterator source;

private PreAllocatedArray<Entry<Key,Value>> nextEntries;
private List<Entry<Key,Value>> nextEntries;
private int index;
private int end;

Expand All @@ -191,7 +191,7 @@ private class ConcurrentIterator implements Iterator<Map.Entry<Key,Value>> {

ConcurrentIterator(Key key) {
// start off with a small read ahead
nextEntries = new PreAllocatedArray<>(1);
nextEntries = PreallocatedList.create(1);

rlock.lock();
try {
Expand All @@ -214,12 +214,12 @@ private void fill() {
int amountRead = 0;

// as we keep filling, increase the read ahead buffer
if (nextEntries.length < MAX_READ_AHEAD_ENTRIES) {
if (nextEntries.size() < MAX_READ_AHEAD_ENTRIES) {
nextEntries =
new PreAllocatedArray<>(Math.min(nextEntries.length * 2, MAX_READ_AHEAD_ENTRIES));
PreallocatedList.create(Math.min(nextEntries.size() * 2, MAX_READ_AHEAD_ENTRIES));
}

while (source.hasNext() && end < nextEntries.length) {
while (source.hasNext() && end < nextEntries.size()) {
Entry<Key,Value> ne = source.next();
nextEntries.set(end++, ne);
amountRead += ne.getKey().getSize() + ne.getValue().getSize();
Expand Down