describe(`DictionaryVector`, () => {
const dictionary = ['foo', 'bar', 'baz'];
const extras = ['abc', '123']; // values to search for that should NOT be found
let offset = 0;
const offsets = Uint32Array.of(0, ...dictionary.map((d) => { offset += d.length; return offset; }));
const dictionary_vec = new Utf8Vector(new FlatListData(new Utf8(), dictionary.length, null, offsets, utf8Encoder.encode(dictionary.join(''))));
const indices = Array.from({length: 50}, () => Math.random() * 3 | 0);
describe(`index with nullCount == 0`, () => {
const indices_data = new FlatData(new Int32(), indices.length, new Uint8Array(0), indices);
const values = Array.from(indices).map((d) => dictionary[d]);
const vector = new DictionaryVector(new DictionaryData(new Dictionary(dictionary_vec.type, indices_data.type), dictionary_vec, indices_data));
basicVectorTests(vector, values, extras);
describe(`sliced`, () => {
basicVectorTests(vector.slice(10, 20), values.slice(10,20), extras);
});
});
describe(`index with nullCount > 0`, () => {
const validity = Array.from({length: indices.length}, () => Math.random() > 0.2 ? true : false);
const indices_data = new FlatData(new Int32(), indices.length, packBools(validity), indices, 0, validity.reduce((acc, d) => acc + (d ? 0 : 1), 0));
const values = Array.from(indices).map((d, i) => validity[i] ? dictionary[d] : null);
const vector = new DictionaryVector(new DictionaryData(new Dictionary(dictionary_vec.type, indices_data.type), dictionary_vec, indices_data));
basicVectorTests(vector, values, ['abc', '123']);
describe(`sliced`, () => {
basicVectorTests(vector.slice(10, 20), values.slice(10,20), extras);
});
});
});
describe(`Utf8Vector`, () => {
const values = ['foo', 'bar', 'baz', 'foo bar', 'bar'], n = values.length;
let offset = 0;
const offsets = Uint32Array.of(0, ...values.map((d) => { offset += d.length; return offset; }));
const vector = new Utf8Vector(new FlatListData(new Utf8(), n, null, offsets, utf8Encoder.encode(values.join(''))));
basicVectorTests(vector, values, ['abc', '123']);
describe(`sliced`, () => {
basicVectorTests(vector.slice(1,3), values.slice(1,3), ['foo', 'abc']);
});
});
function test_encoder() {
const text = "plain text";
let uint8array: Uint8Array;
// constructor
uint8array = new TextEncoder().encode(text);
uint8array = new TextEncoder('utf-8').encode(text);
uint8array = TextEncoder().encode(text);
uint8array = TextEncoder('utf-8').encode(text);
// attributes
const encoder = new TextEncoder();
encoder.encoding = 'utf-8';
const encoding: string = encoder.encoding;
// methods
encoder.encode();
encoder.encode(text);
}
protected readData<T extends DataType>(type: T, { offset }: BufferMetadata = this.getBufferMetadata()) {
const { sources } = this;
if (DataType.isTimestamp(type) === true) {
return new Uint8Array(int64DataFromJSON(sources[offset] as string[]));
} else if ((DataType.isInt(type) || DataType.isTime(type)) && type.bitWidth === 64) {
return new Uint8Array(int64DataFromJSON(sources[offset] as string[]));
} else if (DataType.isDate(type) && type.unit === DateUnit.MILLISECOND) {
return new Uint8Array(int64DataFromJSON(sources[offset] as string[]));
} else if (DataType.isDecimal(type) === true) {
return new Uint8Array(decimalDataFromJSON(sources[offset] as string[]));
} else if (DataType.isBinary(type) === true || DataType.isFixedSizeBinary(type) === true) {
return new Uint8Array(binaryDataFromJSON(sources[offset] as string[]));
} else if (DataType.isBool(type) === true) {
return new Uint8Array(packBools(sources[offset] as number[]).buffer);
} else if (DataType.isUtf8(type) === true) {
return utf8Encoder.encode((sources[offset] as string[]).join(''));
} else {
return toTypedArray(type.ArrayType, sources[offset].map((x) => +x)) as any;
}
}
function createDataArray<T extends TypedArray>(sources: any[][], field: Field, _fieldNode: FieldNode, buffer: Buffer, ArrayConstructor: TypedArrayConstructor<T>): T {
let type = field.type, data: ArrayLike<number> | ArrayBufferLike;
if (type.isTimestamp() === true) {
data = int64sFromJSON(sources[buffer.offset.low] as string[]);
} else if ((type.isInt() || type.isTime()) && type.bitWidth === 64) {
data = int64sFromJSON(sources[buffer.offset.low] as string[]);
} else if (type.isDate() && type.unit === DateUnit.MILLISECOND) {
data = int64sFromJSON(sources[buffer.offset.low] as string[]);
} else if (type.isDecimal() === true) {
data = decimalFromJSON(sources[buffer.offset.low] as string[]);
} else if (type.isBinary() === true) {
data = binaryFromJSON(sources[buffer.offset.low] as string[]);
} else if (type.isBool() === true) {
data = booleanFromJSON(sources[buffer.offset.low] as number[]).buffer;
} else if (type.isUtf8() === true) {
data = encoder.encode((sources[buffer.offset.low] as string[]).join(''));
} else {
data = (sources[buffer.offset.low]).map((x) => +x);
}
return new ArrayConstructor(data);
}