Primeiro commit do projeto Angular
This commit is contained in:
+234
@@ -0,0 +1,234 @@
|
||||
#include "lz4.h"
|
||||
#include "lmdb-js.h"
|
||||
#include <atomic>
|
||||
|
||||
using namespace Napi;
|
||||
|
||||
thread_local LZ4_stream_t* Compression::stream = nullptr;
|
||||
Compression::Compression(const CallbackInfo& info) : ObjectWrap<Compression>(info) {
|
||||
unsigned int compressionThreshold = 1000;
|
||||
char* dictionary = nullptr;
|
||||
size_t dictSize = 0;
|
||||
unsigned int startingOffset = 0;
|
||||
if (info[0].IsObject()) {
|
||||
auto dictionaryOption = info[0].As<Object>().Get("dictionary");
|
||||
if (!dictionaryOption.IsUndefined()) {
|
||||
if (!dictionaryOption.IsTypedArray()) {
|
||||
throwError(info.Env(), "Dictionary must be a buffer");
|
||||
return;
|
||||
}
|
||||
napi_get_buffer_info(info.Env(), dictionaryOption, (void**) &dictionary, &dictSize);
|
||||
dictSize = (dictSize >> 3) << 3; // make sure it is word-aligned
|
||||
}
|
||||
auto thresholdOption = info[0].As<Object>().Get("threshold");
|
||||
if (thresholdOption.IsNumber())
|
||||
compressionThreshold = thresholdOption.As<Number>();
|
||||
auto offsetOption = info[0].As<Object>().Get("startingOffset");
|
||||
if (offsetOption.IsNumber())
|
||||
startingOffset = offsetOption.As<Number>();
|
||||
}
|
||||
this->startingOffset = startingOffset;
|
||||
this->dictionary = this->compressDictionary = dictionary;
|
||||
this->dictionarySize = dictSize;
|
||||
this->decompressTarget = dictionary + dictSize;
|
||||
this->decompressSize = 0;
|
||||
this->acceleration = 1;
|
||||
this->compressionThreshold = compressionThreshold;
|
||||
info.This().As<Object>().Set("address", Number::New(info.Env(), (double) (size_t) this));
|
||||
}
|
||||
|
||||
Napi::Value Compression::setBuffer(const CallbackInfo& info) {
|
||||
size_t length;
|
||||
napi_get_arraybuffer_info(info.Env(), info[0], (void**) &this->decompressTarget, &length);
|
||||
unsigned int offset = info[1].As<Number>();
|
||||
this->decompressTarget += offset;
|
||||
this->decompressSize = info[2].As<Number>();
|
||||
this->dictionarySize = info[4].As<Number>();
|
||||
napi_get_buffer_info(info.Env(), info[3], (void**) &this->dictionary, &length);
|
||||
return info.Env().Undefined();
|
||||
}
|
||||
void Compression::decompress(MDB_val& data, bool &isValid, bool canAllocate) {
|
||||
uint32_t uncompressedLength;
|
||||
int compressionHeaderSize;
|
||||
uint32_t compressedLength = data.mv_size;
|
||||
void* originalData = data.mv_data;
|
||||
unsigned char* charData = (unsigned char*) data.mv_data + startingOffset;
|
||||
|
||||
if (charData[0] == 254) {
|
||||
uncompressedLength = ((uint32_t)charData[1] << 16) | ((uint32_t)charData[2] << 8) | (uint32_t)charData[3];
|
||||
compressionHeaderSize = 4;
|
||||
}
|
||||
else if (charData[0] == 255) {
|
||||
uncompressedLength = ((uint32_t)charData[4] << 24) | ((uint32_t)charData[5] << 16) | ((uint32_t)charData[6] << 8) | (uint32_t)charData[7];
|
||||
compressionHeaderSize = 8;
|
||||
}
|
||||
else {
|
||||
fprintf(stderr, "Unknown status byte %u\n", charData[0]);
|
||||
//if (canAllocate)
|
||||
// Nan::ThrowError("Unknown status byte");
|
||||
isValid = false;
|
||||
return;
|
||||
}
|
||||
//fprintf(stdout, "compressed size %u uncompressedLength %u, target size %u, first byte %u\n", data.mv_size, uncompressedLength + startingOffset, decompressSize, charData[compressionHeaderSize]);
|
||||
data.mv_data = decompressTarget;
|
||||
data.mv_size = uncompressedLength + startingOffset;
|
||||
//TODO: For larger blocks with known encoding, it might make sense to allocate space for it and use an ExternalString
|
||||
if (uncompressedLength + startingOffset > decompressSize) {
|
||||
isValid = false;
|
||||
return;
|
||||
}
|
||||
int written = LZ4_decompress_safe_usingDict(
|
||||
(char*)charData + compressionHeaderSize, decompressTarget + startingOffset,
|
||||
compressedLength - compressionHeaderSize - startingOffset, decompressSize - startingOffset,
|
||||
dictionary, dictionarySize);
|
||||
//fprintf(stdout, "first uncompressed byte %X %X %X %X %X %X\n", uncompressedData[0], uncompressedData[1], uncompressedData[2], uncompressedData[3], uncompressedData[4], uncompressedData[5]);
|
||||
if (written < 0) {
|
||||
fprintf(stderr, "Failed to decompress data %u %u bytes:\n", compressionHeaderSize, uncompressedLength);
|
||||
for (uint32_t i = 0; i < compressedLength; i++) {
|
||||
fprintf(stderr, "%u ", charData[i]);
|
||||
}
|
||||
//if (canAllocate)
|
||||
// Nan::ThrowError("Failed to decompress data");
|
||||
isValid = false;
|
||||
return;
|
||||
}
|
||||
if (startingOffset)
|
||||
memcpy(decompressTarget, originalData, startingOffset);
|
||||
isValid = true;
|
||||
}
|
||||
|
||||
int Compression::compressInstruction(EnvWrap* env, double* compressionAddress) {
|
||||
MDB_val value;
|
||||
value.mv_data = (void*)((size_t) * (compressionAddress - 1));
|
||||
value.mv_size = *(((uint32_t*)compressionAddress) - 3);
|
||||
argtokey_callback_t compressedData = compress(&value, nullptr);
|
||||
if (compressedData) {
|
||||
*(((uint32_t*)compressionAddress) - 3) = value.mv_size;
|
||||
*((size_t*)(compressionAddress - 1)) = (size_t)value.mv_data;
|
||||
int64_t status = std::atomic_exchange((std::atomic<int64_t>*) compressionAddress, (int64_t) 0);
|
||||
if (status == 1 && env) {
|
||||
pthread_mutex_lock(env->writingLock);
|
||||
pthread_cond_signal(env->writingCond);
|
||||
pthread_mutex_unlock(env->writingLock);
|
||||
//fprintf(stderr, "sent compression completion signal\n");
|
||||
}
|
||||
//fprintf(stdout, "compressed to %p %u %u %p\n", value.mv_data, value.mv_size, status, env);
|
||||
return 0;
|
||||
} else {
|
||||
fprintf(stdout, "failed to compress\n");
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
argtokey_callback_t Compression::compress(MDB_val* value, void (*freeValue)(MDB_val&)) {
|
||||
size_t dataLength = value->mv_size - startingOffset;
|
||||
char* data = (char*)value->mv_data;
|
||||
if (value->mv_size < compressionThreshold && !(value->mv_size > startingOffset && ((uint8_t*)data)[startingOffset] >= 250))
|
||||
return freeValue; // don't compress if less than threshold (but we must compress if the first byte is the compression indicator)
|
||||
bool longSize = dataLength >= 0x1000000;
|
||||
int prefixSize = (longSize ? 8 : 4) + startingOffset;
|
||||
int maxCompressedSize = LZ4_COMPRESSBOUND(dataLength);
|
||||
char* compressed = new char[maxCompressedSize + prefixSize];
|
||||
//fprintf(stdout, "compressing %u\n", dataLength);
|
||||
if (!stream)
|
||||
stream = LZ4_createStream();
|
||||
LZ4_loadDict(stream, compressDictionary, dictionarySize);
|
||||
int compressedSize = LZ4_compress_fast_continue(stream, data + startingOffset, compressed + prefixSize, dataLength, maxCompressedSize, acceleration);
|
||||
if (compressedSize > 0) {
|
||||
if (startingOffset > 0) // copy the uncompressed prefix
|
||||
memcpy(compressed, data, startingOffset);
|
||||
if (freeValue)
|
||||
freeValue(*value);
|
||||
uint8_t* compressedData = (uint8_t*)compressed + startingOffset;
|
||||
if (longSize) {
|
||||
compressedData[0] = 255;
|
||||
compressedData[2] = (uint8_t)(dataLength >> 40u);
|
||||
compressedData[3] = (uint8_t)(dataLength >> 32u);
|
||||
compressedData[4] = (uint8_t)(dataLength >> 24u);
|
||||
compressedData[5] = (uint8_t)(dataLength >> 16u);
|
||||
compressedData[6] = (uint8_t)(dataLength >> 8u);
|
||||
compressedData[7] = (uint8_t)dataLength;
|
||||
}
|
||||
else {
|
||||
compressedData[0] = 254;
|
||||
compressedData[1] = (uint8_t)(dataLength >> 16u);
|
||||
compressedData[2] = (uint8_t)(dataLength >> 8u);
|
||||
compressedData[3] = (uint8_t)dataLength;
|
||||
}
|
||||
value->mv_size = compressedSize + prefixSize;
|
||||
value->mv_data = compressed;
|
||||
return ([](MDB_val &value) -> void {
|
||||
delete[] (char*)value.mv_data;
|
||||
});
|
||||
}
|
||||
else {
|
||||
delete[] compressed;
|
||||
return nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
class CompressionWorker : public AsyncWorker {
|
||||
public:
|
||||
CompressionWorker(EnvWrap* env, double* compressionAddress, const Function& callback)
|
||||
: AsyncWorker(callback), env(env), compressionAddress(compressionAddress) {}
|
||||
|
||||
|
||||
void Execute() {
|
||||
uint64_t compressionPointer;
|
||||
compressionPointer = std::atomic_exchange((std::atomic<int64_t>*) compressionAddress, (int64_t) 2);
|
||||
if (compressionPointer > 1) {
|
||||
Compression* compression = (Compression*)(size_t) * ((double*)&compressionPointer);
|
||||
compression->compressInstruction(env, compressionAddress);
|
||||
}
|
||||
}
|
||||
void OnOK() {
|
||||
// don't actually call the callback, no need
|
||||
}
|
||||
|
||||
private:
|
||||
EnvWrap* env;
|
||||
double* compressionAddress;
|
||||
};
|
||||
|
||||
NAPI_FUNCTION(EnvWrap::compress) {
|
||||
ARGS(3)
|
||||
GET_INT64_ARG(0);
|
||||
EnvWrap* ew = (EnvWrap*) i64;
|
||||
napi_get_value_int64(env, args[1], &i64);
|
||||
double* compressionAddress = (double*) i64;
|
||||
CompressionWorker* worker = new CompressionWorker(ew, (double*) compressionAddress, Function(env, args[2]));
|
||||
worker->Queue();
|
||||
RETURN_UNDEFINED;
|
||||
}
|
||||
|
||||
void Compression::setupExports(Napi::Env env, Object exports) {
|
||||
Function CompressionClass = DefineClass(env, "Compression", {
|
||||
Compression::InstanceMethod("setBuffer", &Compression::setBuffer),
|
||||
});
|
||||
exports.Set("Compression", CompressionClass);
|
||||
// compressionTpl->InstanceTemplate()->SetInternalFieldCount(1);
|
||||
}
|
||||
|
||||
|
||||
// This file contains code from the node-lmdb project
|
||||
// Copyright (c) 2013-2017 Timur Kristóf
|
||||
// Copyright (c) 2021 Kristopher Tate
|
||||
// Licensed to you under the terms of the MIT license
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to deal
|
||||
// in the Software without restriction, including without limitation the rights
|
||||
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
// copies of the Software, and to permit persons to whom the Software is
|
||||
// furnished to do so, subject to the following conditions:
|
||||
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
// THE SOFTWARE.
|
||||
Reference in New Issue
Block a user