1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
|
#ifndef _GLIM_SERIALIZABLEPOOL_HPP_INCLUDED
#define _GLIM_SERIALIZABLEPOOL_HPP_INCLUDED
#include "gstring.hpp"
#ifndef _SERIALIZABLEPOOL_NOLDB
# include "ldb.hpp" // Reuse `ldbSerialize` and `ldbDeserialize` in the `with` method.
#endif
namespace glim {
namespace SerializablePoolHelpers {
struct Impl {
/**
* Pool format: \code
* uint32_t valuesStart; // Network byte order.
* uint32_t value0Offset, value1Offset, ... valueNOffset; // Network byte order.
* char value0[]; char zero; char value1[]; char zero; ... char valueN[]; char zero;
* \endcode
*/
gstring _pool;
std::vector<gstring> _changes;
std::vector<bool> _changed;
bool _readOnly = false;
Impl() = default;
Impl (const gstring& poolBytes): _pool (poolBytes), _readOnly (false) {}
};
/// Can be used to avoid a deep copy of the pool and change vectors (pImpl idiom).
/// Example: \code glim::SerializablePool pool; \endcode
struct SharedPtr {
std::shared_ptr<Impl> _impl;
SharedPtr() = default;
SharedPtr (const gstring& poolBytes): _impl (std::make_shared<Impl> (poolBytes)) {}
Impl* get() const {return _impl.get();}
Impl* operator->() const {return _impl.get();}
Impl& operator*() const {return *_impl;}
Impl* instance() {if (!_impl) _impl = std::make_shared<Impl>(); return _impl.get();}
};
/// Can be used instead of SharedPtr to avoid the shared_ptr indirection when the SerializablePoolTpl isn't going to be copied.
/// Example: \code glim::InlineSerializablePool temporaryPool (bytes); \endcode
struct InlinePtr {
Impl _impl;
InlinePtr() = default;
InlinePtr (const gstring& poolBytes): _impl (poolBytes) {}
Impl* get() const {return const_cast<Impl*> (&_impl);}
Impl* operator->() const {return const_cast<Impl*> (&_impl);}
Impl& operator*() const {return const_cast<Impl&> (_impl);}
Impl* instance() {return &_impl;}
};
}
/** Serialization with lazy parsing: fields are accessed without "unpacking" the byte array.
* Changes are stored separately, allowing the user to know exactly what fields has been changed and compare the old values to the new ones. */
template <typename PI>
class SerializablePoolTpl {
protected:
using Impl = SerializablePoolHelpers::Impl;
PI _impl;
/** @param ref Return a zero-copy view. The view should not be used outside of the pool buffer's lifetime. */
static gstring original (const gstring& pool, uint32_t num, bool ref = false) {
uint32_t poolLength = pool.length(); if (poolLength < 4) return gstring();
uint32_t valuesStart = ntohl (*(uint32_t*) pool.data());
assert (valuesStart <= poolLength);
uint32_t valueOffsetOffset = 4 + num * 4;
if ((int) valuesStart - (int) valueOffsetOffset < 4) return gstring(); // num > size
uint32_t valueOffset = ntohl (*(uint32_t*) (pool.data() + valueOffsetOffset));
valueOffsetOffset += 4;
uint32_t nextValueOffset = ((int) valuesStart - (int) valueOffsetOffset < 4)
? poolLength
: ntohl (*(uint32_t*) (pool.data() + valueOffsetOffset));
return gstring (0, (void*) (pool.data() + valueOffset), false, nextValueOffset - 1 - valueOffset, ref);
}
/** How many elements are in the pool. */
static uint32_t poolSize (const gstring& pool) {
if (pool.length() < 4) return 0;
uint32_t valuesStart = ntohl (*(uint32_t*) pool.data());
return (valuesStart - 4) / 4;
}
void toBytes (gstring& newPool, uint32_t size, const gstring* oldPool) const {
newPool.clear();
const Impl* impl = _impl.get();
const std::vector<bool>& changed = impl->_changed;
const std::vector<gstring>& changes = impl->_changes;
if (changed.empty()) return;
uint32_t valuesStart = 4 + size * 4;
uint32_t networkOrder = 0;
newPool.append ((char*) &(networkOrder = htonl (valuesStart)), 4);
for (uint32_t num = 0; num < size; ++num) newPool.append ((char*) &(networkOrder = 0), 4);
for (uint32_t num = 0; num < size; ++num) {
uint32_t start = newPool.length();
if (num < changed.size() && changed[num]) newPool << changes[num];
else newPool << original (oldPool ? *oldPool : impl->_pool, num);
newPool << '\0';
uint32_t valuesOffsetOffset = 4 + num * 4; assert (valuesOffsetOffset < valuesStart);
*(uint32_t*)(newPool.data() + valuesOffsetOffset) = htonl (start);
}
}
public:
/** Field, old value, new value. Might be used to maintain indexes. */
typedef std::function<void(uint32_t, const gstring&, const gstring&)> ChangeVisitor;
SerializablePoolTpl() = default;
/** Copy the given pool bytes from the outside source (e.g. from the database). */
SerializablePoolTpl (const gstring& poolBytes): _impl (poolBytes) {}
/** Returns a view into the original serialized field (ignores the current changes).\n
* Returns an empty string if the field is not in the pool (num > size).
* @param ref Return a zero-copy view. The view becomes invalid after the value has been changed or when the pool's `Impl` is destroyed. */
const gstring original (uint32_t num, bool ref = false) const {return original (_impl->_pool, num, ref);}
/** Returns the original serialized field (ignores the current changes).\n
* Returns an empty string if the field is not in the pool (num > size). */
const char* cstringOriginal (uint32_t num) const {
gstring gs (original (_impl->_pool, num));
return gs.empty() ? "" : gs.data(); // All fields in the _pool are 0-terminated.
}
/** Returns the field.
* @param ref Return a zero-copy view. The view becomes invalid after the value has been changed or when the pool's `Impl` is destroyed. */
const gstring current (uint32_t num, bool ref = false) const {
const Impl* impl = _impl.get(); if (!impl) return gstring();
if (num < impl->_changed.size() && impl->_changed[num]) {
const gstring& value = impl->_changes[num]; return ref ? value.ref() : value;}
return original (impl->_pool, num);
}
/** Set the new value of the field. */
void set (uint32_t num, const gstring& value) {
Impl* impl = _impl.instance();
if (__builtin_expect (impl->_readOnly, 0)) throw std::runtime_error ("Attempt to modify a read-only SerializablePool");
if (num >= impl->_changed.size()) {impl->_changed.resize (num + 1); impl->_changes.resize (num + 1);}
impl->_changed[num] = true;
impl->_changes[num] = value;
}
void reserve (uint32_t fields) {
Impl* impl = _impl.instance();
if (__builtin_expect (impl->_readOnly, 0)) throw std::runtime_error ("Attempt to modify a read-only SerializablePool");
impl->_changed.reserve (fields);
impl->_changes.reserve (fields);
}
/** Peek into the pool.\n
* Returned reference should not be used after the SerializablePool goes out of scope (and destroyed). */
const gstring& originalPool() {return _impl->_pool;}
/** Serialize the pool.
* @param changeVisitor is called for every field that was really changed (e.g. the bytes differ). */
void toBytes (gstring& newPool, ChangeVisitor changeVisitor = ChangeVisitor()) const {
if (changeVisitor) {
const Impl* impl = _impl.get();
const std::vector<bool>& changed = impl->_changed;
const std::vector<gstring>& changes = impl->_changes;
for (uint32_t num = 0, size = changed.size(); num < size; ++num) if (changed[num]) {
const gstring& from = original (impl->_pool, num); const gstring& to = changes[num];
if (from != to) changeVisitor (num, from, to);
}
}
toBytes (newPool, (uint32_t) _impl->_changed.size(), nullptr);
}
/**
* Performs "delta" serialization of the pool: creates a new pool where values which has not changed are copied from the `oldPool`.\n
* \code Use case: 1) pools X and Y are loaded from a database by users A and B;
* 2) user A changes field 0 in pool X; 3) user B changes field 1 in pool Y;
* 4) user A loads `oldPool` from the database, does `toBytesDelta` from pool X and saves to the database;
* 5) user B loads `oldPool` from the database, does `toBytesDelta` from pool Y and saves to the database;
* result: database contains both changes (field 0 from user A and field 1 from user B). \endcode
* @param changeVisitor is called for every field that was changed between the oldPool and the current one.
* Returns `false` and leaves `newPool` *empty* if there are no changes found against the `oldPool`.
*/
bool toBytesDelta (gstring& newPool, const gstring& oldPool, ChangeVisitor changeVisitor = ChangeVisitor()) const {
newPool.clear();
const Impl* impl = _impl.get();
const std::vector<bool>& changed = impl->_changed;
const std::vector<gstring>& changes = impl->_changes;
bool verifiedChanges = false;
for (uint32_t num = 0, size = changed.size(); num < size; ++num) if (changed[num]) {
const gstring& from = original (oldPool, num); const gstring& to = changes[num];
if (from != to) {
verifiedChanges = true;
if (changeVisitor) changeVisitor (num, from, to); else break;
}
}
if (!verifiedChanges) return false;
toBytes (newPool, std::max ((uint32_t) changed.size(), poolSize (oldPool)), &oldPool);
return true;
}
/** True if the field has been `set` in this pool instance.\n
* NB: Does *not* check if the `set` value is equal to the `original` value or not. */
bool changed (uint32_t num) const {const auto& changed = _impl->_changed; return num < changed.size() ? changed[num] : false;}
/** True if a field has been `set` in this pool instance. */
bool changed() const {return !_impl->_changed.empty();}
bool operator == (const SerializablePoolTpl<PI>& rhs) const {return _impl.get() == rhs._impl.get();}
/** Useful for storing SerializablePool in a map. */
intptr_t implId() const {return (intptr_t) _impl.get();}
/** If set to `true` then modifying the pool will throw an exception.\n
* Useful for freezing the pool before sharing it with other threads. */
void readOnly (bool ro) {if (_impl) _impl->_readOnly = true;}
bool readOnly() const {return (_impl ? _impl->_readOnly : false);}
/** Number of elements in the pool. Equals to max(num)-1. */
uint32_t size() const {
Impl* impl = _impl.get(); if (__builtin_expect (!impl, 0)) return 0;
return std::max (poolSize (impl->_pool), (uint32_t) impl->_changed.size());
}
#ifndef _SERIALIZABLEPOOL_NOLDB
/** Serialize the `value` with `ldbSerialize` and `set` it to `num`.
* @param stackSize is the amount of space to preallocate on stack for the temporary buffer. */
template<typename T> void serialize (uint32_t num, const T& value, uint32_t stackSize = 256) {
GSTRING_ON_STACK (bytes, stackSize);
ldbSerialize (bytes, value);
set (num, bytes);
}
/** If the field is not empty then `ldbDeserialize` it into `value`. */
template<typename T> void deserialize (uint32_t num, T& value) const {
const gstring& bytes = current (num);
if (bytes.length()) ldbDeserialize (current (num), value);
}
/** Deserialize the `num` field with `ldbDeserialize`, run `visitor` on it, then optionally serialize the field back using `ldbSerialize`.
* Example: \code
* typedef std::map<std::string, std::string> MyMap;
* pool.with<MyMap> (_myMap, [](MyMap& myMap) {myMap["foo"] = "bar"; return true;});
* \endcode
* @param visitor must return `true` to serialize the field back to the pool.
*/
template<typename T> void with (uint32_t num, std::function<bool(T&)> visitor) {
const gstring& fromBytes = current (num, true);
T value; if (fromBytes.length()) ldbDeserialize (fromBytes, value);
if (visitor (value)) serialize (num, value, 16 + fromBytes.length() * 2);
}
#endif
};
using SerializablePool = SerializablePoolTpl<SerializablePoolHelpers::SharedPtr>;
using InlineSerializablePool = SerializablePoolTpl<SerializablePoolHelpers::InlinePtr>;
}
#endif // _GLIM_SERIALIZABLEPOOL_HPP_INCLUDED
|
