Skip to content

Commit 4efcc9c

Browse files
committed
decimal: add support decimal type in msgpack
This patch provides decimal support for all space operations and as function return result. Decimal type was introduced in Tarantool 2.2. See more about decimal type in [1] and [2]. According to BCD encoding/decoding specification sign is encoded by letters: '0x0a', '0x0c', '0x0e', '0x0f' stands for plus, and '0x0b' and '0x0d' for minus. Tarantool always uses '0x0c' for plus and '0x0d' for minus. Implementation in Golang follows the same rule and in all test samples sign encoded by '0x0d' and '0x0c' for simplification. Because 'c' used by Tarantool. To use decimal with github.com/shopspring/decimal in msgpack, import tarantool/decimal submodule. 1. https://www.tarantool.io/en/doc/latest/book/box/data_model/ 2. https://www.tarantool.io/en/doc/latest/dev_guide/internals/msgpack_extensions/#the-decimal-type 3. https://github.com/douglascrockford/DEC64/blob/663f562a5f0621021b98bfdd4693571993316174/dec64_test.c#L62-L104 4. https://github.com/shopspring/decimal/blob/v1.3.1/decimal_test.go#L27-L64 5. https://github.com/tarantool/tarantool/blob/60fe9d14c1c7896aa7d961e4b68649eddb4d2d6c/test/unit/decimal.c#L154-L171 Lua snippet for encoding number to MsgPack representation: local decimal = require('decimal') local function mp_encode_dec(num) local dec = msgpack.encode(decimal.new(num)) return dec:gsub('.', function (c) return string.format('%02x', string.byte(c)) end) end print(mp_encode_dec(-12.34)) -- 0xd6010201234d Follows up tarantool/tarantool#692 Part of #96
1 parent a9e491f commit 4efcc9c

File tree

9 files changed

+1043
-0
lines changed

9 files changed

+1043
-0
lines changed

CHANGELOG.md

+1
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ Versioning](http://semver.org/spec/v2.0.0.html) except to the first release.
1818
- queue-utube handling (#85)
1919
- Master discovery (#113)
2020
- SQL support (#62)
21+
- Support decimal type in msgpack (#96)
2122

2223
### Fixed
2324

Makefile

+6
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,12 @@ test-main:
5858
go clean -testcache
5959
go test . -v -p 1
6060

61+
.PHONY: test-decimal
62+
test-decimal:
63+
@echo "Running tests in decimal package"
64+
go clean -testcache
65+
go test ./decimal/ -v -p 1
66+
6167
.PHONY: coverage
6268
coverage:
6369
go clean -testcache

decimal/bcd.go

+227
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,227 @@
1+
// Package decimal implements methods to encode and decode BCD.
2+
//
3+
// BCD (Binary-Coded Decimal) is a sequence of bytes representing decimal
4+
// digits of the encoded number (each byte has two decimal digits each encoded
5+
// using 4-bit nibbles), so byte >> 4 is the first digit and byte & 0x0f is the
6+
// second digit. The leftmost digit in the array is the most significant. The
7+
// rightmost digit in the array is the least significant.
8+
//
9+
// The first byte of the BCD array contains the first digit of the number,
10+
// represented as follows:
11+
//
12+
// | 4 bits | 4 bits |
13+
// = 0x = the 1st digit
14+
//
15+
// (The first nibble contains 0 if the decimal number has an even number of
16+
// digits). The last byte of the BCD array contains the last digit of the
17+
// number and the final nibble, represented as follows:
18+
//
19+
// | 4 bits | 4 bits |
20+
// = the last digit = nibble
21+
//
22+
// The final nibble represents the number's sign: 0x0a, 0x0c, 0x0e, 0x0f stand
23+
// for plus, 0x0b and 0x0d stand for minus.
24+
//
25+
// Examples:
26+
//
27+
// The decimal -12.34 will be encoded as 0xd6, 0x01, 0x02, 0x01, 0x23, 0x4d:
28+
//
29+
// |MP_EXT (fixext 4) | MP_DECIMAL | scale | 1 | 2,3 | 4 (minus) |
30+
// | 0xd6 | 0x01 | 0x02 | 0x01 | 0x23 | 0x4d |
31+
//
32+
// The decimal 0.000000000000000000000000000000000010 will be encoded as
33+
// 0xc7, 0x03, 0x01, 0x24, 0x01, 0x0c:
34+
//
35+
// | MP_EXT (ext 8) | length | MP_DECIMAL | scale | 1 | 0 (plus) |
36+
// | 0xc7 | 0x03 | 0x01 | 0x24 | 0x01 | 0x0c |
37+
//
38+
// See also:
39+
//
40+
// * MessagePack extensions https://www.tarantool.io/en/doc/latest/dev_guide/internals/msgpack_extensions/
41+
//
42+
// * An implementation in C language https://github.com/tarantool/decNumber/blob/master/decPacked.c
43+
package decimal
44+
45+
import (
46+
"fmt"
47+
"strings"
48+
)
49+
50+
const (
51+
bytePlus = byte(0x0c)
52+
byteMinus = byte(0x0d)
53+
)
54+
55+
var isNegative = map[byte]bool{
56+
0x0a: false,
57+
0x0b: true,
58+
0x0c: false,
59+
0x0d: true,
60+
0x0e: false,
61+
0x0f: false,
62+
}
63+
64+
// Calculate a number of digits in a buffer with decimal number.
65+
//
66+
// Plus, minus, point and leading zeros do not count.
67+
// Contains a quirk for a zero - returns 1.
68+
//
69+
// Examples (see more examples in tests):
70+
//
71+
// - 0.0000000000000001 - 1 digit
72+
//
73+
// - 00012.34 - 4 digits
74+
//
75+
// - 0.340 - 3 digits
76+
//
77+
// - 0 - 1 digit
78+
func GetNumberLength(buf string) int {
79+
n := 0
80+
for _, ch := range []byte(buf) {
81+
if ch == '0' && n != 0 {
82+
n += 1
83+
continue
84+
}
85+
if ch >= '1' && ch <= '9' {
86+
n += 1
87+
}
88+
}
89+
90+
// Fix a case with a single 0.
91+
if n == 0 {
92+
n = 1
93+
}
94+
95+
return n
96+
}
97+
98+
// EncodeStringToBCD converts a string buffer to BCD Packed Decimal.
99+
//
100+
// The number is converted to a BCD packed decimal byte array, right aligned in
101+
// the BCD array, whose length is indicated by the second parameter. The final
102+
// 4-bit nibble in the array will be a sign nibble, 0x0c for "+" and 0x0d for
103+
// "-". Unused bytes and nibbles to the left of the number are set to 0. scale
104+
// is set to the scale of the number (this is the exponent, negated).
105+
func EncodeStringToBCD(buf string) ([]byte, error) {
106+
if len(buf) == 0 {
107+
return nil, fmt.Errorf("Length of number is zero")
108+
}
109+
signByte := bytePlus // By default number is positive.
110+
if buf[0] == '-' {
111+
signByte = byteMinus
112+
}
113+
114+
// The first nibble should contain 0, if the decimal number has an even
115+
// number of digits. Therefore highNibble is false when decimal number
116+
// is even.
117+
highNibble := true
118+
l := GetNumberLength(buf)
119+
if l%2 == 0 {
120+
highNibble = false
121+
}
122+
scale := 0 // By default decimal number is integer.
123+
var byteBuf []byte
124+
for i, ch := range []byte(buf) {
125+
// Skip leading zeros.
126+
if (len(byteBuf) == 0) && ch == '0' {
127+
continue
128+
}
129+
if (i == 0) && (ch == '-' || ch == '+') {
130+
continue
131+
}
132+
if ch == '.' && scale != 0 {
133+
return nil, fmt.Errorf("Number contains more than one point")
134+
}
135+
// Calculate a number of digits after the decimal point.
136+
if ch == '.' {
137+
scale = len(buf) - i - 1
138+
continue
139+
}
140+
141+
if ch < '0' || ch > '9' {
142+
return nil, fmt.Errorf("Failed to convert symbol '%c' to a digit", ch)
143+
}
144+
digit := byte(ch - '0')
145+
if highNibble {
146+
// Add a digit to a high nibble.
147+
digit = digit << 4
148+
byteBuf = append(byteBuf, digit)
149+
highNibble = false
150+
} else {
151+
if len(byteBuf) == 0 {
152+
byteBuf = make([]byte, 1)
153+
}
154+
// Add a digit to a low nibble.
155+
lowByteIdx := len(byteBuf) - 1
156+
byteBuf[lowByteIdx] = byteBuf[lowByteIdx] | digit
157+
highNibble = true
158+
}
159+
}
160+
if highNibble {
161+
// Put a sign to a high nibble.
162+
byteBuf = append(byteBuf, signByte)
163+
} else {
164+
// Put a sign to a low nibble.
165+
lowByteIdx := len(byteBuf) - 1
166+
byteBuf[lowByteIdx] = byteBuf[lowByteIdx] | signByte
167+
}
168+
byteBuf = append([]byte{byte(scale)}, byteBuf...)
169+
170+
return byteBuf, nil
171+
}
172+
173+
// DecodeStringFromBCD converts a BCD Packed Decimal to a string buffer.
174+
//
175+
// The BCD packed decimal byte array, together with an associated scale, is
176+
// converted to a string. The BCD array is assumed full of digits, and must be
177+
// ended by a 4-bit sign nibble in the least significant four bits of the final
178+
// byte. The scale is used (negated) as the exponent of the decimal number.
179+
// Note that zeros may have a sign and/or a scale.
180+
func DecodeStringFromBCD(bcdBuf []byte) (string, error) {
181+
const scaleIdx = 0 // Index of a byte with scale.
182+
scale := int(bcdBuf[scaleIdx])
183+
// Get a BCD buffer without a byte with scale.
184+
bcdBuf = bcdBuf[scaleIdx+1:]
185+
length := len(bcdBuf)
186+
var digits []string
187+
for i, bcdByte := range bcdBuf {
188+
highNibble := bcdByte >> 4
189+
digits = append(digits, string('0'+highNibble))
190+
lowNibble := bcdByte & 0x0f
191+
// lowNibble of last byte is ignored because it contains a
192+
// sign.
193+
if i != length-1 {
194+
digits = append(digits, string('0'+lowNibble))
195+
}
196+
}
197+
198+
// Add missing zeros when scale is less than current length.
199+
l := len(digits)
200+
if scale >= l {
201+
var zeros []string
202+
for i := 0; i <= scale-l; i++ {
203+
zeros = append(zeros, "0")
204+
}
205+
digits = append(zeros, digits...)
206+
}
207+
208+
// Add a dot when number is fractional.
209+
if scale != 0 {
210+
idx := len(digits) - scale
211+
digits = append(digits, "X") // [1 2 3 X]
212+
copy(digits[idx:], digits[idx-1:]) // [1 2 2 3]
213+
digits[idx] = "." // [1 . 2 3]
214+
}
215+
216+
// Add a sign, it is encoded in a low nibble of a last byte.
217+
lastByte := bcdBuf[length-1]
218+
sign := lastByte & 0x0f
219+
if isNegative[sign] {
220+
digits = append([]string{"-"}, digits...)
221+
}
222+
223+
// Merge slice to a single string.
224+
str := strings.Join(digits, "")
225+
226+
return str, nil
227+
}

decimal/config.lua

+41
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
local decimal = require('decimal')
2+
local msgpack = require('msgpack')
3+
4+
-- Do not set listen for now so connector won't be
5+
-- able to send requests until everything is configured.
6+
box.cfg{
7+
work_dir = os.getenv("TEST_TNT_WORK_DIR"),
8+
}
9+
10+
box.schema.user.create('test', { password = 'test' , if_not_exists = true })
11+
box.schema.user.grant('test', 'execute', 'universe', nil, { if_not_exists = true })
12+
13+
local decimal_msgpack_supported = pcall(msgpack.encode, decimal.new(1))
14+
if not decimal_msgpack_supported then
15+
error('Decimal unsupported, use Tarantool 2.2 or newer')
16+
end
17+
18+
local s = box.schema.space.create('testDecimal', {
19+
id = 524,
20+
if_not_exists = true,
21+
})
22+
s:create_index('primary', {
23+
type = 'TREE',
24+
parts = {
25+
{
26+
field = 1,
27+
type = 'decimal',
28+
},
29+
},
30+
if_not_exists = true
31+
})
32+
s:truncate()
33+
34+
box.schema.user.grant('test', 'read,write', 'space', 'testDecimal', { if_not_exists = true })
35+
36+
-- Set listen only when every other thing is configured.
37+
box.cfg{
38+
listen = os.getenv("TEST_TNT_LISTEN"),
39+
}
40+
41+
require('console').start()

0 commit comments

Comments
 (0)