From b358328824ceead79fca4c450b6b354ff460b74b Mon Sep 17 00:00:00 2001 From: Tiffany Yeh Date: Tue, 20 May 2025 23:52:44 -0400 Subject: [PATCH 1/6] batch float with trailing zero and add a useFloatWithTrailingZero flag to enable --- go.mod | 2 +- replication/binlogsyncer.go | 4 ++++ replication/json_binary.go | 27 ++++++++++++++++++++++++--- replication/parser.go | 6 ++++++ replication/row_event.go | 9 +++++---- 5 files changed, 40 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index f0b704b47..802481165 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.22 toolchain go1.23.1 require ( + filippo.io/edwards25519 v1.1.0 github.com/BurntSushi/toml v1.3.2 github.com/Masterminds/semver v1.5.0 github.com/go-sql-driver/mysql v1.7.1 @@ -19,7 +20,6 @@ require ( ) require ( - filippo.io/edwards25519 v1.1.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 // indirect diff --git a/replication/binlogsyncer.go b/replication/binlogsyncer.go index 612229926..b2665a2a7 100644 --- a/replication/binlogsyncer.go +++ b/replication/binlogsyncer.go @@ -76,6 +76,9 @@ type BinlogSyncerConfig struct { // Use decimal.Decimal structure for decimals. UseDecimal bool + // Use FloatWithTrailingZero structure for floats. + UseFloatWithTrailingZero bool + // RecvBufferSize sets the size in bytes of the operating system's receive buffer associated with the connection. RecvBufferSize int @@ -197,6 +200,7 @@ func NewBinlogSyncer(cfg BinlogSyncerConfig) *BinlogSyncer { b.parser.SetParseTime(b.cfg.ParseTime) b.parser.SetTimestampStringLocation(b.cfg.TimestampStringLocation) b.parser.SetUseDecimal(b.cfg.UseDecimal) + b.parser.SetUseFloatWithTrailingZero(b.cfg.UseFloatWithTrailingZero) b.parser.SetVerifyChecksum(b.cfg.VerifyChecksum) b.parser.SetRowsEventDecodeFunc(b.cfg.RowsEventDecodeFunc) b.parser.SetTableMapOptionalMetaDecodeFunc(b.cfg.TableMapOptionalMetaDecodeFunc) diff --git a/replication/json_binary.go b/replication/json_binary.go index e4fb7e4e3..638498c9a 100644 --- a/replication/json_binary.go +++ b/replication/json_binary.go @@ -3,6 +3,7 @@ package replication import ( "fmt" "math" + "strconv" "github.com/go-mysql-org/go-mysql/mysql" "github.com/go-mysql-org/go-mysql/utils" @@ -52,6 +53,8 @@ type ( JsonDiffOperation byte ) +type FloatWithTrailingZero float64 + const ( // The JSON value in the given path is replaced with a new value. // @@ -96,6 +99,14 @@ func (jd *JsonDiff) String() string { return fmt.Sprintf("json_diff(op:%s path:%s value:%s)", jd.Op, jd.Path, jd.Value) } +func (f FloatWithTrailingZero) MarshalJSON() ([]byte, error) { + if float64(f) == float64(int(f)) { + return []byte(strconv.FormatFloat(float64(f), 'f', 1, 64)), nil + } + + return []byte(strconv.FormatFloat(float64(f), 'f', -1, 64)), nil +} + func jsonbGetOffsetSize(isSmall bool) int { if isSmall { return jsonbSmallOffsetSize @@ -125,6 +136,7 @@ func jsonbGetValueEntrySize(isSmall bool) int { func (e *RowsEvent) decodeJsonBinary(data []byte) ([]byte, error) { d := jsonBinaryDecoder{ useDecimal: e.useDecimal, + useFloatWithTrailingZero: e.useFloatWithTrailingZero, ignoreDecodeErr: e.ignoreJSONDecodeErr, } @@ -141,9 +153,10 @@ func (e *RowsEvent) decodeJsonBinary(data []byte) ([]byte, error) { } type jsonBinaryDecoder struct { - useDecimal bool - ignoreDecodeErr bool - err error + useDecimal bool + useFloatWithTrailingZero bool + ignoreDecodeErr bool + err error } func (d *jsonBinaryDecoder) decodeValue(tp byte, data []byte) interface{} { @@ -175,6 +188,9 @@ func (d *jsonBinaryDecoder) decodeValue(tp byte, data []byte) interface{} { case JSONB_UINT64: return d.decodeUint64(data) case JSONB_DOUBLE: + if d.useFloatWithTrailingZero { + return d.decodeDoubleWithTrailingZero(data) + } return d.decodeDouble(data) case JSONB_STRING: return d.decodeString(data) @@ -395,6 +411,11 @@ func (d *jsonBinaryDecoder) decodeDouble(data []byte) float64 { return v } +func (d *jsonBinaryDecoder) decodeDoubleWithTrailingZero(data []byte) FloatWithTrailingZero { + v := d.decodeDouble(data) + return FloatWithTrailingZero(v) +} + func (d *jsonBinaryDecoder) decodeString(data []byte) string { if d.err != nil { return "" diff --git a/replication/parser.go b/replication/parser.go index eedc205ae..7b1de69a6 100644 --- a/replication/parser.go +++ b/replication/parser.go @@ -36,6 +36,7 @@ type BinlogParser struct { stopProcessing uint32 useDecimal bool + useFloatWithTrailingZero bool ignoreJSONDecodeErr bool verifyChecksum bool @@ -202,6 +203,10 @@ func (p *BinlogParser) SetUseDecimal(useDecimal bool) { p.useDecimal = useDecimal } +func (p *BinlogParser) SetUseFloatWithTrailingZero(useFloatWithTrailingZero bool) { + p.useFloatWithTrailingZero = useFloatWithTrailingZero +} + func (p *BinlogParser) SetIgnoreJSONDecodeError(ignoreJSONDecodeErr bool) { p.ignoreJSONDecodeErr = ignoreJSONDecodeErr } @@ -410,6 +415,7 @@ func (p *BinlogParser) newRowsEvent(h *EventHeader) *RowsEvent { e.parseTime = p.parseTime e.timestampStringLocation = p.timestampStringLocation e.useDecimal = p.useDecimal + e.useFloatWithTrailingZero = p.useFloatWithTrailingZero e.ignoreJSONDecodeErr = p.ignoreJSONDecodeErr switch h.EventType { diff --git a/replication/row_event.go b/replication/row_event.go index 3883ae1dc..886627a86 100644 --- a/replication/row_event.go +++ b/replication/row_event.go @@ -945,10 +945,11 @@ type RowsEvent struct { Rows [][]interface{} SkippedColumns [][]int - parseTime bool - timestampStringLocation *time.Location - useDecimal bool - ignoreJSONDecodeErr bool + parseTime bool + timestampStringLocation *time.Location + useDecimal bool + useFloatWithTrailingZero bool + ignoreJSONDecodeErr bool } // EnumRowsEventType is an abridged type describing the operation which triggered the given RowsEvent. From 731ed05a9a8de40e6a28ddae4dfa7e73e178d412 Mon Sep 17 00:00:00 2001 From: Tiffany Yeh Date: Wed, 21 May 2025 11:09:03 -0400 Subject: [PATCH 2/6] gofumpt --- replication/json_binary.go | 12 ++++++------ replication/parser.go | 6 +++--- replication/row_event.go | 10 +++++----- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/replication/json_binary.go b/replication/json_binary.go index 638498c9a..4568fc678 100644 --- a/replication/json_binary.go +++ b/replication/json_binary.go @@ -135,9 +135,9 @@ func jsonbGetValueEntrySize(isSmall bool) int { // the common JSON encoding data. func (e *RowsEvent) decodeJsonBinary(data []byte) ([]byte, error) { d := jsonBinaryDecoder{ - useDecimal: e.useDecimal, + useDecimal: e.useDecimal, useFloatWithTrailingZero: e.useFloatWithTrailingZero, - ignoreDecodeErr: e.ignoreJSONDecodeErr, + ignoreDecodeErr: e.ignoreJSONDecodeErr, } if d.isDataShort(data, 1) { @@ -153,10 +153,10 @@ func (e *RowsEvent) decodeJsonBinary(data []byte) ([]byte, error) { } type jsonBinaryDecoder struct { - useDecimal bool - useFloatWithTrailingZero bool - ignoreDecodeErr bool - err error + useDecimal bool + useFloatWithTrailingZero bool + ignoreDecodeErr bool + err error } func (d *jsonBinaryDecoder) decodeValue(tp byte, data []byte) interface{} { diff --git a/replication/parser.go b/replication/parser.go index 40861d033..8d235d9ae 100644 --- a/replication/parser.go +++ b/replication/parser.go @@ -35,10 +35,10 @@ type BinlogParser struct { // used to start/stop processing stopProcessing uint32 - useDecimal bool + useDecimal bool useFloatWithTrailingZero bool - ignoreJSONDecodeErr bool - verifyChecksum bool + ignoreJSONDecodeErr bool + verifyChecksum bool rowsEventDecodeFunc func(*RowsEvent, []byte) error diff --git a/replication/row_event.go b/replication/row_event.go index 20d107246..a83a73ca5 100644 --- a/replication/row_event.go +++ b/replication/row_event.go @@ -945,11 +945,11 @@ type RowsEvent struct { Rows [][]interface{} SkippedColumns [][]int - parseTime bool - timestampStringLocation *time.Location - useDecimal bool - useFloatWithTrailingZero bool - ignoreJSONDecodeErr bool + parseTime bool + timestampStringLocation *time.Location + useDecimal bool + useFloatWithTrailingZero bool + ignoreJSONDecodeErr bool } // EnumRowsEventType is an abridged type describing the operation which triggered the given RowsEvent. From 9d46fb322fa73b0d67eed59512c9595d9e2c4cb3 Mon Sep 17 00:00:00 2001 From: Tiffany Yeh Date: Wed, 21 May 2025 12:17:46 -0400 Subject: [PATCH 3/6] add test --- replication/binlogsyncer.go | 2 +- replication/replication_test.go | 78 +++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 1 deletion(-) diff --git a/replication/binlogsyncer.go b/replication/binlogsyncer.go index 7c41a38a6..6bdd88faa 100644 --- a/replication/binlogsyncer.go +++ b/replication/binlogsyncer.go @@ -76,7 +76,7 @@ type BinlogSyncerConfig struct { // Use decimal.Decimal structure for decimals. UseDecimal bool - // Use FloatWithTrailingZero structure for floats. + // FloatWithTrailingZero structure for floats. UseFloatWithTrailingZero bool // RecvBufferSize sets the size in bytes of the operating system's receive buffer associated with the connection. diff --git a/replication/replication_test.go b/replication/replication_test.go index d8bdb3b5e..b8e90b80a 100644 --- a/replication/replication_test.go +++ b/replication/replication_test.go @@ -14,6 +14,7 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + "github.com/goccy/go-json" "github.com/go-mysql-org/go-mysql/client" "github.com/go-mysql-org/go-mysql/mysql" "github.com/go-mysql-org/go-mysql/test_util" @@ -464,3 +465,80 @@ func (t *testSyncerSuite) TestMysqlBinlogCodec() { require.NoError(t.T(), err) } } + +func (t *testSyncerSuite) TestFloatWithTrailingZeros() { + t.setupTest(mysql.MySQLFlavor) + + str := `DROP TABLE IF EXISTS test_float_zeros` + t.testExecute(str) + + // Create table with JSON column containing float values + str = `CREATE TABLE test_float_zeros ( + id INT PRIMARY KEY, + json_val JSON + )` + t.testExecute(str) + + // Test with useFloatWithTrailingZero = true + t.b.cfg.UseFloatWithTrailingZero = true + t.testFloatWithTrailingZerosCase(true) + + // Test with useFloatWithTrailingZero = false + t.b.cfg.UseFloatWithTrailingZero = false + t.testFloatWithTrailingZerosCase(false) +} + +func (t *testSyncerSuite) testFloatWithTrailingZerosCase(useTrailingZero bool) { + // Insert values with trailing zeros in JSON + t.testExecute(`INSERT INTO test_float_zeros VALUES (1, '{"f": 5.1}')`) + t.testExecute(`INSERT INTO test_float_zeros VALUES (2, '{"f": 1.100}')`) + + // Get current position + r, err := t.c.Execute("SHOW MASTER STATUS") + require.NoError(t.T(), err) + binFile, _ := r.GetString(0, 0) + binPos, _ := r.GetInt(0, 1) + + // Start syncing from current position + s, err := t.b.StartSync(mysql.Position{Name: binFile, Pos: uint32(binPos)}) + require.NoError(t.T(), err) + + // Insert another row to trigger binlog events + t.testExecute(`INSERT INTO test_float_zeros VALUES (3, '{"f": 3.0}')`) + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + + for { + evt, err := s.GetEvent(ctx) + require.NoError(t.T(), err) + + // We're interested in RowsEvent + if evt.Header.EventType != WRITE_ROWS_EVENTv2 { + continue + } + + // Type assert to RowsEvent + rowsEvent := evt.Event.(*RowsEvent) + for _, row := range rowsEvent.Rows { + // The third row should contain our test values + if row[0].(int32) == 3 { + // Get the JSON value from binlog + jsonVal := row[1].([]byte) + var data struct { + F float64 `json:"f"` + } + err := json.Unmarshal(jsonVal, &data) + require.NoError(t.T(), err) + + // Check if trailing zero is preserved based on useFloatWithTrailingZero + if useTrailingZero { + require.Equal(t.T(), "3.0", fmt.Sprintf("%.1f", data.F)) + } else { + require.Equal(t.T(), "3", fmt.Sprintf("%.1f", data.F)) + } + return + } + } + } +} From 82ebf931a772750e8d88ab4efce49813956562f9 Mon Sep 17 00:00:00 2001 From: Tiffany Yeh Date: Wed, 4 Jun 2025 12:38:17 -0400 Subject: [PATCH 4/6] fix linting and mysql errors on ci --- replication/replication_test.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/replication/replication_test.go b/replication/replication_test.go index b8e90b80a..a8e789688 100644 --- a/replication/replication_test.go +++ b/replication/replication_test.go @@ -14,11 +14,11 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" - "github.com/goccy/go-json" "github.com/go-mysql-org/go-mysql/client" "github.com/go-mysql-org/go-mysql/mysql" "github.com/go-mysql-org/go-mysql/test_util" "github.com/go-mysql-org/go-mysql/utils" + "github.com/goccy/go-json" ) var testOutputLogs = flag.Bool("out", false, "output binlog event") @@ -494,7 +494,11 @@ func (t *testSyncerSuite) testFloatWithTrailingZerosCase(useTrailingZero bool) { t.testExecute(`INSERT INTO test_float_zeros VALUES (2, '{"f": 1.100}')`) // Get current position - r, err := t.c.Execute("SHOW MASTER STATUS") + showBinlogStatus := "SHOW BINARY LOG STATUS" + if eq, err := t.c.CompareServerVersion("8.4.0"); (err == nil) && (eq < 0) { + showBinlogStatus = "SHOW MASTER STATUS" + } + r, err := t.c.Execute(showBinlogStatus) require.NoError(t.T(), err) binFile, _ := r.GetString(0, 0) binPos, _ := r.GetInt(0, 1) From beeeb017a58554b98e7887dc5c538ecfb8ec4e0c Mon Sep 17 00:00:00 2001 From: Tiffany Yeh Date: Thu, 5 Jun 2025 15:08:18 -0400 Subject: [PATCH 5/6] update unit tests --- replication/json_binary_test.go | 403 ++++++++++++++++++++++++++++++++ replication/replication_test.go | 82 ------- 2 files changed, 403 insertions(+), 82 deletions(-) create mode 100644 replication/json_binary_test.go diff --git a/replication/json_binary_test.go b/replication/json_binary_test.go new file mode 100644 index 000000000..01bb12352 --- /dev/null +++ b/replication/json_binary_test.go @@ -0,0 +1,403 @@ +package replication + +import ( + "testing" + + "github.com/goccy/go-json" + "github.com/stretchr/testify/require" +) + +func TestFloatWithTrailingZero_MarshalJSON(t *testing.T) { + tests := []struct { + name string + input FloatWithTrailingZero + expected string + }{ + { + name: "whole number should have .0", + input: FloatWithTrailingZero(5.0), + expected: "5.0", + }, + { + name: "negative whole number should have .0", + input: FloatWithTrailingZero(-3.0), + expected: "-3.0", + }, + { + name: "decimal number should preserve original format", + input: FloatWithTrailingZero(3.14), + expected: "3.14", + }, + { + name: "negative decimal should preserve original format", + input: FloatWithTrailingZero(-2.5), + expected: "-2.5", + }, + { + name: "zero should have .0", + input: FloatWithTrailingZero(0.0), + expected: "0.0", + }, + { + name: "very small decimal", + input: FloatWithTrailingZero(0.001), + expected: "0.001", + }, + { + name: "large whole number", + input: FloatWithTrailingZero(1000000.0), + expected: "1000000.0", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, err := tt.input.MarshalJSON() + require.NoError(t, err) + require.Equal(t, tt.expected, string(result)) + }) + } +} + +func TestRegularFloat64_MarshalJSON_TruncatesTrailingZero(t *testing.T) { + // Test that regular float64 truncates trailing zeros (the default behavior) + // This demonstrates the difference when UseFloatWithTrailingZero is NOT set + tests := []struct { + name string + input float64 + expected string + }{ + { + name: "whole number truncates .0", + input: 5.0, + expected: "5", + }, + { + name: "negative whole number truncates .0", + input: -3.0, + expected: "-3", + }, + { + name: "decimal number preserves decimals", + input: 3.14, + expected: "3.14", + }, + { + name: "negative decimal preserves decimals", + input: -2.5, + expected: "-2.5", + }, + { + name: "zero truncates .0", + input: 0.0, + expected: "0", + }, + { + name: "large whole number truncates .0", + input: 1000000.0, + expected: "1000000", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, err := json.Marshal(tt.input) + require.NoError(t, err) + require.Equal(t, tt.expected, string(result)) + }) + } +} + +func TestFloatWithTrailingZero_vs_RegularFloat64_Comparison(t *testing.T) { + // Direct comparison test showing the key difference + testCases := []struct { + name string + value float64 + withTrailing string // Expected output with FloatWithTrailingZero + withoutTrailing string // Expected output with regular float64 + }{ + { + name: "whole number 5.0", + value: 5.0, + withTrailing: "5.0", + withoutTrailing: "5", + }, + { + name: "zero", + value: 0.0, + withTrailing: "0.0", + withoutTrailing: "0", + }, + { + name: "negative whole number", + value: -42.0, + withTrailing: "-42.0", + withoutTrailing: "-42", + }, + { + name: "decimal number (no difference)", + value: 3.14, + withTrailing: "3.14", + withoutTrailing: "3.14", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // Test FloatWithTrailingZero + trailingResult, err := json.Marshal(FloatWithTrailingZero(tc.value)) + require.NoError(t, err) + require.Equal(t, tc.withTrailing, string(trailingResult)) + + // Test regular float64 + regularResult, err := json.Marshal(tc.value) + require.NoError(t, err) + require.Equal(t, tc.withoutTrailing, string(regularResult)) + + // Verify they're different for whole numbers + if tc.value == float64(int(tc.value)) { + require.NotEqual(t, string(trailingResult), string(regularResult), + "FloatWithTrailingZero and regular float64 should produce different output for whole numbers") + } + }) + } +} + +func TestJsonBinaryDecoder_decodeDoubleWithTrailingZero(t *testing.T) { + // Test the decodeDoubleWithTrailingZero method directly + decoder := &jsonBinaryDecoder{ + useFloatWithTrailingZero: true, + } + + // Test data representing float64 in binary format (little endian) + // 5.0 as IEEE 754 double precision: 0x4014000000000000 + testData := []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x14, 0x40} + + result := decoder.decodeDoubleWithTrailingZero(testData) + require.NoError(t, decoder.err) + + // Verify the result is FloatWithTrailingZero type + require.IsType(t, FloatWithTrailingZero(0), result) + require.Equal(t, FloatWithTrailingZero(5.0), result) + + // Test JSON marshaling + jsonBytes, err := json.Marshal(result) + require.NoError(t, err) + require.Equal(t, "5.0", string(jsonBytes)) +} + +func TestJsonBinaryDecoder_decodeValue_JSONB_DOUBLE(t *testing.T) { + tests := []struct { + name string + useFloatWithTrailingZero bool + expectedType interface{} + expectedJSONString string + }{ + { + name: "with trailing zero enabled", + useFloatWithTrailingZero: true, + expectedType: FloatWithTrailingZero(0), + expectedJSONString: "5.0", + }, + { + name: "with trailing zero disabled", + useFloatWithTrailingZero: false, + expectedType: float64(0), + expectedJSONString: "5", + }, + } + + // 5.0 as IEEE 754 double precision in little endian + testData := []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x14, 0x40} + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + decoder := &jsonBinaryDecoder{ + useFloatWithTrailingZero: tt.useFloatWithTrailingZero, + } + + result := decoder.decodeValue(JSONB_DOUBLE, testData) + require.NoError(t, decoder.err) + require.IsType(t, tt.expectedType, result) + + // Test JSON marshaling + jsonBytes, err := json.Marshal(result) + require.NoError(t, err) + require.Equal(t, tt.expectedJSONString, string(jsonBytes)) + }) + } +} + +func TestRowsEvent_decodeJsonBinary_WithFloatTrailingZero(t *testing.T) { + // Create a sample JSON binary data representing {"value": 5.0} + // This is a simplified test - in practice the binary format would be more complex + rowsEvent := &RowsEvent{ + useFloatWithTrailingZero: true, + } + + // Mock a simple JSON binary that would contain a double value + // In a real scenario, this would come from actual MySQL binlog data + // For this test, we'll create a minimal valid structure + + // This test would need actual MySQL JSON binary data to be fully functional + // For now, we'll test the decoding path exists and the option is respected + decoder := &jsonBinaryDecoder{ + useFloatWithTrailingZero: rowsEvent.useFloatWithTrailingZero, + } + + require.True(t, decoder.useFloatWithTrailingZero) +} + +func TestBinlogParser_SetUseFloatWithTrailingZero(t *testing.T) { + parser := NewBinlogParser() + + // Test default value + require.False(t, parser.useFloatWithTrailingZero) + + // Test setting to true + parser.SetUseFloatWithTrailingZero(true) + require.True(t, parser.useFloatWithTrailingZero) + + // Test setting to false + parser.SetUseFloatWithTrailingZero(false) + require.False(t, parser.useFloatWithTrailingZero) +} + +func TestBinlogSyncerConfig_UseFloatWithTrailingZero(t *testing.T) { + cfg := BinlogSyncerConfig{ + UseFloatWithTrailingZero: true, + } + + require.True(t, cfg.UseFloatWithTrailingZero) +} + +func TestFloatWithTrailingZero_EdgeCases(t *testing.T) { + tests := []struct { + name string + input FloatWithTrailingZero + expected string + }{ + { + name: "very large whole number", + input: FloatWithTrailingZero(1e15), + expected: "1000000000000000.0", + }, + { + name: "very small positive number", + input: FloatWithTrailingZero(1e-10), + expected: "0.0000000001", + }, + { + name: "scientific notation threshold", + input: FloatWithTrailingZero(1e6), + expected: "1000000.0", + }, + { + name: "negative zero", + input: FloatWithTrailingZero(-0.0), + expected: "0.0", + }, + { + name: "number that looks whole but has tiny fractional part", + input: FloatWithTrailingZero(5.0000000000000001), // This might be rounded to 5.0 due to float64 precision + expected: "5.0", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, err := tt.input.MarshalJSON() + require.NoError(t, err) + require.Equal(t, tt.expected, string(result)) + }) + } +} + +func TestFloatWithTrailingZero_Integration(t *testing.T) { + // Test that demonstrates the full flow with a sample JSON structure + type JSONData struct { + Price FloatWithTrailingZero `json:"price"` + Quantity FloatWithTrailingZero `json:"quantity"` + Total FloatWithTrailingZero `json:"total"` + } + + data := JSONData{ + Price: FloatWithTrailingZero(10.0), // Should become "10.0" + Quantity: FloatWithTrailingZero(2.5), // Should stay "2.5" + Total: FloatWithTrailingZero(25.0), // Should become "25.0" + } + + jsonBytes, err := json.Marshal(data) + require.NoError(t, err) + + expectedJSON := `{"price":10.0,"quantity":2.5,"total":25.0}` + require.Equal(t, expectedJSON, string(jsonBytes)) + + // Verify that regular float64 would behave differently + type RegularJSONData struct { + Price float64 `json:"price"` + Quantity float64 `json:"quantity"` + Total float64 `json:"total"` + } + + regularData := RegularJSONData{ + Price: 10.0, + Quantity: 2.5, + Total: 25.0, + } + + regularJSONBytes, err := json.Marshal(regularData) + require.NoError(t, err) + + regularExpectedJSON := `{"price":10,"quantity":2.5,"total":25}` + require.Equal(t, regularExpectedJSON, string(regularJSONBytes)) + + // Demonstrate the difference + require.NotEqual(t, string(jsonBytes), string(regularJSONBytes)) +} + +func TestRowsEvent_UseFloatWithTrailingZero_Integration(t *testing.T) { + // Test that RowsEvent properly propagates the useFloatWithTrailingZero setting + + // Create table map event (similar to existing tests in replication_test.go) + tableMapEventData := []byte("m\x00\x00\x00\x00\x00\x01\x00\x04test\x00\x03t10\x00\x02\xf5\xf6\x03\x04\n\x00\x03") + + tableMapEvent := new(TableMapEvent) + tableMapEvent.tableIDSize = 6 + err := tableMapEvent.Decode(tableMapEventData) + require.NoError(t, err) + + // Test with useFloatWithTrailingZero enabled + rowsWithTrailingZero := &RowsEvent{ + tableIDSize: 6, + tables: make(map[uint64]*TableMapEvent), + Version: 2, + useFloatWithTrailingZero: true, + } + rowsWithTrailingZero.tables[tableMapEvent.TableID] = tableMapEvent + + // Test with useFloatWithTrailingZero disabled + rowsWithoutTrailingZero := &RowsEvent{ + tableIDSize: 6, + tables: make(map[uint64]*TableMapEvent), + Version: 2, + useFloatWithTrailingZero: false, + } + rowsWithoutTrailingZero.tables[tableMapEvent.TableID] = tableMapEvent + + // Verify that the setting is properly stored + require.True(t, rowsWithTrailingZero.useFloatWithTrailingZero) + require.False(t, rowsWithoutTrailingZero.useFloatWithTrailingZero) + + // Test the decoder creation with the setting + decoderWithTrailing := &jsonBinaryDecoder{ + useFloatWithTrailingZero: rowsWithTrailingZero.useFloatWithTrailingZero, + } + + decoderWithoutTrailing := &jsonBinaryDecoder{ + useFloatWithTrailingZero: rowsWithoutTrailingZero.useFloatWithTrailingZero, + } + + require.True(t, decoderWithTrailing.useFloatWithTrailingZero) + require.False(t, decoderWithoutTrailing.useFloatWithTrailingZero) +} diff --git a/replication/replication_test.go b/replication/replication_test.go index a8e789688..d8bdb3b5e 100644 --- a/replication/replication_test.go +++ b/replication/replication_test.go @@ -18,7 +18,6 @@ import ( "github.com/go-mysql-org/go-mysql/mysql" "github.com/go-mysql-org/go-mysql/test_util" "github.com/go-mysql-org/go-mysql/utils" - "github.com/goccy/go-json" ) var testOutputLogs = flag.Bool("out", false, "output binlog event") @@ -465,84 +464,3 @@ func (t *testSyncerSuite) TestMysqlBinlogCodec() { require.NoError(t.T(), err) } } - -func (t *testSyncerSuite) TestFloatWithTrailingZeros() { - t.setupTest(mysql.MySQLFlavor) - - str := `DROP TABLE IF EXISTS test_float_zeros` - t.testExecute(str) - - // Create table with JSON column containing float values - str = `CREATE TABLE test_float_zeros ( - id INT PRIMARY KEY, - json_val JSON - )` - t.testExecute(str) - - // Test with useFloatWithTrailingZero = true - t.b.cfg.UseFloatWithTrailingZero = true - t.testFloatWithTrailingZerosCase(true) - - // Test with useFloatWithTrailingZero = false - t.b.cfg.UseFloatWithTrailingZero = false - t.testFloatWithTrailingZerosCase(false) -} - -func (t *testSyncerSuite) testFloatWithTrailingZerosCase(useTrailingZero bool) { - // Insert values with trailing zeros in JSON - t.testExecute(`INSERT INTO test_float_zeros VALUES (1, '{"f": 5.1}')`) - t.testExecute(`INSERT INTO test_float_zeros VALUES (2, '{"f": 1.100}')`) - - // Get current position - showBinlogStatus := "SHOW BINARY LOG STATUS" - if eq, err := t.c.CompareServerVersion("8.4.0"); (err == nil) && (eq < 0) { - showBinlogStatus = "SHOW MASTER STATUS" - } - r, err := t.c.Execute(showBinlogStatus) - require.NoError(t.T(), err) - binFile, _ := r.GetString(0, 0) - binPos, _ := r.GetInt(0, 1) - - // Start syncing from current position - s, err := t.b.StartSync(mysql.Position{Name: binFile, Pos: uint32(binPos)}) - require.NoError(t.T(), err) - - // Insert another row to trigger binlog events - t.testExecute(`INSERT INTO test_float_zeros VALUES (3, '{"f": 3.0}')`) - - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) - defer cancel() - - for { - evt, err := s.GetEvent(ctx) - require.NoError(t.T(), err) - - // We're interested in RowsEvent - if evt.Header.EventType != WRITE_ROWS_EVENTv2 { - continue - } - - // Type assert to RowsEvent - rowsEvent := evt.Event.(*RowsEvent) - for _, row := range rowsEvent.Rows { - // The third row should contain our test values - if row[0].(int32) == 3 { - // Get the JSON value from binlog - jsonVal := row[1].([]byte) - var data struct { - F float64 `json:"f"` - } - err := json.Unmarshal(jsonVal, &data) - require.NoError(t.T(), err) - - // Check if trailing zero is preserved based on useFloatWithTrailingZero - if useTrailingZero { - require.Equal(t.T(), "3.0", fmt.Sprintf("%.1f", data.F)) - } else { - require.Equal(t.T(), "3", fmt.Sprintf("%.1f", data.F)) - } - return - } - } - } -} From 33f2c02bb86affdf636276c8098115d65a0315b6 Mon Sep 17 00:00:00 2001 From: Tiffany Yeh Date: Fri, 6 Jun 2025 22:43:57 -0400 Subject: [PATCH 6/6] remove and update tests, and fix linting errors --- replication/json_binary_test.go | 75 +++++++++++++-------------------- 1 file changed, 30 insertions(+), 45 deletions(-) diff --git a/replication/json_binary_test.go b/replication/json_binary_test.go index 01bb12352..c0774f6cd 100644 --- a/replication/json_binary_test.go +++ b/replication/json_binary_test.go @@ -1,6 +1,8 @@ package replication import ( + "encoding/binary" + "math" "testing" "github.com/goccy/go-json" @@ -169,9 +171,9 @@ func TestJsonBinaryDecoder_decodeDoubleWithTrailingZero(t *testing.T) { useFloatWithTrailingZero: true, } - // Test data representing float64 in binary format (little endian) - // 5.0 as IEEE 754 double precision: 0x4014000000000000 - testData := []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x14, 0x40} + // Test data representing 5.0 as IEEE 754 double precision in little endian binary format + testData := make([]byte, 8) + binary.LittleEndian.PutUint64(testData, math.Float64bits(5.0)) result := decoder.decodeDoubleWithTrailingZero(testData) require.NoError(t, decoder.err) @@ -190,28 +192,46 @@ func TestJsonBinaryDecoder_decodeValue_JSONB_DOUBLE(t *testing.T) { tests := []struct { name string useFloatWithTrailingZero bool + value float64 expectedType interface{} expectedJSONString string }{ { - name: "with trailing zero enabled", + name: "positive number with trailing zero enabled", useFloatWithTrailingZero: true, + value: 5.0, expectedType: FloatWithTrailingZero(0), expectedJSONString: "5.0", }, { - name: "with trailing zero disabled", + name: "positive number with trailing zero disabled", useFloatWithTrailingZero: false, + value: 5.0, expectedType: float64(0), expectedJSONString: "5", }, + { + name: "negative zero with trailing zero enabled", + useFloatWithTrailingZero: true, + value: math.Copysign(0.0, -1), + expectedType: FloatWithTrailingZero(0), + expectedJSONString: "-0.0", + }, + { + name: "negative zero with trailing zero disabled", + useFloatWithTrailingZero: false, + value: math.Copysign(0.0, -1), + expectedType: float64(0), + expectedJSONString: "-0", + }, } - // 5.0 as IEEE 754 double precision in little endian - testData := []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x14, 0x40} - for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + // Test data as IEEE 754 double precision in little endian binary format + testData := make([]byte, 8) + binary.LittleEndian.PutUint64(testData, math.Float64bits(tt.value)) + decoder := &jsonBinaryDecoder{ useFloatWithTrailingZero: tt.useFloatWithTrailingZero, } @@ -228,26 +248,6 @@ func TestJsonBinaryDecoder_decodeValue_JSONB_DOUBLE(t *testing.T) { } } -func TestRowsEvent_decodeJsonBinary_WithFloatTrailingZero(t *testing.T) { - // Create a sample JSON binary data representing {"value": 5.0} - // This is a simplified test - in practice the binary format would be more complex - rowsEvent := &RowsEvent{ - useFloatWithTrailingZero: true, - } - - // Mock a simple JSON binary that would contain a double value - // In a real scenario, this would come from actual MySQL binlog data - // For this test, we'll create a minimal valid structure - - // This test would need actual MySQL JSON binary data to be fully functional - // For now, we'll test the decoding path exists and the option is respected - decoder := &jsonBinaryDecoder{ - useFloatWithTrailingZero: rowsEvent.useFloatWithTrailingZero, - } - - require.True(t, decoder.useFloatWithTrailingZero) -} - func TestBinlogParser_SetUseFloatWithTrailingZero(t *testing.T) { parser := NewBinlogParser() @@ -263,14 +263,6 @@ func TestBinlogParser_SetUseFloatWithTrailingZero(t *testing.T) { require.False(t, parser.useFloatWithTrailingZero) } -func TestBinlogSyncerConfig_UseFloatWithTrailingZero(t *testing.T) { - cfg := BinlogSyncerConfig{ - UseFloatWithTrailingZero: true, - } - - require.True(t, cfg.UseFloatWithTrailingZero) -} - func TestFloatWithTrailingZero_EdgeCases(t *testing.T) { tests := []struct { name string @@ -292,11 +284,6 @@ func TestFloatWithTrailingZero_EdgeCases(t *testing.T) { input: FloatWithTrailingZero(1e6), expected: "1000000.0", }, - { - name: "negative zero", - input: FloatWithTrailingZero(-0.0), - expected: "0.0", - }, { name: "number that looks whole but has tiny fractional part", input: FloatWithTrailingZero(5.0000000000000001), // This might be rounded to 5.0 due to float64 precision @@ -367,20 +354,18 @@ func TestRowsEvent_UseFloatWithTrailingZero_Integration(t *testing.T) { err := tableMapEvent.Decode(tableMapEventData) require.NoError(t, err) + require.Greater(t, tableMapEvent.TableID, uint64(0)) + // Test with useFloatWithTrailingZero enabled rowsWithTrailingZero := &RowsEvent{ - tableIDSize: 6, tables: make(map[uint64]*TableMapEvent), - Version: 2, useFloatWithTrailingZero: true, } rowsWithTrailingZero.tables[tableMapEvent.TableID] = tableMapEvent // Test with useFloatWithTrailingZero disabled rowsWithoutTrailingZero := &RowsEvent{ - tableIDSize: 6, tables: make(map[uint64]*TableMapEvent), - Version: 2, useFloatWithTrailingZero: false, } rowsWithoutTrailingZero.tables[tableMapEvent.TableID] = tableMapEvent