Skip to content

Commit 4cbb62e

Browse files
JaySon-Huangti-chi-bot
JaySon-Huang
authored and
ti-chi-bot
committed
This is an automated cherry-pick of pingcap#10682
Signed-off-by: ti-chi-bot <ti-community-prow-bot@tidb.io>
1 parent 68d27be commit 4cbb62e

File tree

8 files changed

+847
-15
lines changed

8 files changed

+847
-15
lines changed

‎dbms/src/Storages/KVStore/Decode/RegionBlockReader.cpp‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,44 @@ bool RegionBlockReader::readImpl(Block & block, const RegionDataReadInfoList & d
111111
if (unlikely(block.columns() != schema_snapshot->column_defines->size()))
112112
throw Exception("block structure doesn't match schema_snapshot.", ErrorCodes::LOGICAL_ERROR);
113113

114+
<<<<<<< HEAD
114115
const auto & read_column_ids = schema_snapshot->sorted_column_id_with_pos;
116+
=======
117+
private:
118+
ColumnUInt64 * raw_version_col = nullptr;
119+
};
120+
121+
template <>
122+
struct VersionColResolver<RegionUncommittedDataList>
123+
{
124+
VersionColResolver() = default;
125+
bool needBuild() const { return false; } // NOLINT conform to main template
126+
void build(ColumnUInt64 * raw_version_col_) { raw_version_col = raw_version_col_; }
127+
void preRead(size_t) {} // NOLINT conform to main template
128+
void read(const RegionUncommittedData &) {} // NOLINT conform to main template
129+
void check(const Block & block, size_t expected) const // NOLINT conform to main template
130+
{
131+
if (unlikely(block.columns() + 1 != expected))
132+
throw Exception(
133+
ErrorCodes::LOGICAL_ERROR,
134+
"Block structure doesn't match schema_snapshot, block={} def={}",
135+
block.columns(),
136+
expected);
137+
}
138+
size_t reservedCount() const { return 2; } // NOLINT conform to main template
139+
140+
private:
141+
ColumnUInt64 * raw_version_col = nullptr;
142+
};
143+
144+
template <TMTPKType pk_type, typename ReadList>
145+
bool RegionBlockReader::readImpl(Block & block, const ReadList & data_list, bool force_decode)
146+
{
147+
VersionColResolver<ReadList> version_col_resolver;
148+
version_col_resolver.check(block, schema_snapshot->column_defines->size());
149+
// The column_ids to read according to schema_snapshot, each elem is (column_id, block_pos)
150+
const auto & read_column_ids = schema_snapshot->getColId2BlockPosMap();
151+
>>>>>>> 01b12dd900 (ddl: Fix default value filling with finer granularity (#10682))
115152
const auto & pk_column_ids = schema_snapshot->pk_column_ids;
116153
const auto & pk_pos_map = schema_snapshot->pk_pos_map;
117154

@@ -199,6 +236,8 @@ bool RegionBlockReader::readImpl(Block & block, const RegionDataReadInfoList & d
199236
else
200237
{
201238
// Parse column value from encoded value
239+
// Decode the column_ids from `column_ids_iter` to `read_column_ids.end()`
240+
// and insert into `block` at position starting from `next_column_pos`
202241
if (!appendRowToBlock(
203242
*value_ptr,
204243
column_ids_iter,

‎dbms/src/Storages/KVStore/tests/gtest_region_block_reader.cpp‎

Lines changed: 173 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -691,5 +691,178 @@ try
691691
}
692692
CATCH
693693

694+
TEST_F(RegionBlockReaderTest, ReadFromRegionDefaultValue)
695+
try
696+
{
697+
// With this table_info, column "c1" is "NOT NULL" and has no origin default
698+
TableInfo table_info_c1_not_null_no_origin_default(
699+
R"({"cols":[{"id":1,"name":{"L":"c0","O":"c0"},"offset":0,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":0,"Flen":4,"Tp":1}},{"id":2,"name":{"L":"handle","O":"handle"},"offset":1,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":515,"Flen":11,"Tp":3}},{"default":"-56083770","id":7,"name":{"L":"c1","O":"c1"},"offset":2,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":1,"Flen":20,"Tp":8}},{"id":4,"name":{"L":"c2","O":"c2"},"offset":3,"origin_default":"0.07954397","state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":-1,"Flag":4097,"Flen":12,"Tp":4}},{"id":5,"name":{"L":"c5","O":"c5"},"offset":4,"origin_default":"0","state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":0,"Flen":10,"Tp":246}},{"default":"247262911","id":6,"name":{"L":"c4","O":"c4"},"offset":5,"origin_default":"247262911","state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":0,"Flen":10,"Tp":246}}],"id":636,"index_info":[],"is_common_handle":false,"keyspace_id":4294967295,"name":{"L":"t0","O":"t0"},"pk_is_handle":true,"state":5,"tiflash_replica":{"Available":true,"Count":1},"update_timestamp":463845180343844895})",
700+
NullspaceID);
701+
702+
// With this table_info, column "c1" is "NOT NULL" and has no default value
703+
TableInfo table_info_c1_not_null_no_default_value(
704+
R"({"cols":[{"id":1,"name":{"L":"c0","O":"c0"},"offset":0,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":0,"Flen":4,"Tp":1}},{"id":2,"name":{"L":"handle","O":"handle"},"offset":1,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":515,"Flen":11,"Tp":3}},{"id":7,"name":{"L":"c1","O":"c1"},"offset":2,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":4097,"Flen":20,"Tp":8}},{"id":4,"name":{"L":"c2","O":"c2"},"offset":3,"origin_default":"0.07954397","state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":-1,"Flag":4097,"Flen":12,"Tp":4}},{"id":5,"name":{"L":"c5","O":"c5"},"offset":4,"origin_default":"0","state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":0,"Flen":10,"Tp":246}},{"default":"247262911","id":6,"name":{"L":"c4","O":"c4"},"offset":5,"origin_default":"247262911","state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":0,"Flen":10,"Tp":246}}],"id":636,"index_info":[],"is_common_handle":false,"keyspace_id":4294967295,"name":{"L":"t0","O":"t0"},"pk_is_handle":true,"state":5,"tiflash_replica":{"Available":true,"Count":1},"update_timestamp":463845180343844895})",
705+
NullspaceID);
706+
707+
// With this table_info, column "c1" has the "NOT NULL" flag and has origin default "-56083770"
708+
TableInfo table_info_c1_not_null_with_origin_default(
709+
R"({"cols":[{"id":1,"name":{"L":"c0","O":"c0"},"offset":0,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":0,"Flen":4,"Tp":1}},{"id":2,"name":{"L":"handle","O":"handle"},"offset":1,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":515,"Flen":11,"Tp":3}},{"origin_default":"-56083770","id":7,"name":{"L":"c1","O":"c1"},"offset":2,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":1,"Flen":20,"Tp":8}},{"id":4,"name":{"L":"c2","O":"c2"},"offset":3,"origin_default":"0.07954397","state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":-1,"Flag":4097,"Flen":12,"Tp":4}},{"id":5,"name":{"L":"c5","O":"c5"},"offset":4,"origin_default":"0","state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":0,"Flen":10,"Tp":246}},{"default":"247262911","id":6,"name":{"L":"c4","O":"c4"},"offset":5,"origin_default":"247262911","state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":0,"Flen":10,"Tp":246}}],"id":636,"index_info":[],"is_common_handle":false,"keyspace_id":4294967295,"name":{"L":"t0","O":"t0"},"pk_is_handle":true,"state":5,"tiflash_replica":{"Available":true,"Count":1},"update_timestamp":463845180343844895})",
710+
NullspaceID);
711+
712+
// With this table_info, column "c1" has the "NOT NULL" flag and has origin default "-56083770", but the state is not public
713+
TableInfo table_info_c1_not_null_with_origin_default_non_public(
714+
R"({"cols":[{"id":1,"name":{"L":"c0","O":"c0"},"offset":0,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":0,"Flen":4,"Tp":1}},{"id":2,"name":{"L":"handle","O":"handle"},"offset":1,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":515,"Flen":11,"Tp":3}},{"origin_default":"-56083770","id":7,"name":{"L":"c1","O":"c1"},"offset":2,"state":3,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":1,"Flen":20,"Tp":8}},{"id":4,"name":{"L":"c2","O":"c2"},"offset":3,"origin_default":"0.07954397","state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":-1,"Flag":4097,"Flen":12,"Tp":4}},{"id":5,"name":{"L":"c5","O":"c5"},"offset":4,"origin_default":"0","state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":0,"Flen":10,"Tp":246}},{"default":"247262911","id":6,"name":{"L":"c4","O":"c4"},"offset":5,"origin_default":"247262911","state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":0,"Flen":10,"Tp":246}}],"id":636,"index_info":[],"is_common_handle":false,"keyspace_id":4294967295,"name":{"L":"t0","O":"t0"},"pk_is_handle":true,"state":5,"tiflash_replica":{"Available":true,"Count":1},"update_timestamp":463845180343844895})",
715+
NullspaceID);
716+
717+
// With this table_info, column "c1" does not have the "NOT NULL" flag
718+
TableInfo table_info_c1_nullable(
719+
R"({"cols":[{"id":1,"name":{"L":"c0","O":"c0"},"offset":0,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":0,"Flen":4,"Tp":1}},{"id":2,"name":{"L":"handle","O":"handle"},"offset":1,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":515,"Flen":11,"Tp":3}},{"id":7,"name":{"L":"c1","O":"c1"},"offset":2,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":0,"Flen":20,"Tp":8}},{"id":4,"name":{"L":"c2","O":"c2"},"offset":3,"origin_default":"0.07954397","state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":-1,"Flag":4097,"Flen":12,"Tp":4}},{"id":5,"name":{"L":"c5","O":"c5"},"offset":4,"origin_default":"0","state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":0,"Flen":10,"Tp":246}},{"default":"247262911","id":6,"name":{"L":"c4","O":"c4"},"offset":5,"origin_default":"247262911","state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":0,"Flen":10,"Tp":246}}],"id":636,"index_info":[],"is_common_handle":false,"keyspace_id":4294967295,"name":{"L":"t0","O":"t0"},"pk_is_handle":true,"state":5,"tiflash_replica":{"Available":true,"Count":1},"update_timestamp":463844343842340870})",
720+
NullspaceID);
721+
722+
RegionID region_id = 4;
723+
// the start_key and end_key for table_id = 68
724+
String region_start_key(bytesFromHexString("7480000000000002FF7C5F720000000000FA"));
725+
String region_end_key(bytesFromHexString("7480000000000002FF7D00000000000000F8"));
726+
auto region = RegionBench::makeRegionForRange(region_id, region_start_key, region_end_key);
727+
// the hex kv dump from RaftLog
728+
std::vector<std::tuple<std::string_view, std::string_view>> kvs = {
729+
{
730+
"7480000000000002FFA95F728000000000FF0000010000000000FAF9901806DEF7FFDA",
731+
"50A380A08892FFF9B706762C8000040000000405060708000F0016001A00BFDC4011A00000000A0080000000000A008000003CA339"
732+
"1ABC85",
733+
},
734+
{
735+
"7480000000000002FFA95F728000000000FF0000010000000000FAF9901806DEF7FFD8",
736+
"50A680A08892FFF9B706762C8000040000000405060708000F0016001A00BFDC4011A00000000A008033E04D600A008000003CA339"
737+
"1ABC85",
738+
},
739+
{
740+
"7480000000000002FFA95F728000000000FF0000020000000000FAF9901806DE33FFE8",
741+
"509680B08E92FFF9B706762580000300000004050608000F001600BF720CDD400000000A0080000000010A00800000393C",
742+
},
743+
};
744+
for (const auto & [k, v] : kvs)
745+
{
746+
region->insertDebug("write", TiKVKey(bytesFromHexString(k)), TiKVValue(bytesFromHexString(v)));
747+
}
748+
749+
auto data_list_read = ReadRegionCommitCache(region, true);
750+
ASSERT_TRUE(data_list_read.has_value());
751+
752+
// Test with `table_info_c1_not_null_no_origin_default`
753+
auto decoding_schema = getDecodingStorageSchemaSnapshot(table_info_c1_not_null_no_origin_default);
754+
{
755+
// force_decode=false can not decode because there are
756+
// missing value for column with not null flag.
757+
auto reader = RegionBlockReader(decoding_schema);
758+
Block res_block = createBlockSortByColumnID(decoding_schema);
759+
ASSERT_FALSE(reader.read(res_block, *data_list_read, false));
760+
}
761+
{
762+
// force_decode=true can decode the block, and filling the default value for c1
763+
auto reader = RegionBlockReader(decoding_schema);
764+
Block res_block = createBlockSortByColumnID(decoding_schema);
765+
ASSERT_TRUE(reader.read(res_block, *data_list_read, true));
766+
LOG_INFO(
767+
Logger::get(),
768+
"Decoded block:\n{}",
769+
DB::tests::getColumnsContent(res_block.getColumnsWithTypeAndName()));
770+
ASSERT_EQ(res_block.getByName("c1").type->getName(), "Int64");
771+
// verify the default value is filled correctly
772+
ASSERT_COLUMN_EQ( //
773+
res_block.getByName("c1"),
774+
createColumn<Int64>({-2051270087, -2051270087, 0}));
775+
}
776+
777+
// Test with `table_info_c1_not_null_no_default_value`
778+
decoding_schema = getDecodingStorageSchemaSnapshot(table_info_c1_not_null_no_default_value);
779+
{
780+
// force_decode=false can not decode because there are
781+
// missing value for column with not null flag.
782+
auto reader = RegionBlockReader(decoding_schema);
783+
Block res_block = createBlockSortByColumnID(decoding_schema);
784+
ASSERT_FALSE(reader.read(res_block, *data_list_read, false));
785+
}
786+
{
787+
// force_decode=true can decode the block, and filling the default value for c1
788+
auto reader = RegionBlockReader(decoding_schema);
789+
Block res_block = createBlockSortByColumnID(decoding_schema);
790+
ASSERT_TRUE(reader.read(res_block, *data_list_read, true));
791+
LOG_INFO(
792+
Logger::get(),
793+
"Decoded block:\n{}",
794+
DB::tests::getColumnsContent(res_block.getColumnsWithTypeAndName()));
795+
ASSERT_EQ(res_block.getByName("c1").type->getName(), "Int64");
796+
// verify the default value is filled correctly
797+
ASSERT_COLUMN_EQ( //
798+
res_block.getByName("c1"),
799+
createColumn<Int64>({-2051270087, -2051270087, 0}));
800+
}
801+
802+
// Test with `table_info_c1_not_null_with_origin_default`
803+
decoding_schema = getDecodingStorageSchemaSnapshot(table_info_c1_not_null_with_origin_default);
804+
{
805+
// force_decode=false can decode because origin_default exists and NoDefaultValue flag is not set
806+
// so RegionBlockReader can use origin_default to fill the missing value`
807+
auto reader = RegionBlockReader(decoding_schema);
808+
Block res_block = createBlockSortByColumnID(decoding_schema);
809+
ASSERT_TRUE(reader.read(res_block, *data_list_read, false));
810+
LOG_INFO(
811+
Logger::get(),
812+
"Decoded block:\n{}",
813+
DB::tests::getColumnsContent(res_block.getColumnsWithTypeAndName()));
814+
ASSERT_EQ(res_block.getByName("c1").type->getName(), "Int64");
815+
// verify the default value is filled correctly
816+
ASSERT_COLUMN_EQ( //
817+
res_block.getByName("c1"),
818+
// the thrid elem is filled wih origin_default
819+
createColumn<Int64>({-2051270087, -2051270087, -56083770}));
820+
}
821+
822+
// Test with `table_info_c1_not_null_with_origin_default_non_public`
823+
decoding_schema = getDecodingStorageSchemaSnapshot(table_info_c1_not_null_with_origin_default_non_public);
824+
{
825+
// force_decode=false can not decode because there are
826+
// missing value for column with not null flag and the state is not public
827+
auto reader = RegionBlockReader(decoding_schema);
828+
Block res_block = createBlockSortByColumnID(decoding_schema);
829+
ASSERT_FALSE(reader.read(res_block, *data_list_read, false));
830+
}
831+
{
832+
// force_decode=true can decode the block, and filling the default value for c1
833+
auto reader = RegionBlockReader(decoding_schema);
834+
Block res_block = createBlockSortByColumnID(decoding_schema);
835+
ASSERT_TRUE(reader.read(res_block, *data_list_read, true));
836+
LOG_INFO(
837+
Logger::get(),
838+
"Decoded block:\n{}",
839+
DB::tests::getColumnsContent(res_block.getColumnsWithTypeAndName()));
840+
ASSERT_EQ(res_block.getByName("c1").type->getName(), "Int64");
841+
// verify the default value is filled correctly
842+
ASSERT_COLUMN_EQ( //
843+
res_block.getByName("c1"),
844+
// the thrid elem is filled wih origin_default
845+
createColumn<Int64>({-2051270087, -2051270087, -56083770}));
846+
}
847+
848+
// Test with `table_info_c1_nullable`
849+
decoding_schema = getDecodingStorageSchemaSnapshot(table_info_c1_nullable);
850+
{
851+
// force_decode=false should be able to decode because c1 is nullable
852+
auto reader = RegionBlockReader(decoding_schema);
853+
Block res_block = createBlockSortByColumnID(decoding_schema);
854+
ASSERT_TRUE(reader.read(res_block, *data_list_read, false));
855+
LOG_INFO(
856+
Logger::get(),
857+
"Decoded block:\n{}",
858+
DB::tests::getColumnsContent(res_block.getColumnsWithTypeAndName()));
859+
ASSERT_EQ(res_block.getByName("c1").type->getName(), "Nullable(Int64)");
860+
// verify the default value is filled with NULL correctly at the last row
861+
ASSERT_COLUMN_EQ( //
862+
res_block.getByName("c1"),
863+
createNullableColumn<Int64>({-2051270087, -2051270087, 0}, {0, 0, 1}));
864+
}
865+
}
866+
CATCH
694867

695868
} // namespace DB::tests

0 commit comments

Comments
 (0)