Skip to content

[refactor](type) Add FIXED_LENGTH_OBJECT type #51577

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 19 additions & 18 deletions be/src/runtime/define_primitive_type.h
Original file line number Diff line number Diff line change
Expand Up @@ -50,24 +50,25 @@ enum PrimitiveType : PrimitiveNative {

TYPE_TIME [[deprecated]], /*TYPE_TIMEV2*/

TYPE_BITMAP, /* 22, bitmap */
TYPE_STRING, /* 23 */
TYPE_QUANTILE_STATE, /* 24 */
TYPE_DATEV2, /* 25 */
TYPE_DATETIMEV2, /* 26 */
TYPE_TIMEV2, /* 27 */
TYPE_DECIMAL32, /* 28 */
TYPE_DECIMAL64, /* 29 */
TYPE_DECIMAL128I, /* 30, v3 128bit */
TYPE_JSONB, /* 31 */
TYPE_VARIANT, /* 32 */
TYPE_LAMBDA_FUNCTION, /* 33 */
TYPE_AGG_STATE, /* 34 */
TYPE_DECIMAL256, /* 35 */
TYPE_IPV4, /* 36 */
TYPE_IPV6, /* 37 */
TYPE_UINT32, /* 38, used as offset */
TYPE_UINT64 /* 39, used as offset */
TYPE_BITMAP, /* 22, bitmap */
TYPE_STRING, /* 23 */
TYPE_QUANTILE_STATE, /* 24 */
TYPE_DATEV2, /* 25 */
TYPE_DATETIMEV2, /* 26 */
TYPE_TIMEV2, /* 27 */
TYPE_DECIMAL32, /* 28 */
TYPE_DECIMAL64, /* 29 */
TYPE_DECIMAL128I, /* 30, v3 128bit */
TYPE_JSONB, /* 31 */
TYPE_VARIANT, /* 32 */
TYPE_LAMBDA_FUNCTION, /* 33 */
TYPE_AGG_STATE, /* 34 */
TYPE_DECIMAL256, /* 35 */
TYPE_IPV4, /* 36 */
TYPE_IPV6, /* 37 */
TYPE_UINT32, /* 38, used as offset */
TYPE_UINT64, /* 39, used as offset */
TYPE_FIXED_LENGTH_OBJECT /* 40, represent fixed-length object on BE */
};

} // namespace doris
9 changes: 2 additions & 7 deletions be/src/vec/data_types/data_type.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -169,13 +169,8 @@ PGenericType_TypeId IDataType::get_pdata_type(const IDataType* data_type) {
return PGenericType::AGG_STATE;
case PrimitiveType::TYPE_TIMEV2:
return PGenericType::TIMEV2;
case PrimitiveType::INVALID_TYPE: {
// TODO(gabriel): we need another primitive type to represent FixedLengthObject
if (data_type->is_fixed_length_object()) {
return PGenericType::FIXEDLENGTHOBJECT;
}
break;
}
case PrimitiveType::TYPE_FIXED_LENGTH_OBJECT:
return PGenericType::FIXEDLENGTHOBJECT;
default:
break;
}
Expand Down
2 changes: 0 additions & 2 deletions be/src/vec/data_types/data_type.h
Original file line number Diff line number Diff line change
Expand Up @@ -97,8 +97,6 @@ class IDataType : private boost::noncopyable {

// get specific serializer or deserializer
virtual DataTypeSerDeSPtr get_serde(int nesting_level = 1) const = 0;
// FIXME(gabriel):Fixed length object type will be used in some cases but it is not supported by primitive type
virtual bool is_fixed_length_object() const { return false; }

protected:
virtual String do_get_name() const;
Expand Down
6 changes: 2 additions & 4 deletions be/src/vec/data_types/data_type_fixed_length_object.h
Original file line number Diff line number Diff line change
Expand Up @@ -38,15 +38,15 @@ class IColumn;
class DataTypeFixedLengthObject final : public IDataType {
public:
using ColumnType = ColumnFixedLengthObject;
static constexpr PrimitiveType PType = INVALID_TYPE;
static constexpr PrimitiveType PType = TYPE_FIXED_LENGTH_OBJECT;

DataTypeFixedLengthObject() = default;

DataTypeFixedLengthObject(const DataTypeFixedLengthObject& other) {}

const std::string get_family_name() const override { return "DataTypeFixedLengthObject"; }

PrimitiveType get_primitive_type() const override { return PrimitiveType::INVALID_TYPE; }
PrimitiveType get_primitive_type() const override { return PType; }

doris::FieldType get_storage_field_type() const override {
return doris::FieldType::OLAP_FIELD_TYPE_NONE;
Expand All @@ -57,7 +57,6 @@ class DataTypeFixedLengthObject final : public IDataType {
[[noreturn]] Field get_field(const TExprNode& node) const override {
throw doris::Exception(ErrorCode::NOT_IMPLEMENTED_ERROR,
"Unimplemented get_field for DataTypeFixedLengthObject");
__builtin_unreachable();
}

bool equals(const IDataType& rhs) const override { return typeid(rhs) == typeid(*this); }
Expand All @@ -76,7 +75,6 @@ class DataTypeFixedLengthObject final : public IDataType {
DataTypeSerDeSPtr get_serde(int nesting_level = 1) const override {
return std::make_shared<DataTypeFixedLengthObjectSerDe>(nesting_level);
};
bool is_fixed_length_object() const override { return true; }
};

} // namespace doris::vectorized
3 changes: 0 additions & 3 deletions be/src/vec/data_types/data_type_nullable.h
Original file line number Diff line number Diff line change
Expand Up @@ -106,9 +106,6 @@ class DataTypeNullable final : public IDataType {
return 1 + nested_data_type->get_size_of_value_in_memory();
}
bool is_nullable() const override { return true; }
bool is_fixed_length_object() const override {
return nested_data_type->is_fixed_length_object();
}
bool can_be_inside_low_cardinality() const override {
return nested_data_type->can_be_inside_low_cardinality();
}
Expand Down
3 changes: 1 addition & 2 deletions be/test/vec/data_types/common_data_type_test.h
Original file line number Diff line number Diff line change
Expand Up @@ -139,8 +139,7 @@ class CommonDataTypeTest : public ::testing::Test {
meta_info.is_value_unambiguously_represented_in_contiguous_memory_region);
if (is_decimal(data_type->get_primitive_type()) || data_type->is_nullable() ||
data_type->get_primitive_type() == TYPE_STRUCT ||
(data_type->get_primitive_type() == INVALID_TYPE &&
!data_type->is_fixed_length_object()) ||
data_type->get_primitive_type() == INVALID_TYPE ||
is_number(data_type->get_primitive_type()) ||
is_int_or_bool(data_type->get_primitive_type()) ||
is_float_or_double(data_type->get_primitive_type()) ||
Expand Down
7 changes: 4 additions & 3 deletions be/test/vec/data_types/data_type_agg_state_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -140,9 +140,10 @@ void insert_data_agg_state(MutableColumns* agg_state_cols, DataTypePtr datatype_
} else {
assert_cast<ColumnFixedLengthObject*>((*agg_state_cols)[0].get())->set_item_size(8);
column_fixed->resize(rows_value);
ASSERT_TRUE(assert_cast<const DataTypeAggState*>(datatype_agg_state.get())
->get_serialized_type()
->is_fixed_length_object());
ASSERT_EQ(assert_cast<const DataTypeAggState*>(datatype_agg_state.get())
->get_serialized_type()
->get_primitive_type(),
TYPE_FIXED_LENGTH_OBJECT);
auto& data = assert_cast<ColumnFixedLengthObject*>((*agg_state_cols)[0].get())->get_data();
for (size_t i = 0; i != rows_value; ++i) {
data[i] = i;
Expand Down
3 changes: 2 additions & 1 deletion gensrc/thrift/Types.thrift
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,8 @@ enum TPrimitiveType {
IPV4,
IPV6,
UINT32, // only used in BE to represent offsets
UINT64 // only used in BE to represent offsets
UINT64, // only used in BE to represent offsets
FIXED_LENGTH_OBJECT // only used in BE to represent fixed-length object
}

enum TTypeNodeType {
Expand Down
Loading