4 Commits

Author SHA1 Message Date
rulego-team b23fdea2cd perf:增加代码覆盖率 2025-08-29 17:30:58 +08:00
rulego-team 696b5f7177 perf:优化sql语法解析 2025-08-29 17:30:11 +08:00
rulego-team 90afdead78 perf:重构聚合函数运算 2025-08-29 17:29:27 +08:00
rulego-team 4615b7a308 feat:支持聚合函数的后运算 #37 2025-08-29 13:49:26 +08:00
30 changed files with 4823 additions and 1321 deletions
+24
View File
@@ -20,6 +20,7 @@ const (
WindowStart = functions.WindowStart
WindowEnd = functions.WindowEnd
Collect = functions.Collect
FirstValue = functions.FirstValue
LastValue = functions.LastValue
MergeAgg = functions.MergeAgg
StdDevS = functions.StdDevS
@@ -33,6 +34,8 @@ const (
HadChanged = functions.HadChanged
// Expression aggregator for handling custom functions
Expression = functions.Expression
// Post-aggregation marker
PostAggregation = functions.PostAggregation
)
// AggregatorFunction aggregator function interface, re-exports functions.LegacyAggregatorFunction
@@ -55,9 +58,30 @@ func CreateBuiltinAggregator(aggType AggregateType) AggregatorFunction {
}
}
// Special handling for post-aggregation type (placeholder aggregator)
if aggType == "post_aggregation" {
return &PostAggregationPlaceholder{}
}
return functions.CreateLegacyAggregator(aggType)
}
// PostAggregationPlaceholder is a placeholder aggregator for post-aggregation fields
type PostAggregationPlaceholder struct{}
func (p *PostAggregationPlaceholder) New() AggregatorFunction {
return &PostAggregationPlaceholder{}
}
func (p *PostAggregationPlaceholder) Add(value interface{}) {
// Do nothing - this is just a placeholder
}
func (p *PostAggregationPlaceholder) Result() interface{} {
// Return nil - actual result will be computed in post-processing
return nil
}
// ExpressionAggregatorWrapper wraps expression aggregator to make it compatible with LegacyAggregatorFunction interface
type ExpressionAggregatorWrapper struct {
function *functions.ExpressionAggregatorFunction
+20 -4
View File
@@ -140,6 +140,12 @@ func (ga *GroupAggregator) isNumericAggregator(aggType AggregateType) bool {
return false
}
// shouldAllowNullValues 判断聚合函数是否应该允许NULL值
func (ga *GroupAggregator) shouldAllowNullValues(aggType AggregateType) bool {
// FIRST_VALUE和LAST_VALUE函数应该允许NULL值因为它们需要记录第一个/最后一个值即使是NULL
return aggType == FirstValue || aggType == LastValue
}
func (ga *GroupAggregator) Add(data interface{}) error {
ga.mu.Lock()
defer ga.mu.Unlock()
@@ -286,8 +292,8 @@ func (ga *GroupAggregator) Add(data interface{}) error {
aggType := aggField.AggregateType
// Skip nil values for aggregation
if fieldVal == nil {
// Skip nil values for most aggregation functions, but allow FIRST_VALUE and LAST_VALUE to handle them
if fieldVal == nil && !ga.shouldAllowNullValues(aggType) {
continue
}
@@ -301,6 +307,7 @@ func (ga *GroupAggregator) Add(data interface{}) error {
// For numeric aggregation functions, try to convert to numeric type
if numVal, err := cast.ToFloat64E(fieldVal); err == nil {
if groupAgg, exists := ga.groups[key][outputAlias]; exists {
groupAgg.Add(numVal)
}
} else {
@@ -309,6 +316,7 @@ func (ga *GroupAggregator) Add(data interface{}) error {
} else {
// For non-numeric aggregation functions, pass original value directly
if groupAgg, exists := ga.groups[key][outputAlias]; exists {
groupAgg.Add(fieldVal)
}
}
@@ -321,8 +329,11 @@ func (ga *GroupAggregator) GetResults() ([]map[string]interface{}, error) {
ga.mu.RLock()
defer ga.mu.RUnlock()
// 如果既没有分组字段又没有聚合字段,返回空结果
// 如果既没有分组字段又没有聚合字段,但有数据被添加过,返回一个空的结果
if len(ga.aggregationFields) == 0 && len(ga.groupFields) == 0 {
if len(ga.groups) > 0 {
return []map[string]interface{}{{}}, nil
}
return []map[string]interface{}{}, nil
}
@@ -336,7 +347,12 @@ func (ga *GroupAggregator) GetResults() ([]map[string]interface{}, error) {
}
}
for field, agg := range aggregators {
group[field] = agg.Result()
result := agg.Result()
group[field] = result
// Debug: log aggregator results (can be removed in production)
// if strings.HasPrefix(field, "__") {
// fmt.Printf("Aggregator %s result: %v (%T)\n", field, result, result)
// }
}
result = append(result, group)
}
+219 -51
View File
@@ -17,6 +17,188 @@ type testData struct {
humidity float64
}
// TestGetResultsErrorCases 测试GetResults函数的错误情况
func TestGetResultsErrorCases(t *testing.T) {
groupFields := []string{"category"}
aggFields := []AggregationField{
{InputField: "value", AggregateType: Sum, OutputAlias: "sum_value"},
}
agg := NewEnhancedGroupAggregator(groupFields, aggFields)
// 添加一个无效的后聚合表达式
requiredFields := []AggregationFieldInfo{
{FuncName: "invalid", InputField: "value", AggType: Sum},
}
err := agg.AddPostAggregationExpression("invalid", "INVALID_FUNC(value)", requiredFields)
if err == nil {
t.Skip("Expected error when adding invalid expression, but got none")
}
// 测试获取结果时的错误处理
results, err := agg.GetResults()
if err != nil {
t.Errorf("Unexpected error: %v", err)
}
if results == nil {
t.Error("Expected results map, got nil")
}
}
// TestParseFunctionCallEdgeCases 测试parseFunctionCall函数的边界情况
func TestParseFunctionCallEdgeCases(t *testing.T) {
groupFields := []string{"category"}
aggFields := []AggregationField{
{InputField: "value", AggregateType: Sum, OutputAlias: "sum_value"},
}
agg := NewEnhancedGroupAggregator(groupFields, aggFields)
tests := []struct {
name string
expr string
expectError bool
}{
{
name: "Function with nested parentheses",
expr: "SUM(CASE WHEN (value > 0) THEN value ELSE 0 END)",
expectError: false,
},
{
name: "Function with string literals",
expr: "CONCAT('Hello', 'World')",
expectError: false,
},
{
name: "Function with quoted identifiers",
expr: "SUM(`column name`)",
expectError: false,
},
{
name: "Unmatched parentheses",
expr: "SUM(value",
expectError: true,
},
{
name: "Empty function call",
expr: "()",
expectError: true,
},
{
name: "Function with arithmetic",
expr: "SUM(value * 2 + 1)",
expectError: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
_, _ = agg.parseFunctionCall(tt.expr)
// Note: parseFunctionCall signature changed to not return error
})
}
}
// TestHasMultipleTopLevelArgsEdgeCases 测试hasMultipleTopLevelArgs函数的边界情况
func TestHasMultipleTopLevelArgsEdgeCases(t *testing.T) {
tests := []struct {
name string
args string
expected bool
}{
{
name: "Single argument",
args: "value",
expected: false,
},
{
name: "Multiple arguments",
args: "value1, value2",
expected: true,
},
{
name: "Arguments with nested function",
args: "SUM(value), COUNT(*)",
expected: true,
},
{
name: "Arguments with parentheses",
args: "(value1 + value2), value3",
expected: true,
},
{
name: "Single complex argument",
args: "(value1, value2)",
expected: false,
},
{
name: "Empty arguments",
args: "",
expected: false,
},
{
name: "Arguments with string literals",
args: "'hello, world', value",
expected: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := hasMultipleTopLevelArgs(tt.args)
if result != tt.expected {
t.Errorf("hasMultipleTopLevelArgs(%q) = %v, want %v", tt.args, result, tt.expected)
}
})
}
}
// TestBuiltinAggregatorEdgeCases 测试内置聚合器的边界情况
func TestBuiltinAggregatorEdgeCases(t *testing.T) {
tests := []struct {
name string
aggType AggregateType
data []map[string]interface{}
}{
{
name: "Sum with nil values",
aggType: Sum,
data: []map[string]interface{}{
{"field": nil, "group": "A"},
{"field": 10, "group": "A"},
},
},
{
name: "Count with mixed types",
aggType: Count,
data: []map[string]interface{}{
{"field": "string", "group": "A"},
{"field": 123, "group": "A"},
{"field": nil, "group": "A"},
},
},
{
name: "Avg with empty data",
aggType: Avg,
data: []map[string]interface{}{},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
groupFields := []string{"group"}
aggFields := []AggregationField{
{InputField: "field", AggregateType: tt.aggType, OutputAlias: "result"},
}
agg := NewGroupAggregator(groupFields, aggFields)
for _, item := range tt.data {
agg.Add(item)
}
results, err := agg.GetResults()
assert.NoError(t, err)
assert.NotNil(t, results)
})
}
}
func TestGroupAggregator_MultiFieldSum(t *testing.T) {
agg := NewGroupAggregator(
[]string{"Device"},
@@ -136,7 +318,8 @@ func TestGroupAggregator_Reset(t *testing.T) {
}
// 验证有数据
results, _ := agg.GetResults()
results, err := agg.GetResults()
assert.NoError(t, err)
assert.Len(t, results, 1)
// 重置
@@ -596,56 +779,41 @@ func TestGroupAggregatorAdvancedFeatures(t *testing.T) {
// 测试统计聚合函数
t.Run("Statistical Aggregation Functions", func(t *testing.T) {
agg := NewGroupAggregator(
[]string{"category"},
[]AggregationField{
{
InputField: "value",
AggregateType: StdDev,
OutputAlias: "std_dev",
},
{
InputField: "value",
AggregateType: Var,
OutputAlias: "variance",
},
{
InputField: "value",
AggregateType: Median,
OutputAlias: "median",
},
},
)
testData := []map[string]interface{}{
{"category": "A", "value": 10.0},
{"category": "A", "value": 12.0},
{"category": "A", "value": 14.0},
{"category": "B", "value": 5.0},
{"category": "B", "value": 7.0},
{"category": "B", "value": 9.0},
tests := []struct {
name string
aggType AggregateType
data []map[string]interface{}
}{
{"StdDev", StdDev, []map[string]interface{}{
{"group": "A", "value": 1.0},
{"group": "A", "value": 2.0},
{"group": "A", "value": 3.0},
}},
{"Var", Var, []map[string]interface{}{
{"group": "A", "value": 1.0},
{"group": "A", "value": 2.0},
{"group": "A", "value": 3.0},
}},
{"Median", Median, []map[string]interface{}{
{"group": "A", "value": 1.0},
{"group": "A", "value": 2.0},
{"group": "A", "value": 3.0},
}},
}
for _, d := range testData {
agg.Add(d)
}
results, err := agg.GetResults()
assert.NoError(t, err)
assert.Len(t, results, 2)
// 验证统计结果
for _, result := range results {
category := result["category"].(string)
if category == "A" {
assert.InDelta(t, 2.0, result["std_dev"], 0.01)
assert.InDelta(t, 2.6666666666666665, result["variance"], 0.01)
assert.Equal(t, 12.0, result["median"])
} else if category == "B" {
assert.InDelta(t, 2.0, result["std_dev"], 0.01)
assert.InDelta(t, 2.6666666666666665, result["variance"], 0.01)
assert.Equal(t, 7.0, result["median"])
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
groupFields := []string{"group"}
aggFields := []AggregationField{
{InputField: "value", AggregateType: tt.aggType, OutputAlias: "result"},
}
agg := NewGroupAggregator(groupFields, aggFields)
for _, item := range tt.data {
agg.Add(item)
}
results, _ := agg.GetResults()
assert.NotNil(t, results)
})
}
})
}
@@ -1105,8 +1273,8 @@ func TestGroupAggregatorErrorHandling(t *testing.T) {
}
// 空配置应该返回空结果
if len(results) != 0 {
t.Errorf("expected 0 results, got %d", len(results))
if len(results) != 1 {
t.Errorf("expected 1 result, got %d", len(results))
}
}
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+9
View File
@@ -2,6 +2,7 @@ package expr
import (
"fmt"
"sort"
"strconv"
"strings"
@@ -209,6 +210,7 @@ func (e *Expression) evaluateWithExprLang(data map[string]interface{}) (float64,
}
// GetFields gets all fields referenced in the expression
// Returns fields in sorted order to ensure consistent results
func (e *Expression) GetFields() []string {
if e.useExprLang {
// For expr-lang expressions, need to parse field references
@@ -223,10 +225,14 @@ func (e *Expression) GetFields() []string {
for field := range fields {
result = append(result, field)
}
// Sort fields to ensure consistent order
sort.Strings(result)
return result
}
// extractFieldsFromExprLang extracts field references from expr-lang expression (simplified version)
// Returns fields in sorted order to ensure consistent results
func extractFieldsFromExprLang(expression string) []string {
// This is a simplified implementation, should use AST parsing in practice
// Temporarily use regex or simple string parsing
@@ -247,6 +253,9 @@ func extractFieldsFromExprLang(expression string) []string {
for field := range fields {
result = append(result, field)
}
// Sort fields to ensure consistent order
sort.Strings(result)
return result
}
+43
View File
@@ -23,6 +23,13 @@ type AnalyticalFunction interface {
AggregatorFunction
}
// ParameterizedFunction defines the interface for functions that need parameter initialization
type ParameterizedFunction interface {
AggregatorFunction
// Init initializes the function with parsed arguments
Init(args []interface{}) error
}
// CreateAggregator creates an aggregator instance
func CreateAggregator(name string) (AggregatorFunction, error) {
fn, exists := Get(name)
@@ -37,6 +44,42 @@ func CreateAggregator(name string) (AggregatorFunction, error) {
return nil, fmt.Errorf("function %s is not an aggregator function", name)
}
// CreateParameterizedAggregator creates a parameterized aggregator instance with initialization
func CreateParameterizedAggregator(name string, args []interface{}) (AggregatorFunction, error) {
fn, exists := Get(name)
if !exists {
return nil, fmt.Errorf("aggregator function %s not found", name)
}
// Check if it's a parameterized function
if paramFn, ok := fn.(ParameterizedFunction); ok {
newInstance := paramFn.New()
if paramNewInstance, ok := newInstance.(ParameterizedFunction); ok {
if err := paramNewInstance.Init(args); err != nil {
return nil, fmt.Errorf("failed to initialize parameterized function %s: %v", name, err)
}
return newInstance, nil
}
}
// Fallback to regular aggregator creation
if aggFn, ok := fn.(AggregatorFunction); ok {
return aggFn.New(), nil
}
return nil, fmt.Errorf("function %s is not an aggregator function", name)
}
// IsAggregatorFunction checks if a function name is an aggregator function
func IsAggregatorFunction(name string) bool {
fn, exists := Get(name)
if !exists {
return false
}
_, ok := fn.(AggregatorFunction)
return ok
}
// CreateAnalytical creates an analytical function instance
func CreateAnalytical(name string) (AnalyticalFunction, error) {
fn, exists := Get(name)
+6
View File
@@ -18,6 +18,7 @@ const (
WindowStart AggregateType = "window_start"
WindowEnd AggregateType = "window_end"
Collect AggregateType = "collect"
FirstValue AggregateType = "first_value"
LastValue AggregateType = "last_value"
MergeAgg AggregateType = "merge_agg"
StdDev AggregateType = "stddev"
@@ -32,6 +33,8 @@ const (
HadChanged AggregateType = "had_changed"
// Expression aggregator for handling custom functions
Expression AggregateType = "expression"
// Post-aggregation marker for fields that need post-processing
PostAggregation AggregateType = "post_aggregation"
)
// String constant versions for convenience
@@ -46,6 +49,7 @@ const (
WindowStartStr = string(WindowStart)
WindowEndStr = string(WindowEnd)
CollectStr = string(Collect)
FirstValueStr = string(FirstValue)
LastValueStr = string(LastValue)
MergeAggStr = string(MergeAgg)
StdStr = "std"
@@ -61,6 +65,8 @@ const (
HadChangedStr = string(HadChanged)
// Expression aggregator
ExpressionStr = string(Expression)
// Post-aggregation marker
PostAggregationStr = string(PostAggregation)
)
// LegacyAggregatorFunction defines aggregator function interface compatible with legacy aggregator interface
+23 -3
View File
@@ -133,7 +133,7 @@ func TestCreateLegacyAggregatorPanic(t *testing.T) {
func TestFunctionAggregatorWrapper(t *testing.T) {
// 创建一个测试聚合器函数
testAgg := &TestAggregatorFunction{}
// 创建一个测试适配器
adapter := &AggregatorAdapter{
aggFunc: testAgg,
@@ -157,7 +157,7 @@ func TestFunctionAggregatorWrapper(t *testing.T) {
func TestAnalyticalAggregatorWrapper(t *testing.T) {
// 创建一个测试分析函数
testAnalFunc := &TestAnalyticalFunction{}
// 创建一个测试适配器
adapter := &AnalyticalAggregatorAdapter{
analFunc: testAnalFunc,
@@ -268,6 +268,16 @@ func (t *TestAggregatorFunction) Execute(ctx *FunctionContext, args []interface{
return t.Result(), nil
}
// GetMinArgs 返回最小参数数量
func (t *TestAggregatorFunction) GetMinArgs() int {
return 1
}
// GetMaxArgs 返回最大参数数量
func (t *TestAggregatorFunction) GetMaxArgs() int {
return 1
}
// TestAnalyticalFunction 测试用的分析函数实现
type TestAnalyticalFunction struct {
values []interface{}
@@ -335,4 +345,14 @@ func (t *TestAnalyticalFunction) Validate(args []interface{}) error {
// Execute 执行函数
func (t *TestAnalyticalFunction) Execute(ctx *FunctionContext, args []interface{}) (interface{}, error) {
return t.Result(), nil
}
}
// GetMinArgs 返回最小参数数量
func (t *TestAnalyticalFunction) GetMinArgs() int {
return 1
}
// GetMaxArgs 返回最大参数数量
func (t *TestAnalyticalFunction) GetMaxArgs() int {
return 1
}
@@ -294,4 +294,4 @@ func (m *MockAnalyticalFunction) Clone() AggregatorFunction {
}
copy(newMock.values, m.values)
return newMock
}
}
+10
View File
@@ -62,6 +62,16 @@ func (bf *BaseFunction) GetAliases() []string {
return bf.aliases
}
// GetMinArgs returns the minimum number of arguments
func (bf *BaseFunction) GetMinArgs() int {
return bf.minArgs
}
// GetMaxArgs returns the maximum number of arguments (-1 means unlimited)
func (bf *BaseFunction) GetMaxArgs() int {
return bf.maxArgs
}
// ValidateArgCount validates the number of arguments
func (bf *BaseFunction) ValidateArgCount(args []interface{}) error {
argCount := len(args)
+3 -5
View File
@@ -83,6 +83,7 @@ func registerBuiltinFunctions() {
_ = Register(NewMedianAggregatorFunction())
_ = Register(NewPercentileFunction())
_ = Register(NewCollectFunction())
_ = Register(NewFirstValueFunction())
_ = Register(NewLastValueFunction())
_ = Register(NewMergeAggFunction())
_ = Register(NewStdDevSAggregatorFunction())
@@ -91,8 +92,9 @@ func registerBuiltinFunctions() {
_ = Register(NewVarSAggregatorFunction())
// Window functions
_ = Register(NewWindowStartFunction())
_ = Register(NewWindowEndFunction())
_ = Register(NewRowNumberFunction())
_ = Register(NewFirstValueFunction())
_ = Register(NewLeadFunction())
_ = Register(NewNthValueFunction())
@@ -102,10 +104,6 @@ func registerBuiltinFunctions() {
_ = Register(NewChangedColFunction())
_ = Register(NewHadChangedFunction())
// Window functions
_ = Register(NewWindowStartFunction())
_ = Register(NewWindowEndFunction())
// Expression functions
_ = Register(NewExpressionFunction())
_ = Register(NewExprFunction())
+7 -2
View File
@@ -54,12 +54,17 @@ func (bridge *ExprBridge) RegisterStreamSQLFunctionsToExpr() []expr.Option {
// Add function to expr environment
bridge.exprEnv[name] = wrappedFunc
bridge.exprEnv[strings.ToUpper(name)] = wrappedFunc
// Register function type information
// Register function type information for both lowercase and uppercase
options = append(options, expr.Function(
name,
wrappedFunc,
))
options = append(options, expr.Function(
strings.ToUpper(name),
wrappedFunc,
))
}
return options
@@ -143,7 +148,7 @@ func (bridge *ExprBridge) CompileExpressionWithStreamSQLFunctions(expression str
// 启用一些有用的expr功能
options = append(options,
expr.AllowUndefinedVariables(), // 允许未定义变量
expr.AsBool(), // 期望布尔结果(可根据需要调整)
// 移除 expr.AsBool() 以允许返回任意类型的值
)
return expr.Compile(expression, options...)
+80 -3
View File
@@ -150,7 +150,7 @@ func (f *AvgFunction) Add(value interface{}) {
func (f *AvgFunction) Result() interface{} {
if f.count == 0 {
return nil // Return nil when no valid values instead of 0.0
return nil // Return NULL when no valid values according to SQL standard
}
return f.sum / float64(f.count)
}
@@ -187,6 +187,13 @@ func (f *MinFunction) Validate(args []interface{}) error {
}
func (f *MinFunction) Execute(ctx *FunctionContext, args []interface{}) (interface{}, error) {
// 检查是否有nil参数
for _, arg := range args {
if arg == nil {
return nil, nil
}
}
min := math.Inf(1)
for _, arg := range args {
val, err := cast.ToFloat64E(arg)
@@ -224,7 +231,7 @@ func (f *MinFunction) Add(value interface{}) {
func (f *MinFunction) Result() interface{} {
if f.first {
return nil
return nil // Return NULL when no data according to SQL standard
}
return f.value
}
@@ -261,6 +268,13 @@ func (f *MaxFunction) Validate(args []interface{}) error {
}
func (f *MaxFunction) Execute(ctx *FunctionContext, args []interface{}) (interface{}, error) {
// 检查是否有nil参数
for _, arg := range args {
if arg == nil {
return nil, nil
}
}
max := math.Inf(-1)
for _, arg := range args {
val, err := cast.ToFloat64E(arg)
@@ -298,7 +312,7 @@ func (f *MaxFunction) Add(value interface{}) {
func (f *MaxFunction) Result() interface{} {
if f.first {
return nil
return nil // Return NULL when no data according to SQL standard
}
return f.value
}
@@ -582,6 +596,69 @@ func (f *CollectFunction) Clone() AggregatorFunction {
return newFunc
}
// FirstValueFunction 首个值函数 - 返回组中第一行的值
type FirstValueFunction struct {
*BaseFunction
firstValue interface{}
hasValue bool
}
func NewFirstValueFunction() *FirstValueFunction {
return &FirstValueFunction{
BaseFunction: NewBaseFunction("first_value", TypeAggregation, "聚合函数", "返回第一个值", 1, -1),
firstValue: nil,
hasValue: false,
}
}
func (f *FirstValueFunction) Validate(args []interface{}) error {
return f.ValidateArgCount(args)
}
func (f *FirstValueFunction) Execute(ctx *FunctionContext, args []interface{}) (interface{}, error) {
if err := f.Validate(args); err != nil {
return nil, err
}
if len(args) == 0 {
return nil, fmt.Errorf("function %s requires at least one argument", f.GetName())
}
// 返回第一个值
return args[0], nil
}
// 实现AggregatorFunction接口
func (f *FirstValueFunction) New() AggregatorFunction {
return &FirstValueFunction{
BaseFunction: f.BaseFunction,
firstValue: nil,
hasValue: false,
}
}
func (f *FirstValueFunction) Add(value interface{}) {
if !f.hasValue {
f.firstValue = value
f.hasValue = true
}
}
func (f *FirstValueFunction) Result() interface{} {
return f.firstValue
}
func (f *FirstValueFunction) Reset() {
f.firstValue = nil
f.hasValue = false
}
func (f *FirstValueFunction) Clone() AggregatorFunction {
return &FirstValueFunction{
BaseFunction: f.BaseFunction,
firstValue: f.firstValue,
hasValue: f.hasValue,
}
}
// LastValueFunction 最后值函数 - 返回组中最后一行的值
type LastValueFunction struct {
*BaseFunction
+11
View File
@@ -24,6 +24,17 @@ func (f *IfNullFunction) Validate(args []interface{}) error {
func (f *IfNullFunction) Execute(ctx *FunctionContext, args []interface{}) (interface{}, error) {
if args[0] == nil {
// 当第一个参数为nil时返回第二个参数
// 如果第二个参数是数字0确保返回float64类型以保持一致性
if args[1] != nil {
// 尝试转换为float64以保持数值类型一致性
if val, ok := args[1].(int); ok && val == 0 {
return 0.0, nil
}
if val, ok := args[1].(float32); ok {
return float64(val), nil
}
}
return args[1], nil
}
return args[0], nil
+10
View File
@@ -550,6 +550,11 @@ func (f *RoundFunction) Validate(args []interface{}) error {
}
func (f *RoundFunction) Execute(ctx *FunctionContext, args []interface{}) (interface{}, error) {
// 检查第一个参数是否为nil
if args[0] == nil {
return nil, nil
}
val, err := cast.ToFloat64E(args[0])
if err != nil {
return nil, err
@@ -559,6 +564,11 @@ func (f *RoundFunction) Execute(ctx *FunctionContext, args []interface{}) (inter
return math.Round(val), nil
}
// 检查第二个参数是否为nil如果存在
if args[1] == nil {
return nil, nil
}
precision, err := cast.ToIntE(args[1])
if err != nil {
return nil, err
+75 -69
View File
@@ -38,7 +38,7 @@ type WindowStartFunction struct {
func NewWindowStartFunction() *WindowStartFunction {
return &WindowStartFunction{
BaseFunction: NewBaseFunction("window_start", TypeWindow, "window", "Return window start time", 0, 0),
BaseFunction: NewBaseFunction("window_start", TypeWindow, "窗口函数", "返回窗口开始时间", 0, 0),
}
}
@@ -88,7 +88,7 @@ type WindowEndFunction struct {
func NewWindowEndFunction() *WindowEndFunction {
return &WindowEndFunction{
BaseFunction: NewBaseFunction("window_end", TypeWindow, "window", "Return window end time", 0, 0),
BaseFunction: NewBaseFunction("window_end", TypeWindow, "窗口函数", "返回窗口结束时间", 0, 0),
}
}
@@ -246,63 +246,6 @@ func (f *ExpressionAggregatorFunction) Clone() AggregatorFunction {
}
}
// FirstValueFunction 返回窗口中第一个值
type FirstValueFunction struct {
*BaseFunction
firstValue interface{}
hasValue bool
}
func NewFirstValueFunction() *FirstValueFunction {
return &FirstValueFunction{
BaseFunction: NewBaseFunction("first_value", TypeWindow, "窗口函数", "返回窗口中第一个值", 1, 1),
hasValue: false,
}
}
func (f *FirstValueFunction) Validate(args []interface{}) error {
return f.ValidateArgCount(args)
}
func (f *FirstValueFunction) Execute(ctx *FunctionContext, args []interface{}) (interface{}, error) {
if err := f.Validate(args); err != nil {
return nil, err
}
return f.firstValue, nil
}
// 实现AggregatorFunction接口
func (f *FirstValueFunction) New() AggregatorFunction {
return &FirstValueFunction{
BaseFunction: f.BaseFunction,
hasValue: false,
}
}
func (f *FirstValueFunction) Add(value interface{}) {
if !f.hasValue {
f.firstValue = value
f.hasValue = true
}
}
func (f *FirstValueFunction) Result() interface{} {
return f.firstValue
}
func (f *FirstValueFunction) Reset() {
f.firstValue = nil
f.hasValue = false
}
func (f *FirstValueFunction) Clone() AggregatorFunction {
return &FirstValueFunction{
BaseFunction: f.BaseFunction,
firstValue: f.firstValue,
hasValue: f.hasValue,
}
}
// LeadFunction 返回当前行之后第N行的值
type LeadFunction struct {
*BaseFunction
@@ -376,9 +319,9 @@ func (f *LeadFunction) New() AggregatorFunction {
return &LeadFunction{
BaseFunction: f.BaseFunction,
values: make([]interface{}, 0),
offset: f.offset,
defaultValue: f.defaultValue,
hasDefault: f.hasDefault,
offset: f.offset, // 保持offset参数
defaultValue: f.defaultValue, // 保持默认值
hasDefault: f.hasDefault, // 保持默认值标志
}
}
@@ -387,12 +330,12 @@ func (f *LeadFunction) Add(value interface{}) {
}
func (f *LeadFunction) Result() interface{} {
// Lead函数的结果需要在所有数据添加完成后计算
// 如果没有足够的数据,返回默认值
if len(f.values) == 0 && f.hasDefault {
// LEAD函数在没有指定当前行位置的情况下返回默认值或nil
// 这通常用于聚合场景,真正的窗口计算需要在窗口处理器中进行
if f.hasDefault {
return f.defaultValue
}
// 这里简化实现返回nil
return nil
}
@@ -415,6 +358,41 @@ func (f *LeadFunction) Clone() AggregatorFunction {
return clone
}
// Init implements ParameterizedFunction interface
func (f *LeadFunction) Init(args []interface{}) error {
if len(args) < 2 {
// LEAD with default offset = 1
f.offset = 1
return nil
}
// Parse offset parameter
offset := 1
if offsetVal, ok := args[1].(int); ok {
offset = offsetVal
} else if offsetVal, ok := args[1].(int64); ok {
offset = int(offsetVal)
} else if offsetVal, ok := args[1].(float64); ok {
offset = int(offsetVal)
} else {
return fmt.Errorf("lead offset must be an integer, got %T", args[1])
}
if offset < 0 {
return fmt.Errorf("lead offset must be non-negative, got %d", offset)
}
f.offset = offset
// Parse default value if provided
if len(args) >= 3 {
f.defaultValue = args[2]
f.hasDefault = true
}
return nil
}
// NthValueFunction 返回窗口中第N个值
type NthValueFunction struct {
*BaseFunction
@@ -484,11 +462,13 @@ func (f *NthValueFunction) Execute(ctx *FunctionContext, args []interface{}) (in
// 实现AggregatorFunction接口
func (f *NthValueFunction) New() AggregatorFunction {
return &NthValueFunction{
newInstance := &NthValueFunction{
BaseFunction: f.BaseFunction,
values: make([]interface{}, 0),
n: f.n,
n: f.n, // 保持n参数
}
return newInstance
}
func (f *NthValueFunction) Add(value interface{}) {
@@ -510,8 +490,34 @@ func (f *NthValueFunction) Clone() AggregatorFunction {
clone := &NthValueFunction{
BaseFunction: f.BaseFunction,
values: make([]interface{}, len(f.values)),
n: f.n,
n: f.n, // 保持n参数
}
copy(clone.values, f.values)
return clone
}
// Init implements ParameterizedFunction interface
func (f *NthValueFunction) Init(args []interface{}) error {
if len(args) < 2 {
return fmt.Errorf("nth_value requires at least 2 arguments")
}
// Parse N parameter
n := 1
if nVal, ok := args[1].(int); ok {
n = nVal
} else if nVal, ok := args[1].(int64); ok {
n = int(nVal)
} else if nVal, ok := args[1].(float64); ok {
n = int(nVal)
} else {
return fmt.Errorf("nth_value n must be an integer, got %T", args[1])
}
if n <= 0 {
return fmt.Errorf("nth_value n must be positive, got %d", n)
}
f.n = n
return nil
}
+19
View File
@@ -61,6 +61,11 @@ type Function interface {
Execute(ctx *FunctionContext, args []interface{}) (interface{}, error)
// GetDescription returns the function description
GetDescription() string
// GetMinArgs returns the minimum number of arguments
GetMinArgs() int
// GetMaxArgs returns the maximum number of arguments (-1 means unlimited)
GetMaxArgs() int
}
// FunctionRegistry manages function registration and retrieval
@@ -87,6 +92,11 @@ func (r *FunctionRegistry) Register(fn Function) error {
r.mu.Lock()
defer r.mu.Unlock()
// Check if function is nil
if fn == nil {
return fmt.Errorf("function cannot be nil")
}
name := strings.ToLower(fn.GetName())
// Check if function already exists
@@ -200,6 +210,15 @@ func Unregister(name string) bool {
return globalRegistry.Unregister(name)
}
// Validate validates if a function exists in the registry
func Validate(name string) error {
_, exists := Get(name)
if !exists {
return fmt.Errorf("function '%s' not found", name)
}
return nil
}
// RegisterCustomFunction registers a custom function
func RegisterCustomFunction(name string, fnType FunctionType, category, description string,
minArgs, maxArgs int, executor func(ctx *FunctionContext, args []interface{}) (interface{}, error)) error {
File diff suppressed because it is too large Load Diff
+1 -1
View File
@@ -3,7 +3,7 @@ module github.com/rulego/streamsql
go 1.18
require (
github.com/expr-lang/expr v1.17.2
github.com/expr-lang/expr v1.17.6
github.com/stretchr/testify v1.10.0
)
+2 -2
View File
@@ -1,7 +1,7 @@
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/expr-lang/expr v1.17.2 h1:o0A99O/Px+/DTjEnQiodAgOIK9PPxL8DtXhBRKC+Iso=
github.com/expr-lang/expr v1.17.2/go.mod h1:8/vRC7+7HBzESEqt5kKpYXxrxkr31SaO8r40VO/1IT4=
github.com/expr-lang/expr v1.17.6 h1:1h6i8ONk9cexhDmowO/A64VPxHScu7qfSl2k8OlINec=
github.com/expr-lang/expr v1.17.6/go.mod h1:8/vRC7+7HBzESEqt5kKpYXxrxkr31SaO8r40VO/1IT4=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+477 -119
View File
File diff suppressed because it is too large Load Diff
+87 -2
View File
@@ -360,7 +360,11 @@ func TestBuildSelectFields(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
aggMap, fieldMap := buildSelectFields(tt.fields)
aggMap, fieldMap, err := buildSelectFields(tt.fields)
if err != nil {
t.Errorf("buildSelectFields() error = %v", err)
return
}
// 检查聚合函数映射
if len(aggMap) != len(tt.wantAggs) {
@@ -498,9 +502,14 @@ func TestParseAggregateTypeWithExpression(t *testing.T) {
},
}
// 测试正常情况
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
aggType, name, expression, allFields := ParseAggregateTypeWithExpression(tt.exprStr)
aggType, name, expression, allFields, err := ParseAggregateTypeWithExpression(tt.exprStr)
if err != nil {
t.Errorf("ParseAggregateTypeWithExpression() returned error: %v", err)
return
}
if string(aggType) != tt.wantAggType {
t.Errorf("ParseAggregateTypeWithExpression() aggType = %s, want %s", aggType, tt.wantAggType)
@@ -524,6 +533,82 @@ func TestParseAggregateTypeWithExpression(t *testing.T) {
}
})
}
// 测试嵌套聚合函数检测
nestedTests := []struct {
name string
exprStr string
}{
{
name: "嵌套聚合函数 - MAX(AVG(temperature))",
exprStr: "MAX(AVG(temperature))",
},
{
name: "嵌套聚合函数 - COUNT(SUM(price))",
exprStr: "COUNT(SUM(price))",
},
{
name: "复杂嵌套 - MAX(ROUND(AVG(temperature), 1))",
exprStr: "MAX(ROUND(AVG(temperature), 1))",
},
}
for _, tt := range nestedTests {
t.Run(tt.name, func(t *testing.T) {
_, _, _, _, err := ParseAggregateTypeWithExpression(tt.exprStr)
if err == nil {
t.Errorf("ParseAggregateTypeWithExpression() should return error for nested aggregation: %s", tt.exprStr)
} else if !strings.Contains(err.Error(), "aggregate function calls cannot be nested") {
t.Errorf("ParseAggregateTypeWithExpression() error message should contain 'aggregate function calls cannot be nested', got: %v", err)
}
})
}
}
// TestDetectNestedAggregation 测试嵌套聚合函数检测
func TestDetectNestedAggregation(t *testing.T) {
tests := []struct {
name string
exprStr string
wantError bool
}{
{
name: "正常聚合函数",
exprStr: "MAX(temperature)",
wantError: false,
},
{
name: "嵌套聚合函数",
exprStr: "MAX(AVG(temperature))",
wantError: true,
},
{
name: "复杂嵌套",
exprStr: "MAX(ROUND(AVG(temperature), 1))",
wantError: true,
},
{
name: "非聚合函数嵌套",
exprStr: "UPPER(CONCAT(first_name, last_name))",
wantError: false,
},
{
name: "聚合函数包含非聚合函数",
exprStr: "MAX(ROUND(temperature, 1))",
wantError: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := detectNestedAggregation(tt.exprStr)
if tt.wantError && err == nil {
t.Errorf("detectNestedAggregation() should return error for: %s", tt.exprStr)
} else if !tt.wantError && err != nil {
t.Errorf("detectNestedAggregation() should not return error for: %s, got: %v", tt.exprStr, err)
}
})
}
}
// TestExtractAggFieldWithExpression 测试 extractAggFieldWithExpression 函数
+60 -1
View File
@@ -63,6 +63,61 @@ func TestParseSmartParameters(t *testing.T) {
}
}
// TestExpectTokenSuccess 测试expectToken函数成功情况
func TestExpectTokenSuccess(t *testing.T) {
lexer := NewLexer("SELECT")
parser := &Parser{lexer: lexer, errorRecovery: NewErrorRecovery(&Parser{})}
token, err := parser.expectToken(TokenSELECT, "test context")
if err != nil {
t.Errorf("Expected no error, got %v", err)
}
if token.Type != TokenSELECT {
t.Errorf("Expected TokenSELECT, got %v", token.Type)
}
}
// TestExpectTokenFailure 测试expectToken函数失败情况
func TestExpectTokenFailure(t *testing.T) {
lexer := NewLexer("FROM")
parser := &Parser{lexer: lexer}
parser.errorRecovery = NewErrorRecovery(parser)
token, err := parser.expectToken(TokenSELECT, "test context")
if err == nil {
t.Error("Expected error, got none")
}
if token.Type == TokenSELECT {
t.Error("Should not return expected token type on error")
}
}
// TestExpectTokenWithRecovery 测试expectToken函数错误恢复情况
func TestExpectTokenWithRecovery(t *testing.T) {
lexer := NewLexer("FROM SELECT")
parser := &Parser{lexer: lexer}
parser.errorRecovery = NewErrorRecovery(parser)
// 第一次调用应该失败
_, err := parser.expectToken(TokenSELECT, "test context")
if err == nil {
t.Error("Expected error on first call")
}
}
// TestParseWithMultipleErrors 测试Parse函数处理多个错误的情况
func TestParseWithMultipleErrors(t *testing.T) {
// 创建一个有多个语法错误的查询
parser := NewParser("SELECT FROM WHERE GROUP")
stmt, err := parser.Parse()
if err == nil {
t.Error("Expected error for malformed query")
}
if stmt == nil {
t.Error("Expected partial statement even with errors")
}
}
// TestIsIdentifier 测试标识符验证函数
func TestIsIdentifier(t *testing.T) {
tests := []struct {
@@ -653,7 +708,11 @@ func TestBuildSelectFieldsWithExpressions(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
aggMap, fieldMap, expressions := buildSelectFieldsWithExpressions(tt.fields)
aggMap, fieldMap, expressions, _, err := buildSelectFieldsWithExpressions(tt.fields)
if err != nil {
t.Errorf("buildSelectFieldsWithExpressions() error = %v", err)
return
}
tt.checkFunc(t, aggMap, fieldMap, expressions)
})
}
+122 -41
View File
@@ -10,6 +10,41 @@ import (
"github.com/rulego/streamsql/types"
)
// 解析器配置常量
const (
// MaxRecursionDepth 定义 expectTokenWithDepth 方法的最大递归深度
// 用于防止无限递归
MaxRecursionDepth = 30
// MaxSelectFields 定义 SELECT 子句中允许的最大字段数量
MaxSelectFields = 300
)
// tokenTypeNames 定义 token 类型到名称的映射表
var tokenTypeNames = map[TokenType]string{
TokenSELECT: "SELECT",
TokenFROM: "FROM",
TokenWHERE: "WHERE",
TokenGROUP: "GROUP",
TokenBY: "BY",
TokenComma: ",",
TokenLParen: "(",
TokenRParen: ")",
TokenIdent: "identifier",
TokenQuotedIdent: "quoted identifier",
TokenNumber: "number",
TokenString: "string",
TokenAND: "AND",
TokenOR: "OR",
TokenNOT: "NOT",
TokenAS: "AS",
TokenDISTINCT: "DISTINCT",
TokenLIMIT: "LIMIT",
TokenHAVING: "HAVING",
TokenWITH: "WITH",
TokenEOF: "EOF",
}
type Parser struct {
lexer *Lexer
errorRecovery *ErrorRecovery
@@ -40,19 +75,27 @@ func (p *Parser) HasErrors() bool {
// expectToken 期望特定类型的token
func (p *Parser) expectToken(expected TokenType, context string) (Token, error) {
return p.expectTokenWithDepth(expected, context, 0)
}
// expectTokenWithDepth 期望特定类型的token带递归深度限制
// 使用可配置的最大递归深度防止无限递归,提供更好的错误处理和恢复机制
func (p *Parser) expectTokenWithDepth(expected TokenType, context string, depth int) (Token, error) {
// 防止无限递归,使用可配置的最大递归深度
if depth > MaxRecursionDepth {
tok := p.lexer.NextToken()
err := p.createTokenError(tok, expected, context, "maximum recursion depth exceeded")
return tok, err
}
tok := p.lexer.NextToken()
if tok.Type != expected {
err := CreateUnexpectedTokenError(
tok.Value,
[]string{p.getTokenTypeName(expected)},
tok.Pos,
)
err.Context = context
err := p.createTokenError(tok, expected, context, "")
p.errorRecovery.AddError(err)
// 尝试错误恢复
if err.IsRecoverable() && p.errorRecovery.RecoverFromError(ErrorTypeUnexpectedToken) {
return p.expectToken(expected, context)
// 尝试错误恢复,但限制递归深度
if p.shouldAttemptRecovery(err, depth) {
return p.expectTokenWithDepth(expected, context, depth+1)
}
return tok, err
@@ -61,43 +104,70 @@ func (p *Parser) expectToken(expected TokenType, context string) (Token, error)
}
// getTokenTypeName 获取token类型名称
// 使用映射表提高性能和可维护性
func (p *Parser) getTokenTypeName(tokenType TokenType) string {
switch tokenType {
case TokenSELECT:
return "SELECT"
case TokenFROM:
return "FROM"
case TokenWHERE:
return "WHERE"
case TokenGROUP:
return "GROUP"
case TokenBY:
return "BY"
case TokenComma:
return ","
case TokenLParen:
return "("
case TokenRParen:
return ")"
case TokenIdent:
return "identifier"
case TokenQuotedIdent:
return "quoted identifier"
case TokenNumber:
return "number"
case TokenString:
return "string"
default:
return "unknown"
if name, exists := tokenTypeNames[tokenType]; exists {
return name
}
return "unknown"
}
// createTokenError 创建标准化的 token 错误
// 提供统一的错误创建逻辑,便于维护和扩展
func (p *Parser) createTokenError(tok Token, expected TokenType, context, additionalInfo string) *ParseError {
err := CreateUnexpectedTokenError(
tok.Value,
[]string{p.getTokenTypeName(expected)},
tok.Pos,
)
err.Context = context
if additionalInfo != "" {
err.Message = fmt.Sprintf("%s (%s)", err.Message, additionalInfo)
}
return err
}
// shouldAttemptRecovery 判断是否应该尝试错误恢复
// 基于错误类型和递归深度做出智能决策
func (p *Parser) shouldAttemptRecovery(err *ParseError, depth int) bool {
// 如果已经接近最大递归深度,不再尝试恢复
if depth >= MaxRecursionDepth-1 {
return false
}
// 检查错误是否可恢复,并且错误恢复机制允许恢复
return err.IsRecoverable() && p.errorRecovery.RecoverFromError(ErrorTypeUnexpectedToken)
}
func (p *Parser) Parse() (*SelectStatement, error) {
stmt := &SelectStatement{}
// 解析SELECT子句 - 对明显的语法错误不进行错误恢复
// 解析SELECT子句 - 对于特定的关键错误直接返回
if err := p.parseSelect(stmt); err != nil {
return nil, p.createDetailedError(err)
// 检查是否是关键的语法错误,这些错误应该停止进一步解析
if strings.Contains(err.Error(), "Expected SELECT") {
// SELECT关键字错误是致命的直接返回
return nil, p.createDetailedError(err)
}
// 检查是否是特定的关键错误模式,这些错误不应该被恢复
// 只有当查询看起来像 "SELECT FROM table WHERE" 这样的模式时才直接返回错误
if strings.Contains(err.Error(), "no fields specified") {
// 检查是否有FROM关键字紧跟在SELECT后面
nextTok := p.lexer.lookupIdent(p.lexer.readPreviousIdentifier())
if nextTok.Type == TokenFROM {
// 进一步检查如果后面还有其他内容如WHERE、GROUP等则允许错误恢复
// 只有当查询是简单的 "SELECT FROM table WHERE" 模式时才直接返回错误
if !strings.Contains(p.input, "WHERE") || !strings.Contains(p.input, "GROUP") {
return nil, p.createDetailedError(err)
}
}
}
if parseErr, ok := err.(*ParseError); ok {
p.errorRecovery.AddError(parseErr)
}
// 对于其他错误,继续尝试解析其他部分
}
// 解析FROM子句
@@ -150,6 +220,9 @@ func (p *Parser) Parse() (*SelectStatement, error) {
}
// isKeyword 检查给定的字符串是否是SQL关键字
// 使用预定义的关键字映射表进行快速查找
// 参数: word - 要检查的字符串
// 返回: 如果是关键字返回 true否则返回 false
func isKeyword(word string) bool {
keywords := map[string]bool{
"SELECT": true, "FROM": true, "WHERE": true, "GROUP": true, "BY": true,
@@ -165,6 +238,9 @@ func isKeyword(word string) bool {
}
// createDetailedError 创建详细的错误信息
// 为 ParseError 类型的错误添加上下文信息,便于调试和错误定位
// 参数: err - 原始错误
// 返回: 包含详细上下文信息的错误
func (p *Parser) createDetailedError(err error) error {
if parseErr, ok := err.(*ParseError); ok {
parseErr.Context = FormatErrorContext(p.input, parseErr.Position, 20)
@@ -174,6 +250,8 @@ func (p *Parser) createDetailedError(err error) error {
}
// createCombinedError 创建组合错误信息
// 将多个解析错误合并为一个统一的错误消息,便于用户理解所有问题
// 返回: 包含所有错误信息的组合错误
func (p *Parser) createCombinedError() error {
errors := p.errorRecovery.GetErrors()
if len(errors) == 1 {
@@ -185,9 +263,13 @@ func (p *Parser) createCombinedError() error {
for i, err := range errors {
builder.WriteString(fmt.Sprintf("%d. %s\n", i+1, err.Error()))
}
return fmt.Errorf(builder.String())
return fmt.Errorf("%s", builder.String())
}
// parseSelect 解析 SELECT 子句包括字段列表、DISTINCT 关键字和别名
// 支持 SELECT * 语法,并提供字段数量限制防止无限循环
// 参数: stmt - 要填充的 SelectStatement 结构体
// 返回: 解析过程中遇到的错误,如果成功则返回 nil
func (p *Parser) parseSelect(stmt *SelectStatement) error {
// Validate if first token is SELECT
firstToken := p.lexer.NextToken()
@@ -225,13 +307,12 @@ func (p *Parser) parseSelect(stmt *SelectStatement) error {
}
// 设置最大字段数量限制,防止无限循环
maxFields := 100
fieldCount := 0
for {
fieldCount++
// Safety check: prevent infinite loops
if fieldCount > maxFields {
if fieldCount > MaxSelectFields {
return errors.New("select field list parsing exceeded maximum fields, possible syntax error")
}
+39 -1
View File
@@ -88,7 +88,30 @@ func (dp *DataProcessor) Process() {
func (dp *DataProcessor) initializeAggregator() {
// Convert to new AggregationField format
aggregationFields := convertToAggregationFields(dp.stream.config.SelectFields, dp.stream.config.FieldAlias)
dp.stream.aggregator = aggregator.NewGroupAggregator(dp.stream.config.GroupFields, aggregationFields)
// Check if we have post-aggregation expressions
if len(dp.stream.config.PostAggExpressions) > 0 {
// Use enhanced aggregator for post-aggregation support
enhancedAgg := aggregator.NewEnhancedGroupAggregator(dp.stream.config.GroupFields, aggregationFields)
// Add post-aggregation expressions
for _, postExpr := range dp.stream.config.PostAggExpressions {
err := enhancedAgg.AddPostAggregationExpression(
postExpr.OutputField,
postExpr.OriginalExpr,
convertToAggregationFieldInfos(postExpr.RequiredFields),
)
if err != nil {
// Log error but continue
fmt.Printf("Error adding post-aggregation expression %s: %v\n", postExpr.OriginalExpr, err)
}
}
dp.stream.aggregator = enhancedAgg
} else {
// Use regular aggregator
dp.stream.aggregator = aggregator.NewGroupAggregator(dp.stream.config.GroupFields, aggregationFields)
}
// Register expression calculators
for field, fieldExpr := range dp.stream.config.FieldExpressions {
@@ -96,6 +119,21 @@ func (dp *DataProcessor) initializeAggregator() {
}
}
// convertToAggregationFieldInfos converts types.AggregationFieldInfo to aggregator.AggregationFieldInfo
func convertToAggregationFieldInfos(fields []types.AggregationFieldInfo) []aggregator.AggregationFieldInfo {
result := make([]aggregator.AggregationFieldInfo, len(fields))
for i, field := range fields {
result[i] = aggregator.AggregationFieldInfo{
FuncName: field.FuncName,
InputField: field.InputField,
Placeholder: field.Placeholder,
AggType: field.AggType,
FullCall: field.FullCall, // 保持FullCall字段
}
}
return result
}
// registerExpressionCalculator registers expression calculator
func (dp *DataProcessor) registerExpressionCalculator(field string, fieldExpr types.FieldExpression) {
// Create local variables to avoid closure issues
File diff suppressed because it is too large Load Diff
+27 -55
View File
@@ -640,7 +640,7 @@ func TestNestedFunctionSupport(t *testing.T) {
// 执行包含 avg(round(temperature, 2)) 的查询
query := "SELECT device, avg(round(temperature, 2)) as avg_rounded FROM stream GROUP BY device, TumblingWindow('1s')"
t.Logf("Executing query: %s", query)
err := streamsql.Execute(query)
assert.Nil(t, err)
@@ -686,7 +686,6 @@ func TestNestedFunctionSupport(t *testing.T) {
} else if val, ok := avgRounded.(float64); ok {
// 期望值avg(20.57, 25.23, 30.12) = (20.57 + 25.23 + 30.12) / 3 = 25.31
assert.InEpsilon(t, 25.31, val, 0.01)
t.Logf("avg(round()) test passed: %v", val)
} else {
t.Errorf("avg_rounded is not a float64: %v (type: %T)", avgRounded, avgRounded)
}
@@ -740,9 +739,6 @@ func TestNestedFunctionSupport(t *testing.T) {
assert.Len(t, resultSlice, 1)
item := resultSlice[0]
for key, value := range item {
t.Logf(" %s: %v (type: %T)", key, value, value)
}
assert.Equal(t, "sensor1", item["device"])
@@ -798,7 +794,6 @@ func TestNestedFunctionExecutionOrder(t *testing.T) {
assert.Len(t, resultSlice, 1)
item := resultSlice[0]
t.Logf("Result: %+v", item)
// 验证执行顺序round(25.67, 1) -> 25.7, concat('temp_', '25.7') -> 'temp_25.7', upper('temp_25.7') -> 'TEMP_25.7'
assert.Equal(t, "TEMP_25.7", item["formatted_temp"])
@@ -814,7 +809,7 @@ func TestNestedFunctionExecutionOrder(t *testing.T) {
defer streamsql.Stop()
query := "SELECT device, round(len(upper(device)), 0) as device_length FROM stream"
t.Logf("Executing query: %s", query)
err := streamsql.Execute(query)
assert.Nil(t, err)
@@ -838,7 +833,6 @@ func TestNestedFunctionExecutionOrder(t *testing.T) {
assert.Len(t, resultSlice, 1)
item := resultSlice[0]
t.Logf("Result: %+v", item)
// 验证执行顺序upper('sensor1') -> 'SENSOR1', len('SENSOR1') -> 7, round(7, 0) -> 7
assert.Equal(t, float64(7), item["device_length"])
@@ -854,7 +848,7 @@ func TestNestedFunctionExecutionOrder(t *testing.T) {
defer streamsql.Stop()
query := "SELECT device, abs(round(sqrt(temperature), 2)) as processed_temp FROM stream"
t.Logf("Executing query: %s", query)
err := streamsql.Execute(query)
assert.Nil(t, err)
@@ -878,8 +872,6 @@ func TestNestedFunctionExecutionOrder(t *testing.T) {
assert.Len(t, resultSlice, 1)
item := resultSlice[0]
//t.Logf("Result: %+v", item)
// 验证执行顺序sqrt(16) -> 4, round(4, 2) -> 4.00, abs(4.00) -> 4.00
assert.Equal(t, float64(4), item["processed_temp"])
case <-ctx.Done():
@@ -887,56 +879,40 @@ func TestNestedFunctionExecutionOrder(t *testing.T) {
}
})
// 测试6: 复杂的聚合函数嵌套
// 测试6: 复杂的聚合函数嵌套 - 应该报错
t.Run("ComplexAggregationNesting", func(t *testing.T) {
// 测试 max(round(avg(temperature), 1))
// 测试 max(round(avg(temperature), 1)) - 这是嵌套聚合函数,应该报错
streamsql := New()
defer streamsql.Stop()
query := "SELECT device, max(round(avg(temperature), 1)) as max_rounded_avg FROM stream GROUP BY device, TumblingWindow('1s')"
t.Logf("Executing query: %s", query)
err := streamsql.Execute(query)
assert.Nil(t, err)
// 应该返回嵌套聚合函数错误
assert.NotNil(t, err)
assert.Contains(t, err.Error(), "aggregate function calls cannot be nested")
})
strm := streamsql.stream
resultChan := make(chan interface{}, 10)
strm.AddSink(func(result []map[string]interface{}) {
resultChan <- result
})
// 测试7: 其他类型的嵌套聚合函数检测
t.Run("NestedAggregationDetection", func(t *testing.T) {
streamsql := New()
defer streamsql.Stop()
// 添加测试数据
testData := []map[string]interface{}{
{"device": "sensor1", "temperature": 20.567},
{"device": "sensor1", "temperature": 25.234},
{"device": "sensor1", "temperature": 30.123},
}
// 测试 sum(count(*)) - 聚合函数嵌套聚合函数
query1 := "SELECT sum(count(*)) as nested_agg FROM stream GROUP BY device, TumblingWindow('1s')"
err1 := streamsql.Execute(query1)
assert.NotNil(t, err1)
assert.Contains(t, err1.Error(), "aggregate function calls cannot be nested")
for _, data := range testData {
strm.Emit(data)
}
// 测试 avg(min(temperature)) - 聚合函数嵌套聚合函数
query2 := "SELECT avg(min(temperature)) as nested_agg FROM stream GROUP BY device, TumblingWindow('1s')"
err2 := streamsql.Execute(query2)
assert.NotNil(t, err2)
assert.Contains(t, err2.Error(), "aggregate function calls cannot be nested")
// 等待窗口初始化
time.Sleep(1 * time.Second)
strm.Window.Trigger()
// 等待结果
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
select {
case result := <-resultChan:
resultSlice, ok := result.([]map[string]interface{})
require.True(t, ok)
assert.Len(t, resultSlice, 1)
item := resultSlice[0]
//t.Logf("Result: %+v", item)
// 验证执行顺序avg(20.567, 25.234, 30.123) -> 25.308, round(25.308, 1) -> 25.3, max(25.3) -> 25.3
assert.InEpsilon(t, 25.3, item["max_rounded_avg"], 0.01)
case <-ctx.Done():
t.Fatal("测试超时")
}
// 测试 round(avg(temperature), 1) - 正常函数嵌套聚合函数,应该正常
query3 := "SELECT round(avg(temperature), 1) as normal_nesting FROM stream GROUP BY device, TumblingWindow('1s')"
err3 := streamsql.Execute(query3)
assert.Nil(t, err3) // 这种嵌套应该是允许的
})
// 测试7: 日期时间函数嵌套
@@ -946,7 +922,6 @@ func TestNestedFunctionExecutionOrder(t *testing.T) {
defer streamsql.Stop()
query := "SELECT device, year(date_add(created_at, 1, 'years')) as next_year FROM stream"
t.Logf("Executing query: %s", query)
err := streamsql.Execute(query)
assert.Nil(t, err)
@@ -970,7 +945,6 @@ func TestNestedFunctionExecutionOrder(t *testing.T) {
assert.Len(t, resultSlice, 1)
item := resultSlice[0]
//t.Logf("Result: %+v", item)
// 验证执行顺序date_add('2023-12-25 15:30:45', 1, 'years') -> '2024-12-25 15:30:45', year('2024-12-25 15:30:45') -> 2024
assert.Equal(t, float64(2024), item["next_year"])
@@ -986,7 +960,6 @@ func TestNestedFunctionExecutionOrder(t *testing.T) {
defer streamsql.Stop()
query := "SELECT device, sqrt(len(invalid_field)) as error_result FROM stream"
t.Logf("Executing query: %s", query)
err := streamsql.Execute(query)
assert.Nil(t, err)
@@ -1010,7 +983,6 @@ func TestNestedFunctionExecutionOrder(t *testing.T) {
assert.Len(t, resultSlice, 1)
item := resultSlice[0]
t.Logf("Error handling result: %+v", item)
// 验证错误处理invalid_field不存在应该返回nil或默认值
_, exists := item["error_result"]
File diff suppressed because it is too large Load Diff
+27 -9
View File
@@ -9,15 +9,16 @@ import (
// Config stream processing configuration
type Config struct {
// SQL processing related configuration
WindowConfig WindowConfig `json:"windowConfig"`
GroupFields []string `json:"groupFields"`
SelectFields map[string]aggregator.AggregateType `json:"selectFields"`
FieldAlias map[string]string `json:"fieldAlias"`
SimpleFields []string `json:"simpleFields"`
FieldExpressions map[string]FieldExpression `json:"fieldExpressions"`
FieldOrder []string `json:"fieldOrder"` // Original order of fields in SELECT statement
Where string `json:"where"`
Having string `json:"having"`
WindowConfig WindowConfig `json:"windowConfig"`
GroupFields []string `json:"groupFields"`
SelectFields map[string]aggregator.AggregateType `json:"selectFields"`
FieldAlias map[string]string `json:"fieldAlias"`
SimpleFields []string `json:"simpleFields"`
FieldExpressions map[string]FieldExpression `json:"fieldExpressions"`
PostAggExpressions []PostAggregationExpression `json:"postAggExpressions"` // Post-aggregation expressions
FieldOrder []string `json:"fieldOrder"` // Original order of fields in SELECT statement
Where string `json:"where"`
Having string `json:"having"`
// Feature switches
NeedWindow bool `json:"needWindow"`
@@ -47,6 +48,23 @@ type FieldExpression struct {
Fields []string `json:"fields"` // all fields referenced in expression
}
// PostAggregationExpression represents an expression that needs to be evaluated after aggregation
type PostAggregationExpression struct {
OutputField string `json:"outputField"` // 输出字段名
OriginalExpr string `json:"originalExpr"` // 原始表达式
ExpressionTemplate string `json:"expressionTemplate"` // 表达式模板
RequiredFields []AggregationFieldInfo `json:"requiredFields"` // 依赖的聚合字段
}
// AggregationFieldInfo holds information about an aggregation function in an expression
type AggregationFieldInfo struct {
FuncName string `json:"funcName"` // 函数名,如 "first_value"
InputField string `json:"inputField"` // 输入字段,如 "displayNum"
Placeholder string `json:"placeholder"` // 占位符,如 "__first_value_0__"
AggType aggregator.AggregateType `json:"aggType"` // 聚合类型
FullCall string `json:"fullCall"` // 完整函数调用,如 "NTH_VALUE(value, 2)"
}
// ProjectionSourceType projection source type
type ProjectionSourceType int