From af0c5c6ca889a9f4d626342f60b97f00d182376d Mon Sep 17 00:00:00 2001 From: rulego-team Date: Fri, 19 Jan 2024 18:09:21 +0800 Subject: [PATCH] init --- .gitignore | 48 +++ builtin/builtin.go | 111 +++++++ builtin/function.go | 38 +++ collector/aggregate_collector.go | 26 ++ collector/collector.go | 263 +++++++++++++++++ collector/collector_test.go | 41 +++ conf/config.go | 17 ++ dataset/key.go | 134 +++++++++ dataset/kv.go | 86 ++++++ dataset/row.go | 99 +++++++ dataset/value.go | 269 +++++++++++++++++ go.mod | 9 + go.sum | 10 + operator/aggregate_op.go | 89 ++++++ operator/alias_op.go | 17 ++ operator/filter_op.go | 42 +++ operator/function_op.go | 21 ++ operator/group_by_op.go | 77 +++++ operator/having_op.go | 17 ++ operator/limit_op.go | 35 +++ operator/lookup_field_op.go | 50 ++++ operator/operator.go | 37 +++ operator/order_op.go | 35 +++ operator/table_op.go | 35 +++ operator/window_op.go | 51 ++++ planner/filter_plan.go | 54 ++++ planner/group_by_plan.go | 56 ++++ planner/limit_plan.go | 48 +++ planner/look_field_plan.go | 93 ++++++ planner/optimizer.go | 26 ++ planner/order_by_plan.go | 48 +++ planner/plan_test.go | 46 +++ planner/planer.go | 102 +++++++ planner/select_statement_plan.go | 64 ++++ planner/table_plan.go | 48 +++ rsql/ast.go | 412 ++++++++++++++++++++++++++ rsql/lexer.go | 290 ++++++++++++++++++ rsql/parser.go | 489 +++++++++++++++++++++++++++++++ rsql/parser_test.go | 35 +++ rsql/token.go | 161 ++++++++++ streamsql.go | 24 ++ types/errors.go | 24 ++ types/group_fields.go | 50 ++++ types/msg.go | 167 +++++++++++ types/types.go | 122 ++++++++ utils/attribute/attribute.go | 109 +++++++ utils/cast/cast.go | 63 ++++ utils/compress/compress.go | 72 +++++ utils/compress/rawhelpers.go | 41 +++ utils/expr/expr_test.go | 85 ++++++ utils/queue/queue.go | 196 +++++++++++++ window/count_window.go | 148 ++++++++++ window/session_window.go | 160 ++++++++++ window/sliding_window.go | 175 +++++++++++ window/tumbling_window.go | 166 +++++++++++ window/window.go | 125 ++++++++ 56 files changed, 5356 insertions(+) create mode 100644 .gitignore create mode 100644 builtin/builtin.go create mode 100644 builtin/function.go create mode 100644 collector/aggregate_collector.go create mode 100644 collector/collector.go create mode 100644 collector/collector_test.go create mode 100644 conf/config.go create mode 100644 dataset/key.go create mode 100644 dataset/kv.go create mode 100644 dataset/row.go create mode 100644 dataset/value.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 operator/aggregate_op.go create mode 100644 operator/alias_op.go create mode 100644 operator/filter_op.go create mode 100644 operator/function_op.go create mode 100644 operator/group_by_op.go create mode 100644 operator/having_op.go create mode 100644 operator/limit_op.go create mode 100644 operator/lookup_field_op.go create mode 100644 operator/operator.go create mode 100644 operator/order_op.go create mode 100644 operator/table_op.go create mode 100644 operator/window_op.go create mode 100644 planner/filter_plan.go create mode 100644 planner/group_by_plan.go create mode 100644 planner/limit_plan.go create mode 100644 planner/look_field_plan.go create mode 100644 planner/optimizer.go create mode 100644 planner/order_by_plan.go create mode 100644 planner/plan_test.go create mode 100644 planner/planer.go create mode 100644 planner/select_statement_plan.go create mode 100644 planner/table_plan.go create mode 100644 rsql/ast.go create mode 100644 rsql/lexer.go create mode 100644 rsql/parser.go create mode 100644 rsql/parser_test.go create mode 100644 rsql/token.go create mode 100644 streamsql.go create mode 100644 types/errors.go create mode 100644 types/group_fields.go create mode 100644 types/msg.go create mode 100644 types/types.go create mode 100644 utils/attribute/attribute.go create mode 100644 utils/cast/cast.go create mode 100644 utils/compress/compress.go create mode 100644 utils/compress/rawhelpers.go create mode 100644 utils/expr/expr_test.go create mode 100644 utils/queue/queue.go create mode 100644 window/count_window.go create mode 100644 window/session_window.go create mode 100644 window/sliding_window.go create mode 100644 window/tumbling_window.go create mode 100644 window/window.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..fe97e90 --- /dev/null +++ b/.gitignore @@ -0,0 +1,48 @@ +# JetBrains template +.idea +*.iml +out/ + +# File-based project format +*.iws + +# JIRA plugin +atlassian-ide-plugin.xml + +### Go template +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Dependency directories (remove the comment below to include it) +vendor/ + +### macOS template +.DS_Store +.AppleDouble +.LSOverride + +# Thumbnails +._* + +# VSCode +.vscode + +# log +*.log + +# coverage file +coverage.html + +examples/server/rules/ +examples/server/js/ +examples/server/plugins/ \ No newline at end of file diff --git a/builtin/builtin.go b/builtin/builtin.go new file mode 100644 index 0000000..faddd68 --- /dev/null +++ b/builtin/builtin.go @@ -0,0 +1,111 @@ +/* + * Copyright 2024 The RuleGo Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package builtin + +import ( + "fmt" + "github.com/montanaflynn/stats" +) + +var AggregateBuiltins = map[string]*AggregateFunction{} + +func init() { + for _, item := range aggregateBuiltins { + AggregateBuiltins[item.Name] = item + for _, name := range item.Alias { + AggregateBuiltins[name] = item + } + } +} + +var aggregateBuiltins = []*AggregateFunction{ + { + Name: "avg", + Func: func(input []float64, args ...any) (any, error) { + return stats.Mean(input) + }, + }, + { + Name: "count", + Func: func(input []float64, args ...any) (any, error) { + return len(input), nil + }, + }, + { + Name: "sum", + Func: func(input []float64, args ...any) (any, error) { + return stats.Sum(input) + }, + }, + { + Name: "min", + Func: func(input []float64, args ...any) (any, error) { + return stats.Min(input) + }, + }, + { + Name: "max", + Func: func(input []float64, args ...any) (any, error) { + return stats.Max(input) + }, + }, + //返回组中所有值的标准差。空值不参与计算。 + { + Name: "stddev", + Func: func(input []float64, args ...any) (any, error) { + return stats.StandardDeviation(input) + }, + }, + //返回组中所有值的样本标准差。空值不参与计算。 + { + Name: "stddevs", + Func: func(input []float64, args ...any) (any, error) { + return stats.StandardDeviationSample(input) + }, + }, + { + Name: "var", + Func: func(input []float64, args ...any) (any, error) { + return stats.Variance(input) + }, + }, + { + Name: "vars", + Func: func(input []float64, args ...any) (any, error) { + return stats.VarS(input) + }, + }, + { + Name: "median", + Func: func(input []float64, args ...any) (any, error) { + return stats.Median(input) + }, + }, + { + Name: "percentile", + Func: func(input []float64, args ...any) (any, error) { + if len(args) < 1 { + return 0, fmt.Errorf("invalid number of arguments for percentile (expected 1, got %d)", len(args)) + } + if percent, ok := args[0].(float64); !ok { + return 0, fmt.Errorf("percent need float64 type (got %d)", len(args)) + } else { + return stats.Percentile(input, percent) + } + }, + }, +} diff --git a/builtin/function.go b/builtin/function.go new file mode 100644 index 0000000..3b8fe54 --- /dev/null +++ b/builtin/function.go @@ -0,0 +1,38 @@ +/* + * Copyright 2024 The RuleGo Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package builtin + +//type Function struct { +// AuthorName string +// Func func(args ...any) (any, error) +// Fast func(arg any) any +// ValidateArgs func(args ...any) (any, error) +// Types []reflect.Type +// Validate func(args []reflect.Type) (reflect.Type, error) +// Predicate bool +//} + +type AggregateFunction struct { + //函数名 + Name string + //别名 + Alias []string + //函数 + Func func(input []float64, args ...any) (any, error) + //验证参数 + ValidateArgs func(input []float64, args ...any) (any, error) +} diff --git a/collector/aggregate_collector.go b/collector/aggregate_collector.go new file mode 100644 index 0000000..d711f20 --- /dev/null +++ b/collector/aggregate_collector.go @@ -0,0 +1,26 @@ +/* + * Copyright 2024 The RuleGo Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package collector + +import ( + "github.com/rulego/streamsql/types" +) + +type aggregateCollector struct { + groupBy []string + window types.Window +} diff --git a/collector/collector.go b/collector/collector.go new file mode 100644 index 0000000..0ec5fbf --- /dev/null +++ b/collector/collector.go @@ -0,0 +1,263 @@ +/* + * Copyright 2024 The RuleGo Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package collector + +import ( + "context" + "errors" + "github.com/rulego/streamsql/dataset" + "github.com/rulego/streamsql/types" + "sync" +) + +// DefaultSelectStreamSqlContext sql执行生命周期上下文 +type DefaultSelectStreamSqlContext struct { + context.Context + Collector *StreamCollector + // 当前传入已经转成map的流数据 + msgAsMap map[string]interface{} + msg types.Msg + locker sync.RWMutex + groupValues types.GroupValues +} + +func NewDefaultSelectStreamSqlContext(ctx context.Context, collector *StreamCollector, msg types.Msg) *DefaultSelectStreamSqlContext { + return &DefaultSelectStreamSqlContext{ + Context: ctx, + Collector: collector, + msg: msg, + } +} +func (c *DefaultSelectStreamSqlContext) Decode() error { + if c.msg.Data == nil { + return errors.New("data can not nil") + } + //已经解码 + if c.msgAsMap != nil { + return nil + } + if mapData, err := c.msg.Data.DecodeAsMap(); err != nil { + return err + } else { + c.msgAsMap = mapData + } + return nil +} + +func (c *DefaultSelectStreamSqlContext) InputAsMap() map[string]interface{} { + c.locker.RLock() + defer c.locker.RUnlock() + return c.msgAsMap +} + +func (c *DefaultSelectStreamSqlContext) RawInput() types.Msg { + return c.msg +} + +func (c *DefaultSelectStreamSqlContext) SetGroupByKey(groupByKey types.GroupFields) { + c.Collector.SetGroupByKey(groupByKey) +} +func (c *DefaultSelectStreamSqlContext) GetGroupByKey() types.GroupFields { + return c.Collector.GetGroupByKey() +} +func (c *DefaultSelectStreamSqlContext) IsInitWindow(groupValues types.GroupValues) bool { + return c.Collector.IsInitWindow(groupValues) +} + +func (c *DefaultSelectStreamSqlContext) AddWindow(groupByKey types.GroupValues, window types.Window) { + c.Collector.AddWindow(groupByKey, window) +} + +func (c *DefaultSelectStreamSqlContext) CreteWindowObserver() types.WindowObserver { + return c.Collector.CreteWindowObserver() +} + +func (c *DefaultSelectStreamSqlContext) GetWindow(groupByKey types.GroupValues) types.Window { + return c.Collector.GetWindow(groupByKey) +} + +func (c *DefaultSelectStreamSqlContext) GetRow(groupValues types.GroupValues) (*dataset.Row, bool) { + return c.Collector.GetRow(groupValues) +} + +func (c *DefaultSelectStreamSqlContext) AddColumn(groupValues types.GroupValues, kv dataset.KeyValue) { + c.Collector.AddColumn(groupValues, kv) +} + +func (c *DefaultSelectStreamSqlContext) GetColumn(groupValues types.GroupValues, key dataset.Key) (dataset.KeyValue, bool) { + return c.Collector.GetColumn(groupValues, key) +} + +//func (c *DefaultSelectStreamSqlContext) AddField(fieldId string, value any) { +// c.locker.Lock() +// defer c.locker.Unlock() +// c.msgAsMap[fieldId] = value +// +//} +// +//func (c *DefaultSelectStreamSqlContext) GetField(fieldId string) (any, bool) { +// c.locker.RLock() +// defer c.locker.RUnlock() +// v, ok := c.msgAsMap[fieldId] +// return v, ok +//} + +func (c *DefaultSelectStreamSqlContext) SetCurrentGroupValues(groupValues types.GroupValues) { + c.groupValues = groupValues +} + +func (c *DefaultSelectStreamSqlContext) GetCurrentGroupValues() types.GroupValues { + return c.groupValues +} + +func (c *DefaultSelectStreamSqlContext) AddFieldAggregateValue(groupValues types.GroupValues, fieldId string, value float64) { + c.Collector.AddFieldAggregateValue(groupValues, fieldId, value) +} + +func (c *DefaultSelectStreamSqlContext) GetFieldAggregateValue(groupValues types.GroupValues, fieldId string) []float64 { + return c.Collector.GetFieldAggregateValue(groupValues, fieldId) +} + +type aggregateFieldValue struct { + GroupFields types.GroupFields + FieldWindows map[string]types.Window +} + +func (afv *aggregateFieldValue) AddFieldValue(fieldId string, data float64) { + if w, ok := afv.FieldWindows[fieldId]; !ok { + //afv.FieldWindows[fieldId] = newQueue + } else { + w.Add(data) + } +} + +func (afv *aggregateFieldValue) GetFieldValues(fieldId string) []float64 { + return nil +} + +// StreamCollector 收集器 +type StreamCollector struct { + context.Context + keyedWindow map[types.GroupValues]types.Window + planner types.LogicalPlan + aggregateOperators []types.AggregateOperator + groupByKey types.GroupFields + aggregateFieldValues map[types.GroupValues]*aggregateFieldValue + rows *dataset.Rows + sync.Mutex +} + +func (c *StreamCollector) SetGroupByKey(groupByKey types.GroupFields) { + c.groupByKey = groupByKey +} +func (c *StreamCollector) GetGroupByKey() types.GroupFields { + return c.groupByKey +} + +func (c *StreamCollector) IsInitWindow(groupValues types.GroupValues) bool { + _, ok := c.keyedWindow[groupValues] + return ok +} + +func (c *StreamCollector) AddWindow(groupByKey types.GroupValues, window types.Window) { + c.keyedWindow[groupByKey] = window +} + +func (c *StreamCollector) GetWindow(groupByKey types.GroupValues) types.Window { + return c.keyedWindow[groupByKey] +} + +func (c *StreamCollector) CreteWindowObserver() types.WindowObserver { + return types.WindowObserver{ + AddHandler: func(context types.StreamSqlOperatorContext, data float64) { + for _, op := range c.aggregateOperators { + op.AddHandler(context, data) + } + }, + ArchiveHandler: func(context types.StreamSqlOperatorContext, dataList []float64) { + for _, op := range c.aggregateOperators { + op.ArchiveHandler(context, dataList) + } + }, + StartHandler: func(context types.StreamSqlOperatorContext) { + }, + EndHandler: func(context types.StreamSqlOperatorContext, dataList []float64) { + for _, op := range c.aggregateOperators { + op.EndHandler(context, dataList) + } + + }, + } +} +func (c *StreamCollector) AddFieldAggregateValue(groupValues types.GroupValues, fieldId string, data float64) { + c.Lock() + defer c.Unlock() + if w, ok := c.keyedWindow[groupValues]; ok { + w.Add(data) + } +} + +func (c *StreamCollector) GetFieldAggregateValue(groupValues types.GroupValues, fieldId string) []float64 { + c.Lock() + defer c.Unlock() + if _, ok := c.keyedWindow[groupValues]; ok { + return nil + } + return nil +} + +func (c *StreamCollector) GetRow(groupValues types.GroupValues) (*dataset.Row, bool) { + c.Lock() + defer c.Unlock() + return c.rows.GetRow(groupValues) +} + +func (c *StreamCollector) AddColumn(groupValues types.GroupValues, kv dataset.KeyValue) { + c.Lock() + defer c.Unlock() + c.rows.AddColumn(groupValues, kv) +} + +func (c *StreamCollector) GetColumn(groupValues types.GroupValues, key dataset.Key) (dataset.KeyValue, bool) { + c.Lock() + defer c.Unlock() + return c.rows.GetColumn(groupValues, key) +} + +func NewStreamCollector(planner types.LogicalPlan) *StreamCollector { + c := &StreamCollector{ + planner: planner, + aggregateOperators: planner.AggregateOperators(), + keyedWindow: make(map[types.GroupValues]types.Window), + aggregateFieldValues: make(map[types.GroupValues]*aggregateFieldValue), + rows: dataset.NewRows(), + } + return c +} + +// Init 初始化 +func (c *StreamCollector) Init() error { + return c.planner.Init(c) +} + +func (c *StreamCollector) Collect(ctx context.Context, msg types.Msg) error { + selectStreamSqlContext := NewDefaultSelectStreamSqlContext(ctx, c, msg) + if err := selectStreamSqlContext.Decode(); err != nil { + return err + } + return c.planner.Apply(selectStreamSqlContext) +} diff --git a/collector/collector_test.go b/collector/collector_test.go new file mode 100644 index 0000000..2c6951c --- /dev/null +++ b/collector/collector_test.go @@ -0,0 +1,41 @@ +/* + * Copyright 2023 The RuleGo Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package collector + +import ( + "bytes" + "fmt" + "github.com/rulego/streamsql/rsql" + "os" + "testing" +) + +func TestCollector(t *testing.T) { + sql := "select deviceId,avg(temperature+20/20,aa)/2 as aa from Input where deviceId='aa' OR deviceId='bb' group by deviceId,TumblingWindow(10m) ;" + newParser := rsql.NewParser(rsql.NewLexer(sql)) + statement := newParser.ParseStatement() + buf := new(bytes.Buffer) + statement.Format(buf) + buf.WriteTo(os.Stdout) + fmt.Println(statement) + //collector, err := planner.CreateSelectPlanner(statement.(*rsql.Select)) + //fmt.Println(err) + //msg := types.NewJsonMsg(0, types.NewMetadata(), `{"temperature":50}`) + //err = collector.Collect(context.TODO(), msg) + //fmt.Println(err) + //time.Sleep(time.Second * 10) +} diff --git a/conf/config.go b/conf/config.go new file mode 100644 index 0000000..ff863b2 --- /dev/null +++ b/conf/config.go @@ -0,0 +1,17 @@ +/* + * Copyright 2024 The RuleGo Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package conf diff --git a/dataset/key.go b/dataset/key.go new file mode 100644 index 0000000..d52a353 --- /dev/null +++ b/dataset/key.go @@ -0,0 +1,134 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dataset // import "go.opentelemetry.io/otel/attribute" + +// Key represents the key part in key-value pairs. It's a string. The +// allowed character set in the key depends on the use of the key. +type Key string + +// Bool creates a KeyValue instance with a BOOL Value. +// +// If creating both a key and value at the same time, use the provided +// convenience function instead -- Bool(name, value). +func (k Key) Bool(v bool) KeyValue { + return KeyValue{ + Key: k, + Value: BoolValue(v), + } +} + +// BoolSlice creates a KeyValue instance with a BOOLSLICE Value. +// +// If creating both a key and value at the same time, use the provided +// convenience function instead -- BoolSlice(name, value). +func (k Key) BoolSlice(v []bool) KeyValue { + return KeyValue{ + Key: k, + Value: BoolSliceValue(v), + } +} + +// Int creates a KeyValue instance with an INT64 Value. +// +// If creating both a key and value at the same time, use the provided +// convenience function instead -- Int(name, value). +func (k Key) Int(v int) KeyValue { + return KeyValue{ + Key: k, + Value: IntValue(v), + } +} + +// IntSlice creates a KeyValue instance with an INT64SLICE Value. +// +// If creating both a key and value at the same time, use the provided +// convenience function instead -- IntSlice(name, value). +func (k Key) IntSlice(v []int) KeyValue { + return KeyValue{ + Key: k, + Value: IntSliceValue(v), + } +} + +// Int64 creates a KeyValue instance with an INT64 Value. +// +// If creating both a key and value at the same time, use the provided +// convenience function instead -- Int64(name, value). +func (k Key) Int64(v int64) KeyValue { + return KeyValue{ + Key: k, + Value: Int64Value(v), + } +} + +// Int64Slice creates a KeyValue instance with an INT64SLICE Value. +// +// If creating both a key and value at the same time, use the provided +// convenience function instead -- Int64Slice(name, value). +func (k Key) Int64Slice(v []int64) KeyValue { + return KeyValue{ + Key: k, + Value: Int64SliceValue(v), + } +} + +// Float64 creates a KeyValue instance with a FLOAT64 Value. +// +// If creating both a key and value at the same time, use the provided +// convenience function instead -- Float64(name, value). +func (k Key) Float64(v float64) KeyValue { + return KeyValue{ + Key: k, + Value: Float64Value(v), + } +} + +// Float64Slice creates a KeyValue instance with a FLOAT64SLICE Value. +// +// If creating both a key and value at the same time, use the provided +// convenience function instead -- Float64(name, value). +func (k Key) Float64Slice(v []float64) KeyValue { + return KeyValue{ + Key: k, + Value: Float64SliceValue(v), + } +} + +// String creates a KeyValue instance with a STRING Value. +// +// If creating both a key and value at the same time, use the provided +// convenience function instead -- String(name, value). +func (k Key) String(v string) KeyValue { + return KeyValue{ + Key: k, + Value: StringValue(v), + } +} + +// StringSlice creates a KeyValue instance with a STRINGSLICE Value. +// +// If creating both a key and value at the same time, use the provided +// convenience function instead -- StringSlice(name, value). +func (k Key) StringSlice(v []string) KeyValue { + return KeyValue{ + Key: k, + Value: StringSliceValue(v), + } +} + +// Defined returns true for non-empty keys. +func (k Key) Defined() bool { + return len(k) != 0 +} diff --git a/dataset/kv.go b/dataset/kv.go new file mode 100644 index 0000000..03f6cfe --- /dev/null +++ b/dataset/kv.go @@ -0,0 +1,86 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dataset + +import ( + "fmt" +) + +// KeyValue holds a key and value pair. +type KeyValue struct { + Key Key + Value Value +} + +// Valid returns if kv is a valid OpenTelemetry attribute. +func (kv KeyValue) Valid() bool { + return kv.Key.Defined() && kv.Value.Type() != INVALID +} + +// Bool creates a KeyValue with a BOOL Value type. +func Bool(k string, v bool) KeyValue { + return Key(k).Bool(v) +} + +// BoolSlice creates a KeyValue with a BOOLSLICE Value type. +func BoolSlice(k string, v []bool) KeyValue { + return Key(k).BoolSlice(v) +} + +// Int creates a KeyValue with an INT64 Value type. +func Int(k string, v int) KeyValue { + return Key(k).Int(v) +} + +// IntSlice creates a KeyValue with an INT64SLICE Value type. +func IntSlice(k string, v []int) KeyValue { + return Key(k).IntSlice(v) +} + +// Int64 creates a KeyValue with an INT64 Value type. +func Int64(k string, v int64) KeyValue { + return Key(k).Int64(v) +} + +// Int64Slice creates a KeyValue with an INT64SLICE Value type. +func Int64Slice(k string, v []int64) KeyValue { + return Key(k).Int64Slice(v) +} + +// Float64 creates a KeyValue with a FLOAT64 Value type. +func Float64(k string, v float64) KeyValue { + return Key(k).Float64(v) +} + +// Float64Slice creates a KeyValue with a FLOAT64SLICE Value type. +func Float64Slice(k string, v []float64) KeyValue { + return Key(k).Float64Slice(v) +} + +// String creates a KeyValue with a STRING Value type. +func String(k, v string) KeyValue { + return Key(k).String(v) +} + +// StringSlice creates a KeyValue with a STRINGSLICE Value type. +func StringSlice(k string, v []string) KeyValue { + return Key(k).StringSlice(v) +} + +// Stringer creates a new key-value pair with a passed name and a string +// value generated by the passed Stringer interface. +func Stringer(k string, v fmt.Stringer) KeyValue { + return Key(k).String(v.String()) +} diff --git a/dataset/row.go b/dataset/row.go new file mode 100644 index 0000000..0b68838 --- /dev/null +++ b/dataset/row.go @@ -0,0 +1,99 @@ +/* + * Copyright 2024 The RuleGo Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dataset + +import "github.com/rulego/streamsql/types" + +type Row struct { + GroupValues types.GroupValues + Columns map[Key]KeyValue +} + +func NewRow(groupValues types.GroupValues) *Row { + r := &Row{ + GroupValues: groupValues, + Columns: make(map[Key]KeyValue), + } + return r +} + +func (r *Row) AddColumn(kv KeyValue) { + if r.Columns == nil { + r.Columns = make(map[Key]KeyValue) + } + r.Columns[kv.Key] = kv +} + +func (r *Row) GetColumn(key Key) (KeyValue, bool) { + if r.Columns == nil { + return KeyValue{}, false + } + v, ok := r.Columns[key] + return v, ok +} + +type Rows struct { + Rows map[types.GroupValues]*Row +} + +func NewRows() *Rows { + r := &Rows{ + Rows: make(map[types.GroupValues]*Row), + } + return r +} + +func (r *Rows) AddColumn(groupValues types.GroupValues, kv KeyValue) { + if r.Rows == nil { + r.Rows = make(map[types.GroupValues]*Row) + } + row, ok := r.Rows[groupValues] + if !ok { + row = NewRow(groupValues) + r.Rows[groupValues] = row + } + row.AddColumn(kv) +} + +func (r *Rows) GetColumn(groupValues types.GroupValues, key Key) (KeyValue, bool) { + if r.Rows == nil { + return KeyValue{}, false + } + row, ok := r.Rows[groupValues] + if !ok { + return KeyValue{}, false + } + return row.GetColumn(key) +} + +func (r *Rows) GetRow(groupValues types.GroupValues) (*Row, bool) { + if r.Rows == nil { + return nil, false + } + v, ok := r.Rows[groupValues] + return v, ok +} + +func (r *Rows) AddRow(row *Row) { + if row == nil { + return + } + if r.Rows == nil { + r.Rows = make(map[types.GroupValues]*Row) + } + r.Rows[row.GroupValues] = row +} diff --git a/dataset/value.go b/dataset/value.go new file mode 100644 index 0000000..fbebca4 --- /dev/null +++ b/dataset/value.go @@ -0,0 +1,269 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dataset + +import ( + "encoding/json" + "fmt" + "github.com/rulego/streamsql/utils/attribute" + "github.com/rulego/streamsql/utils/compress" + "reflect" + "strconv" +) + +//go:generate stringer -type=Type + +// Type describes the type of the data Value holds. +type Type int // nolint: revive // redefines builtin Type. + +// Value represents the value part in key-value pairs. +type Value struct { + vtype Type + numeric uint64 + stringly string + slice interface{} +} + +const ( + // INVALID is used for a Value with no value set. + INVALID Type = iota + // BOOL is a boolean Type Value. + BOOL + // INT64 is a 64-bit signed integral Type Value. + INT64 + // FLOAT64 is a 64-bit floating point Type Value. + FLOAT64 + // STRING is a string Type Value. + STRING + // BOOLSLICE is a slice of booleans Type Value. + BOOLSLICE + // INT64SLICE is a slice of 64-bit signed integral numbers Type Value. + INT64SLICE + // FLOAT64SLICE is a slice of 64-bit floating point numbers Type Value. + FLOAT64SLICE + // STRINGSLICE is a slice of strings Type Value. + STRINGSLICE +) + +// BoolValue creates a BOOL Value. +func BoolValue(v bool) Value { + return Value{ + vtype: BOOL, + numeric: compress.BoolToRaw(v), + } +} + +// BoolSliceValue creates a BOOLSLICE Value. +func BoolSliceValue(v []bool) Value { + return Value{vtype: BOOLSLICE, slice: attribute.BoolSliceValue(v)} +} + +// IntValue creates an INT64 Value. +func IntValue(v int) Value { + return Int64Value(int64(v)) +} + +// IntSliceValue creates an INTSLICE Value. +func IntSliceValue(v []int) Value { + var int64Val int64 + cp := reflect.New(reflect.ArrayOf(len(v), reflect.TypeOf(int64Val))) + for i, val := range v { + cp.Elem().Index(i).SetInt(int64(val)) + } + return Value{ + vtype: INT64SLICE, + slice: cp.Elem().Interface(), + } +} + +// Int64Value creates an INT64 Value. +func Int64Value(v int64) Value { + return Value{ + vtype: INT64, + numeric: compress.Int64ToRaw(v), + } +} + +// Int64SliceValue creates an INT64SLICE Value. +func Int64SliceValue(v []int64) Value { + return Value{vtype: INT64SLICE, slice: attribute.Int64SliceValue(v)} +} + +// Float64Value creates a FLOAT64 Value. +func Float64Value(v float64) Value { + return Value{ + vtype: FLOAT64, + numeric: compress.Float64ToRaw(v), + } +} + +// Float64SliceValue creates a FLOAT64SLICE Value. +func Float64SliceValue(v []float64) Value { + return Value{vtype: FLOAT64SLICE, slice: attribute.Float64SliceValue(v)} +} + +// StringValue creates a STRING Value. +func StringValue(v string) Value { + return Value{ + vtype: STRING, + stringly: v, + } +} + +// StringSliceValue creates a STRINGSLICE Value. +func StringSliceValue(v []string) Value { + return Value{vtype: STRINGSLICE, slice: attribute.StringSliceValue(v)} +} + +// Type returns a type of the Value. +func (v Value) Type() Type { + return v.vtype +} + +// AsBool returns the bool value. Make sure that the Value's type is +// BOOL. +func (v Value) AsBool() bool { + return compress.RawToBool(v.numeric) +} + +// AsBoolSlice returns the []bool value. Make sure that the Value's type is +// BOOLSLICE. +func (v Value) AsBoolSlice() []bool { + if v.vtype != BOOLSLICE { + return nil + } + return v.asBoolSlice() +} + +func (v Value) asBoolSlice() []bool { + return attribute.AsBoolSlice(v.slice) +} + +// AsInt64 returns the int64 value. Make sure that the Value's type is +// INT64. +func (v Value) AsInt64() int64 { + return compress.RawToInt64(v.numeric) +} + +// AsInt64Slice returns the []int64 value. Make sure that the Value's type is +// INT64SLICE. +func (v Value) AsInt64Slice() []int64 { + if v.vtype != INT64SLICE { + return nil + } + return v.asInt64Slice() +} + +func (v Value) asInt64Slice() []int64 { + return attribute.AsInt64Slice(v.slice) +} + +// AsFloat64 returns the float64 value. Make sure that the Value's +// type is FLOAT64. +func (v Value) AsFloat64() float64 { + return compress.RawToFloat64(v.numeric) +} + +// AsFloat64Slice returns the []float64 value. Make sure that the Value's type is +// FLOAT64SLICE. +func (v Value) AsFloat64Slice() []float64 { + if v.vtype != FLOAT64SLICE { + return nil + } + return v.asFloat64Slice() +} + +func (v Value) asFloat64Slice() []float64 { + return attribute.AsFloat64Slice(v.slice) +} + +// AsString returns the string value. Make sure that the Value's type +// is STRING. +func (v Value) AsString() string { + return v.stringly +} + +// AsStringSlice returns the []string value. Make sure that the Value's type is +// STRINGSLICE. +func (v Value) AsStringSlice() []string { + if v.vtype != STRINGSLICE { + return nil + } + return v.asStringSlice() +} + +func (v Value) asStringSlice() []string { + return attribute.AsStringSlice(v.slice) +} + +type unknownValueType struct{} + +// AsInterface returns Value's data as interface{}. +func (v Value) AsInterface() interface{} { + switch v.Type() { + case BOOL: + return v.AsBool() + case BOOLSLICE: + return v.asBoolSlice() + case INT64: + return v.AsInt64() + case INT64SLICE: + return v.asInt64Slice() + case FLOAT64: + return v.AsFloat64() + case FLOAT64SLICE: + return v.asFloat64Slice() + case STRING: + return v.stringly + case STRINGSLICE: + return v.asStringSlice() + } + return unknownValueType{} +} + +// Emit returns a string representation of Value's data. +func (v Value) Emit() string { + switch v.Type() { + case BOOLSLICE: + return fmt.Sprint(v.asBoolSlice()) + case BOOL: + return strconv.FormatBool(v.AsBool()) + case INT64SLICE: + return fmt.Sprint(v.asInt64Slice()) + case INT64: + return strconv.FormatInt(v.AsInt64(), 10) + case FLOAT64SLICE: + return fmt.Sprint(v.asFloat64Slice()) + case FLOAT64: + return fmt.Sprint(v.AsFloat64()) + case STRINGSLICE: + return fmt.Sprint(v.asStringSlice()) + case STRING: + return v.stringly + default: + return "unknown" + } +} + +// MarshalJSON returns the JSON encoding of the Value. +func (v Value) MarshalJSON() ([]byte, error) { + var jsonVal struct { + Type string + Value interface{} + } + jsonVal.Type = v.Type() + jsonVal.Value = v.AsInterface() + return json.Marshal(jsonVal) +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..f802a19 --- /dev/null +++ b/go.mod @@ -0,0 +1,9 @@ +module github.com/rulego/streamsql + +go 1.18 + +require ( + github.com/expr-lang/expr v1.16.0 + github.com/golang/snappy v0.0.4 + github.com/montanaflynn/stats v0.7.1 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..3a8e31c --- /dev/null +++ b/go.sum @@ -0,0 +1,10 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/expr-lang/expr v1.16.0 h1:BQabx+PbjsL2PEQwkJ4GIn3CcuUh8flduHhJ0lHjWwE= +github.com/expr-lang/expr v1.16.0/go.mod h1:uCkhfG+x7fcZ5A5sXHKuQ07jGZRl6J0FCAaf2k4PtVQ= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE= +github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/operator/aggregate_op.go b/operator/aggregate_op.go new file mode 100644 index 0000000..b581b7a --- /dev/null +++ b/operator/aggregate_op.go @@ -0,0 +1,89 @@ +/* + * Copyright 2024 The RuleGo Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package operator + +import ( + "fmt" + "github.com/expr-lang/expr/vm" + "github.com/rulego/streamsql/builtin" + "github.com/rulego/streamsql/dataset" + "github.com/rulego/streamsql/rsql" + "github.com/rulego/streamsql/types" + "github.com/rulego/streamsql/utils/cast" +) + +type AggregateOp struct { + BaseOp + //WindowType rsql.WindowType + AggregateFunc *rsql.FunctionCall + //dataList *queue.Queue + //groupDataList map[types.GroupFields]*queue.Queue + ArgsProgram []*vm.Program +} + +func (o *AggregateOp) Init(context types.StreamSqlContext) error { + //o.groupDataList = make(map[types.GroupFields]*queue.Queue) + + //o.dataList = queue.NewCircleQueue(10000) + return nil +} + +func (o *AggregateOp) Apply(context types.StreamSqlContext) error { + if ctx, ok := context.(types.SelectStreamSqlContext); ok { + for index, arg := range o.AggregateFunc.Args { + fieldId := arg.(*rsql.ExpressionLang).Val + if v, ok := ctx.GetColumn(ctx.GetCurrentGroupValues(), dataset.Key(fieldId)); ok { + ctx.AddFieldAggregateValue(ctx.GetCurrentGroupValues(), fieldId, cast.ToFloat(v)) + } else { + if result, err := o.eval(o.ArgsProgram[index], ctx.InputAsMap()); err != nil { + return err + } else { + ctx.AddColumn(ctx.GetCurrentGroupValues(), dataset.Float64(fieldId, cast.ToFloat(result))) + ctx.AddFieldAggregateValue(ctx.GetCurrentGroupValues(), fieldId, cast.ToFloat(result)) + } + } + } + } + return nil +} + +// AddHandler 窗口添加数据事件 +func (o *AggregateOp) AddHandler(context types.StreamSqlOperatorContext, data float64) { + fmt.Println(data) +} + +// ArchiveHandler 清除原始数据，观察者需要保存中间过程 +func (o *AggregateOp) ArchiveHandler(context types.StreamSqlOperatorContext, dataList []float64) { + +} + +// StartHandler 窗口开始事件 +func (o *AggregateOp) StartHandler(context types.StreamSqlOperatorContext) { + +} + +// EndHandler 窗口结束事件 +func (o *AggregateOp) EndHandler(context types.StreamSqlOperatorContext, dataList []float64) { + if f, ok := builtin.AggregateBuiltins[o.AggregateFunc.Name]; ok { + + fieldId := o.AggregateFunc.Args[0].(*rsql.ExpressionLang).Val + values := context.GetFieldAggregateValue(fieldId) + result, err := f.Func(values) + fmt.Println(result, err) + //o.dataList.Reset() + } +} diff --git a/operator/alias_op.go b/operator/alias_op.go new file mode 100644 index 0000000..12270f5 --- /dev/null +++ b/operator/alias_op.go @@ -0,0 +1,17 @@ +/* + * Copyright 2024 The RuleGo Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package operator diff --git a/operator/filter_op.go b/operator/filter_op.go new file mode 100644 index 0000000..ae67071 --- /dev/null +++ b/operator/filter_op.go @@ -0,0 +1,42 @@ +/* + * Copyright 2024 The RuleGo Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package operator + +import ( + "github.com/expr-lang/expr/vm" + types2 "github.com/rulego/streamsql/types" +) + +type FilterOp struct { + BaseOp + Program *vm.Program +} + +func (o *FilterOp) Init(context types2.StreamSqlContext) error { + return nil +} + +func (o *FilterOp) Apply(context types2.StreamSqlContext) error { + if ctx, ok := context.(types2.SelectStreamSqlContext); ok { + if result, err := o.eval(o.Program, ctx.InputAsMap()); err != nil { + return err + } else if !AsBool(result) { + return types2.ErrNotMatch + } + } + return nil +} diff --git a/operator/function_op.go b/operator/function_op.go new file mode 100644 index 0000000..8fffad7 --- /dev/null +++ b/operator/function_op.go @@ -0,0 +1,21 @@ +/* + * Copyright 2024 The RuleGo Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package operator + +type FunctionOp struct { + *BaseOp +} diff --git a/operator/group_by_op.go b/operator/group_by_op.go new file mode 100644 index 0000000..ee74dc6 --- /dev/null +++ b/operator/group_by_op.go @@ -0,0 +1,77 @@ +/* + * Copyright 2024 The RuleGo Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package operator + +import ( + "github.com/rulego/streamsql/types" + "github.com/rulego/streamsql/utils/cast" +) + +//// CreateGroupByKey 创建GroupByKey +//func CreateGroupByKey(fields ...string) GroupFields { +// return fields +//} + +type GroupByOp struct { + *BaseOp + GroupByKey types.GroupFields + keys []string +} + +func (o *GroupByOp) Init(context types.StreamSqlContext) error { + o.keys = o.GroupByKey.ToList() + //if selectStreamSqlContext, ok := context.(types.StreamSqlOperatorContext); ok { + // selectStreamSqlContext.SetGroupByKey(o.GroupByKey) + //} + return nil +} + +func (o *GroupByOp) Apply(context types.StreamSqlContext) error { + if selectStreamSqlContext, ok := context.(types.SelectStreamSqlContext); ok { + if groupValues, ok := o.getGroupValues(selectStreamSqlContext.InputAsMap()); ok { + selectStreamSqlContext.SetCurrentGroupValues(groupValues) + } else { + selectStreamSqlContext.SetCurrentGroupValues(types.EmptyGroupValues) + } + } + return nil +} + +func (o *GroupByOp) getGroupValues(data map[string]interface{}) (types.GroupValues, bool) { + var list []string + for _, key := range o.keys { + if v, ok := data[key]; ok { + list = append(list, cast.ToString(v)) + } else { + return "", false + } + } + return types.NewGroupValues(list...), true +} + +//// 检查是否是当前分组数据 +//func (o *GroupByOp) checkIsGroup(data map[string]interface{}) bool { +// if data == nil { +// return false +// } +// for _, key := range o.keys { +// if _, ok := data[key]; ok { +// return true +// } +// } +// return false +//} diff --git a/operator/having_op.go b/operator/having_op.go new file mode 100644 index 0000000..12270f5 --- /dev/null +++ b/operator/having_op.go @@ -0,0 +1,17 @@ +/* + * Copyright 2024 The RuleGo Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package operator diff --git a/operator/limit_op.go b/operator/limit_op.go new file mode 100644 index 0000000..8f855bf --- /dev/null +++ b/operator/limit_op.go @@ -0,0 +1,35 @@ +/* + * Copyright 2024 The RuleGo Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package operator + +import ( + "github.com/rulego/streamsql/rsql" + "github.com/rulego/streamsql/types" +) + +type LimitOp struct { + *BaseOp + Limit *rsql.Limit +} + +func (o *LimitOp) Init(context types.StreamSqlContext) error { + return nil +} + +func (o *LimitOp) Apply(context types.StreamSqlContext) error { + return nil +} diff --git a/operator/lookup_field_op.go b/operator/lookup_field_op.go new file mode 100644 index 0000000..2716968 --- /dev/null +++ b/operator/lookup_field_op.go @@ -0,0 +1,50 @@ +/* + * Copyright 2024 The RuleGo Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package operator + +import ( + "github.com/expr-lang/expr/vm" + "github.com/rulego/streamsql/types" +) + +type LookupFieldOp struct { + *BaseOp + Field EvalField +} + +func (o *LookupFieldOp) Init(context types.StreamSqlContext) error { + return nil +} + +func (o *LookupFieldOp) Apply(context types.StreamSqlContext) error { + if ctx, ok := context.(types.SelectStreamSqlContext); ok { + if o.Field.EvalProgram != nil { + if result, err := o.eval(o.Field.EvalProgram, ctx.InputAsMap()); err == nil { + ctx.AddField(o.Field.FieldId, result) + } else { + return err + } + } + } + return nil +} + +type EvalField struct { + EvalProgram *vm.Program + FieldId string + Alias string +} diff --git a/operator/operator.go b/operator/operator.go new file mode 100644 index 0000000..5f1c4ba --- /dev/null +++ b/operator/operator.go @@ -0,0 +1,37 @@ +/* + * Copyright 2024 The RuleGo Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package operator + +import ( + "github.com/expr-lang/expr" + "github.com/expr-lang/expr/vm" +) + +type BaseOp struct { +} + +func (o *BaseOp) eval(program *vm.Program, env interface{}) (any, error) { + return expr.Run(program, env) +} + +// AsBool convert any to bool +func AsBool(input any) bool { + if v, ok := input.(bool); ok { + return v + } + return false +} diff --git a/operator/order_op.go b/operator/order_op.go new file mode 100644 index 0000000..1d1f19c --- /dev/null +++ b/operator/order_op.go @@ -0,0 +1,35 @@ +/* + * Copyright 2024 The RuleGo Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package operator + +import ( + "github.com/rulego/streamsql/rsql" + "github.com/rulego/streamsql/types" +) + +type OrderByOp struct { + *BaseOp + OrderBy []rsql.OrderByItem +} + +func (o *OrderByOp) Init(context types.StreamSqlContext) error { + return nil +} + +func (o *OrderByOp) Apply(context types.StreamSqlContext) error { + return nil +} diff --git a/operator/table_op.go b/operator/table_op.go new file mode 100644 index 0000000..51ded4c --- /dev/null +++ b/operator/table_op.go @@ -0,0 +1,35 @@ +/* + * Copyright 2024 The RuleGo Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package operator + +import ( + "github.com/rulego/streamsql/rsql" + "github.com/rulego/streamsql/types" +) + +type TableOp struct { + *BaseOp + From []rsql.Expression +} + +func (o *TableOp) Init(context types.StreamSqlContext) error { + return nil +} + +func (o *TableOp) Apply(context types.StreamSqlContext) error { + return nil +} diff --git a/operator/window_op.go b/operator/window_op.go new file mode 100644 index 0000000..9dcf011 --- /dev/null +++ b/operator/window_op.go @@ -0,0 +1,51 @@ +/* + * Copyright 2024 The RuleGo Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package operator + +import ( + "github.com/rulego/streamsql/rsql" + "github.com/rulego/streamsql/types" + "github.com/rulego/streamsql/window" + "time" +) + +type WindowOp struct { + *BaseOp + WindowType rsql.WindowType +} + +func (o *WindowOp) Init(context types.StreamSqlContext) error { + return nil +} + +func (o *WindowOp) Apply(context types.StreamSqlContext) error { + if ctx, ok := context.(types.SelectStreamSqlContext); ok && !ctx.IsInitWindow(ctx.GetCurrentGroupValues()) { + var w types.Window + switch o.WindowType { + case rsql.TUMBLING_WINDOW: + w = window.NewTumblingWindow(ctx, "", 10*time.Second, ctx.CreteWindowObserver()) + case rsql.COUNT_WINDOW: + w = window.NewCountWindow("", 100, ctx.CreteWindowObserver()) + case rsql.SLIDING_WINDOW: + w = window.NewSlidingWindow("", 10*time.Second, 5*time.Second, ctx.CreteWindowObserver()) + case rsql.SESSION_WINDOW: + w = window.NewSessionWindow("", 10*time.Second, ctx.CreteWindowObserver()) + } + ctx.AddWindow(ctx.GetCurrentGroupValues(), w) + } + return nil +} diff --git a/planner/filter_plan.go b/planner/filter_plan.go new file mode 100644 index 0000000..96f6d4e --- /dev/null +++ b/planner/filter_plan.go @@ -0,0 +1,54 @@ +/* + * Copyright 2024 The RuleGo Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package planner + +import ( + "github.com/expr-lang/expr" + "github.com/rulego/streamsql/operator" + "github.com/rulego/streamsql/rsql" + "github.com/rulego/streamsql/types" +) + +type FilterPlan struct { + *BaseLogicalPlan +} + +func (p *FilterPlan) New() types.LogicalPlan { + return &FilterPlan{BaseLogicalPlan: &BaseLogicalPlan{}} +} + +func (p *FilterPlan) Plan(statement rsql.Statement) error { + if selectStatement, ok := statement.(*rsql.Select); ok { + if expressionLang, ok := selectStatement.Where.(*rsql.ExpressionLang); ok { + code := expressionLang.Val + if program, err := expr.Compile(code); err != nil { + return err + } else { + p.AddOperators(&operator.FilterOp{Program: program}) + } + } + } + return nil +} + +func (p *FilterPlan) Explain() string { + return "" +} + +func (p *FilterPlan) Type() string { + return "FilterPlan" +} diff --git a/planner/group_by_plan.go b/planner/group_by_plan.go new file mode 100644 index 0000000..2ee3c9e --- /dev/null +++ b/planner/group_by_plan.go @@ -0,0 +1,56 @@ +/* + * Copyright 2024 The RuleGo Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package planner + +import ( + "github.com/rulego/streamsql/operator" + "github.com/rulego/streamsql/rsql" + "github.com/rulego/streamsql/types" +) + +type GroupByPlan struct { + *BaseLogicalPlan +} + +func (p *GroupByPlan) New() types.LogicalPlan { + return &GroupByPlan{BaseLogicalPlan: &BaseLogicalPlan{}} +} + +func (p *GroupByPlan) Plan(statement rsql.Statement) error { + if selectStatement, ok := statement.(*rsql.Select); ok { + for _, item := range selectStatement.GroupBy { + if functionCallField, ok := item.(*rsql.FunctionCall); ok && functionCallField.IsWindow { + p.AddOperators(&operator.WindowOp{ + WindowType: rsql.LookupIsWindowFunc(functionCallField.Name), + }) + } else { + p.AddOperators(&operator.GroupByOp{ + GroupByKey: types.GroupFields(item.(*rsql.Identifier).Val), + }) + } + } + } + return nil +} + +func (p *GroupByPlan) Explain() string { + return "" +} + +func (p *GroupByPlan) Type() string { + return "GroupByPlan" +} diff --git a/planner/limit_plan.go b/planner/limit_plan.go new file mode 100644 index 0000000..62dfbaf --- /dev/null +++ b/planner/limit_plan.go @@ -0,0 +1,48 @@ +/* + * Copyright 2024 The RuleGo Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package planner + +import ( + "github.com/rulego/streamsql/operator" + "github.com/rulego/streamsql/rsql" + "github.com/rulego/streamsql/types" +) + +type LimitPlan struct { + *BaseLogicalPlan +} + +func (p *LimitPlan) New() types.LogicalPlan { + return &LimitPlan{BaseLogicalPlan: &BaseLogicalPlan{}} +} + +func (p *LimitPlan) Plan(statement rsql.Statement) error { + if selectStatement, ok := statement.(*rsql.Select); ok { + p.AddOperators(&operator.LimitOp{ + Limit: selectStatement.Limit, + }) + } + return nil +} + +func (p *LimitPlan) Explain() string { + return "" +} + +func (p *LimitPlan) Type() string { + return "LimitPlan" +} diff --git a/planner/look_field_plan.go b/planner/look_field_plan.go new file mode 100644 index 0000000..699315f --- /dev/null +++ b/planner/look_field_plan.go @@ -0,0 +1,93 @@ +/* + * Copyright 2024 The RuleGo Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package planner + +import ( + "github.com/expr-lang/expr" + "github.com/expr-lang/expr/vm" + "github.com/rulego/streamsql/operator" + "github.com/rulego/streamsql/rsql" + "github.com/rulego/streamsql/types" +) + +type LookFieldPlan struct { + *BaseLogicalPlan +} + +func (p *LookFieldPlan) New() types.LogicalPlan { + return &LookFieldPlan{BaseLogicalPlan: &BaseLogicalPlan{}} +} + +func (p *LookFieldPlan) Plan(statement rsql.Statement) error { + if selectStatement, ok := statement.(*rsql.Select); ok { + for _, field := range selectStatement.SelectExprs { + if functionCallField, ok := field.Expr.(*rsql.FunctionCall); ok && functionCallField.IsAggregate { + if len(functionCallField.Args) > 0 { + code, ok := functionCallField.Args[0].(*rsql.ExpressionLang) + if !ok { + return nil + } + if program, err := expr.Compile(code.Val); err != nil { + return err + } else { + //p.AddOperators(&operator.LookupFieldOp{ + // Field: operator.EvalField{ + // EvalProgram: program, + // FieldId: code.Val, + // }, + //}) + p.AddOperators(&operator.AggregateOp{ + AggregateFunc: functionCallField, + ArgsProgram: []*vm.Program{program}, + }) + } + + } else { + p.AddOperators(&operator.AggregateOp{ + AggregateFunc: functionCallField, + }) + } + + } else { + code, ok := field.Expr.(*rsql.Identifier) + if !ok { + return nil + } + if program, err := expr.Compile(code.Val); err != nil { + return err + } else { + p.AddOperators(&operator.LookupFieldOp{ + Field: operator.EvalField{ + EvalProgram: program, + FieldId: code.Val, + Alias: field.Alias, + }, + }) + } + } + } + } + return nil +} + +func (p *LookFieldPlan) Explain() string { + return "" +} + +func (p *LookFieldPlan) Type() string { + return "LookFieldPlan" +} diff --git a/planner/optimizer.go b/planner/optimizer.go new file mode 100644 index 0000000..d30c0a9 --- /dev/null +++ b/planner/optimizer.go @@ -0,0 +1,26 @@ +/* + * Copyright 2024 The RuleGo Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package planner + +import ( + "github.com/rulego/streamsql/types" +) + +// Optimize 优化 +func Optimize(p types.LogicalPlan) (types.LogicalPlan, error) { + return p, nil +} diff --git a/planner/order_by_plan.go b/planner/order_by_plan.go new file mode 100644 index 0000000..af6436c --- /dev/null +++ b/planner/order_by_plan.go @@ -0,0 +1,48 @@ +/* + * Copyright 2024 The RuleGo Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package planner + +import ( + "github.com/rulego/streamsql/operator" + "github.com/rulego/streamsql/rsql" + "github.com/rulego/streamsql/types" +) + +type OrderByPlan struct { + *BaseLogicalPlan +} + +func (p *OrderByPlan) New() types.LogicalPlan { + return &OrderByPlan{BaseLogicalPlan: &BaseLogicalPlan{}} +} + +func (p *OrderByPlan) Plan(statement rsql.Statement) error { + if selectStatement, ok := statement.(*rsql.Select); ok { + p.AddOperators(&operator.OrderByOp{ + OrderBy: selectStatement.OrderBy, + }) + } + return nil +} + +func (p *OrderByPlan) Explain() string { + return "" +} + +func (p *OrderByPlan) Type() string { + return "OrderByPlan" +} diff --git a/planner/plan_test.go b/planner/plan_test.go new file mode 100644 index 0000000..3430097 --- /dev/null +++ b/planner/plan_test.go @@ -0,0 +1,46 @@ +/* + * Copyright 2023 The RuleGo Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package planner + +import ( + "bytes" + "context" + "fmt" + "github.com/rulego/streamsql/rsql" + "github.com/rulego/streamsql/types" + "os" + "testing" + "time" +) + +func TestCollector(t *testing.T) { + sql := "select deviceId,avg((temperature+20)/20) as aa from Input where deviceId='aa' || deviceId='bb' group by deviceId,TumblingWindow(10m) ;" + newParser := rsql.NewParser(rsql.NewLexer(sql)) + statement := newParser.ParseStatement() + buf := new(bytes.Buffer) + statement.Format(buf) + buf.WriteTo(os.Stdout) + fmt.Println(statement) + collector, err := CreateSelectPlanner(statement.(*rsql.Select)) + fmt.Println(err) + msg := types.NewJsonMsg(0, types.NewMetadata(), `{"temperature":50,"deviceId":"aa"}`) + err = collector.Collect(context.TODO(), msg) + msg = types.NewJsonMsg(0, types.NewMetadata(), `{"temperature":55,"deviceId":"bb"}`) + err = collector.Collect(context.TODO(), msg) + fmt.Println(err) + time.Sleep(time.Second * 10) +} diff --git a/planner/planer.go b/planner/planer.go new file mode 100644 index 0000000..59c0ff3 --- /dev/null +++ b/planner/planer.go @@ -0,0 +1,102 @@ +/* + * Copyright 2024 The RuleGo Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package planner + +import ( + "github.com/rulego/streamsql/collector" + "github.com/rulego/streamsql/rsql" + "github.com/rulego/streamsql/types" +) + +type BaseLogicalPlan struct { + // operators 计划的实际处理逻辑 + operators []types.Operator + // children 子计划 + children []types.LogicalPlan +} + +// AddOperators 添加操作 +func (p *BaseLogicalPlan) AddOperators(operators ...types.Operator) { + p.operators = append(p.operators, operators...) +} + +// AddChildren 添加子计划 +func (p *BaseLogicalPlan) AddChildren(plans ...types.LogicalPlan) { + p.children = append(p.children, plans...) +} + +func (p *BaseLogicalPlan) Init(context types.StreamSqlContext) error { + for _, op := range p.operators { + if err := op.Init(context); err != nil { + return err + } + } + for _, plan := range p.children { + if err := plan.Init(context); err != nil { + return err + } + } + return nil +} + +// Apply 执行 +func (p *BaseLogicalPlan) Apply(context types.StreamSqlContext) error { + for _, op := range p.operators { + if err := op.Apply(context); err != nil { + return err + } + } + for _, plan := range p.children { + if err := plan.Apply(context); err != nil { + return err + } + } + return nil +} + +func (p *BaseLogicalPlan) AllOperators() []types.Operator { + var operators []types.Operator + operators = append(operators, p.operators...) + for _, plan := range p.children { + operators = append(operators, plan.AllOperators()...) + } + return operators +} + +func (p *BaseLogicalPlan) AggregateOperators() []types.AggregateOperator { + var aggregateOperators []types.AggregateOperator + var operators = p.AllOperators() + for _, op := range operators { + if aggregateOp, ok := op.(types.AggregateOperator); ok { + aggregateOperators = append(aggregateOperators, aggregateOp) + } + } + return aggregateOperators +} + +func CreateSelectPlanner(statement *rsql.Select) (types.Collector, error) { + var err error + selectStatementPlan := NewSelectStatementPlan() + err = selectStatementPlan.Plan(statement) + // 优化 + plan, err := Optimize(selectStatementPlan) + + c := collector.NewStreamCollector(plan) + // 初始化 + err = c.Init() + return c, err +} diff --git a/planner/select_statement_plan.go b/planner/select_statement_plan.go new file mode 100644 index 0000000..7a79733 --- /dev/null +++ b/planner/select_statement_plan.go @@ -0,0 +1,64 @@ +/* + * Copyright 2024 The RuleGo Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package planner + +import ( + "github.com/rulego/streamsql/rsql" + "github.com/rulego/streamsql/types" +) + +// selectPlans 注册select statement planner组件 +// 注册顺序不能乱，注册顺序会影响执行结果 +var selectPlans = []types.LogicalPlan{ + &TablePlan{BaseLogicalPlan: &BaseLogicalPlan{}}, + &FilterPlan{BaseLogicalPlan: &BaseLogicalPlan{}}, + &GroupByPlan{BaseLogicalPlan: &BaseLogicalPlan{}}, + &LookFieldPlan{BaseLogicalPlan: &BaseLogicalPlan{}}, + &LimitPlan{BaseLogicalPlan: &BaseLogicalPlan{}}, + &OrderByPlan{BaseLogicalPlan: &BaseLogicalPlan{}}, +} + +type SelectStatementPlan struct { + *BaseLogicalPlan +} + +func NewSelectStatementPlan() *SelectStatementPlan { + selectStatementPlan := &SelectStatementPlan{BaseLogicalPlan: &BaseLogicalPlan{}} + selectStatementPlan.AddChildren(selectPlans...) + return selectStatementPlan +} + +func (p *SelectStatementPlan) New() types.LogicalPlan { + return &SelectStatementPlan{BaseLogicalPlan: &BaseLogicalPlan{}} +} + +func (p *SelectStatementPlan) Plan(statement rsql.Statement) error { + for _, plan := range p.children { + if err := plan.Plan(statement); err != nil { + return err + } + } + return nil +} + +func (p *SelectStatementPlan) Explain() string { + return "" +} + +func (p *SelectStatementPlan) Type() string { + return "SelectStatementPlan" +} diff --git a/planner/table_plan.go b/planner/table_plan.go new file mode 100644 index 0000000..7af0de0 --- /dev/null +++ b/planner/table_plan.go @@ -0,0 +1,48 @@ +/* + * Copyright 2024 The RuleGo Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package planner + +import ( + "github.com/rulego/streamsql/operator" + "github.com/rulego/streamsql/rsql" + "github.com/rulego/streamsql/types" +) + +type TablePlan struct { + *BaseLogicalPlan +} + +func (p *TablePlan) New() types.LogicalPlan { + return &TablePlan{BaseLogicalPlan: &BaseLogicalPlan{}} +} + +func (p *TablePlan) Plan(statement rsql.Statement) error { + if selectStatement, ok := statement.(*rsql.Select); ok { + p.AddOperators(&operator.TableOp{ + From: selectStatement.From, + }) + } + return nil +} + +func (p *TablePlan) Explain() string { + return "" +} + +func (p *TablePlan) Type() string { + return "TablePlan" +} diff --git a/rsql/ast.go b/rsql/ast.go new file mode 100644 index 0000000..08c479f --- /dev/null +++ b/rsql/ast.go @@ -0,0 +1,412 @@ +/* + * Copyright 2024 The RuleGo Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// ast.go 文件定义了抽象语法树（AST）的结构和方法 + +package rsql + +import "bytes" + +// Node 是 AST 的基础接口，所有的 AST 节点都实现了这个接口 +type Node interface { + // Format 用于将节点格式化为字符串 + Format(buf *bytes.Buffer) +} + +// Statement 是表示 SQL 语句的接口，它继承了 Node 接口 +type Statement interface { + Node + // Statement 接口没有额外的方法，只是为了区分不同类型的节点 +} + +// SelectStatement 是表示 SELECT 语句的接口，它继承了 Statement 接口 +type SelectStatement interface { + Statement + // SelectStatement 接口没有额外的方法，只是为了区分不同类型的语句 +} + +// DDLStatement 是表示数据定义语言（DDL）语句的接口，它继承了 Statement 接口 +type DDLStatement interface { + Statement + // DDLStatement 接口没有额外的方法，只是为了区分不同类型的语句 +} + +// Expression 是表示 SQL 表达式的接口，它继承了 Node 接口 +type Expression interface { + Node + // Expression 接口没有额外的方法，只是为了区分不同类型的节点 +} + +// Literal 是表示 SQL 字面量的接口，它继承了 Expression 接口 +type Literal interface { + Expression + // Literal 接口没有额外的方法，只是为了区分不同类型的表达式 +} + +//// Identifier 是表示 SQL 标识符的接口，它继承了 Expression 接口 +//type Identifier interface { +// Expression +// // Identifier 接口没有额外的方法，只是为了区分不同类型的表达式 +//} + +// Select 是表示 SELECT 语句的结构体，它实现了 SelectStatement 接口 +type Select struct { + // Distinct 表示是否有 DISTINCT 关键字 + Distinct bool + // SelectExprs 表示选择的列或表达式，它是一个 Expression 的切片 + SelectExprs []Field + // From 表示 FROM 子句，它是一个 Expression 的切片 + From []Expression + // Where 表示 WHERE 子句，它是一个 Expression 的切片 + Where Expression + // GroupBy 表示 GROUP BY 子句，它是一个 Expression 的切片 + GroupBy []Expression + // Having 表示 HAVING 子句，它是一个 Expression + Having Expression + // OrderBy 表示 ORDER BY 子句，它是一个 OrderByItem 的切片 + OrderBy []OrderByItem + // Limit 表示 LIMIT 子句，它是一个 Limit 结构体 + Limit *Limit + // Offset 表示 OFFSET 子句，它是一个 Expression + Offset Expression +} + +// Format 实现了 Node 接口的 Format 方法，用于将 Select 结构体格式化为字符串 +func (s *Select) Format(buf *bytes.Buffer) { + // 先输出 SELECT 关键字 + buf.WriteString("SELECT ") + // 如果有 DISTINCT 关键字，就输出 DISTINCT + if s.Distinct { + buf.WriteString("DISTINCT ") + } + // 遍历选择的列或表达式，用逗号分隔，调用各自的 Format 方法 + for i, expr := range s.SelectExprs { + if i > 0 { + buf.WriteString(", ") + } + expr.Format(buf) + } + // 如果有 FROM 子句，就输出 FROM 关键字 + if len(s.From) > 0 { + buf.WriteString(" FROM ") + // 遍历来源的表或子查询，用逗号分隔，调用各自的 Format 方法 + for i, expr := range s.From { + if i > 0 { + buf.WriteString(", ") + } + expr.Format(buf) + } + } + // 如果有 WHERE 子句，就输出 WHERE 关键字，调用条件表达式的 Format 方法 + if len(s.From) > 0 { + buf.WriteString(" WHERE ") + s.Where.Format(buf) + } + // 如果有 GROUP BY 子句，就输出 GROUP BY 关键字 + if len(s.GroupBy) > 0 { + buf.WriteString(" GROUP BY ") + // 遍历分组的列或表达式，用逗号分隔，调用各自的 Format 方法 + for i, expr := range s.GroupBy { + if i > 0 { + buf.WriteString(", ") + } + expr.Format(buf) + } + } + // 如果有 HAVING 子句，就输出 HAVING 关键字，调用条件表达式的 Format 方法 + if s.Having != nil { + buf.WriteString(" HAVING ") + s.Having.Format(buf) + } + // 如果有 ORDER BY 子句，就输出 ORDER BY 关键字 + if len(s.OrderBy) > 0 { + buf.WriteString(" ORDER BY ") + // 遍历排序的项，用逗号分隔，调用各自的 Format 方法 + for i, item := range s.OrderBy { + if i > 0 { + buf.WriteString(", ") + } + item.Format(buf) + } + } + // 如果有 LIMIT 子句，就输出 LIMIT 关键字，调用 Limit 结构体的 Format 方法 + if s.Limit != nil { + buf.WriteString(" LIMIT ") + s.Limit.Format(buf) + } + // 如果有 OFFSET 子句，就输出 OFFSET 关键字，调用偏移量表达式的 Format 方法 + if s.Offset != nil { + buf.WriteString(" OFFSET ") + s.Offset.Format(buf) + } +} + +type Field struct { + Expr Expression + Alias string +} + +func (f *Field) Format(buf *bytes.Buffer) { + f.Expr.Format(buf) + if f.Alias != "" { + buf.WriteString(" AS " + f.Alias) + } +} + +type ExpressionLang struct { + Val string + Type string +} + +func (f *ExpressionLang) Format(buf *bytes.Buffer) { + if f.Type == "" { + buf.WriteString("expr") + } else { + buf.WriteString(f.Type) + } + buf.WriteString(" ") + buf.WriteString(f.Val) +} + +// OrderByItem 是表示 ORDER BY 子句中的一个排序项的结构体 +type OrderByItem struct { + // Expr 表示排序的列或表达式，它是一个 Expression + Expr Expression + // Desc 表示是否降序排序 + Desc bool +} + +// Format 实现了 Node 接口的 Format 方法，用于将 OrderByItem 结构体格式化为字符串 +func (o *OrderByItem) Format(buf *bytes.Buffer) { + // 先输出排序的列或表达式，调用其 Format 方法 + o.Expr.Format(buf) + // 如果是降序排序，就输出 DESC 关键字 + if o.Desc { + buf.WriteString(" DESC") + } +} + +// Limit 是表示 LIMIT 子句的结构体 +type Limit struct { + // RowCount 表示限制的行数，它是一个 Expression + RowCount Expression +} + +// Format 实现了 Node 接口的 Format 方法，用于将 Limit 结构体格式化为字符串 +func (l *Limit) Format(buf *bytes.Buffer) { + // 直接输出限制的行数，调用其 Format 方法 + l.RowCount.Format(buf) +} + +// Create 是表示 CREATE 语句的结构体，它实现了 DDLStatement 接口 +type Create struct { + // Table 表示要创建的表的名称，它是一个 Identifier + Table Identifier + // Columns 表示要创建的表的列的定义，它是一个 ColumnDefinition 的切片 + Columns []ColumnDefinition +} + +// Format 实现了 Node 接口的 Format 方法，用于将 Create 结构体格式化为字符串 +func (c *Create) Format(buf *bytes.Buffer) { + // 先输出 CREATE TABLE 关键字 + buf.WriteString("CREATE TABLE ") + // 然后输出要创建的表的名称，调用其 Format 方法 + c.Table.Format(buf) + // 然后输出左大括号 + buf.WriteString(" {") + // 遍历要创建的表的列的定义，用逗号分隔，调用各自的 Format 方法 + for i, col := range c.Columns { + if i > 0 { + buf.WriteString(", ") + } + col.Format(buf) + } + // 最后输出右大括号 + buf.WriteString("}") +} + +// ColumnDefinition 是表示表的列的定义的结构体 +type ColumnDefinition struct { + // Name 表示列的名称，它是一个 Identifier + Name Identifier + // Type 表示列的数据类型，它是一个 DataType + Type DataType + // Constraints 表示列的约束条件，它是一个 Constraint 的切片 + Constraints []Constraint +} + +// Format 实现了 Node 接口的 Format 方法，用于将 ColumnDefinition 结构体格式化为字符串 +func (c *ColumnDefinition) Format(buf *bytes.Buffer) { + // 先输出列的名称，调用其 Format 方法 + c.Name.Format(buf) + // 然后输出空格 + buf.WriteString(" ") + // 然后输出列的数据类型，调用其 Format 方法 + c.Type.Format(buf) + // 如果有约束条件，就输出空格 + if len(c.Constraints) > 0 { + buf.WriteString(" ") + } + // 遍历列的约束条件，用空格分隔，调用各自的 Format 方法 + for i, cons := range c.Constraints { + if i > 0 { + buf.WriteString(" ") + } + cons.Format(buf) + } +} + +// DataType 是表示数据类型的接口，它继承了 Node 接口 +type DataType interface { + Node + // DataType 接口没有额外的方法，只是为了区分不同类型的节点 +} + +// Integer 是表示整数类型的结构体，它实现了 DataType 接口 +type Integer struct { + val int64 + // Unsigned 表示是否是无符号整数 + Unsigned bool +} + +// Format 实现了 Node 接口的 Format 方法，用于将 Integer 结构体格式化为字符串 +func (i *Integer) Format(buf *bytes.Buffer) { + // 先输出 INT 关键字 + buf.WriteString("INT") + // 如果是无符号整数，就输出 UNSIGNED 关键字 + if i.Unsigned { + buf.WriteString(" UNSIGNED") + } +} + +// Constraint 是表示约束条件的接口，它继承了 Node 接口 +type Constraint interface { + Node + // Constraint 接口没有额外的方法，只是为了区分不同类型的节点 +} + +// NotNull 是表示非空约束的结构体，它实现了 Constraint 接口 +type NotNull struct { + // NotNull 结构体没有额外的字段，只是为了区分不同类型的约束 +} + +// Format 实现了 Node 接口的 Format 方法，用于将 NotNull 结构体格式化为字符串 +func (n *NotNull) Format(buf *bytes.Buffer) { + // 直接输出 NOT NULL 关键字 + buf.WriteString("NOT NULL") +} + +// StringLiteral 是表示字符串字面量的结构体，它实现了 Literal 接口 +type StringLiteral struct { + // Val 表示字符串的值，它是一个 string + Val string +} + +// Format 实现了 Node 接口的 Format 方法，用于将 StringLiteral 结构体格式化为字符串 +func (s *StringLiteral) Format(buf *bytes.Buffer) { + // 先输出一个单引号 + buf.WriteString("'") + // 然后输出字符串的值，对于特殊字符，需要进行转义 + for _, r := range s.Val { + switch r { + case '\n': + buf.WriteString("\\n") + case '\r': + buf.WriteString("\\r") + case '\t': + buf.WriteString("\\t") + case '\'': + buf.WriteString("\\'") + case '\\': + buf.WriteString("\\\\") + default: + buf.WriteRune(r) + } + } + // 最后输出一个单引号 + buf.WriteString("'") +} + +// Identifier 是表示字符串字面量的结构体，它实现了 Literal 接口 +type Identifier struct { + // Val 表示字符串的值，它是一个 string + Val string +} + +// Format 实现了 Node 接口的 Format 方法，用于将 Identifier 结构体格式化为字符串 +func (s *Identifier) Format(buf *bytes.Buffer) { + +} + +// ComparisonExpr 是表示比较表达式的结构体，它实现了 Expression 接口 +type ComparisonExpr struct { + // Left 是比较表达式的左操作数，它是一个 Expression 类型 + Left Expression + // Op 是比较表达式的运算符，它是一个 Token 类型 + Op Token + // Right 是比较表达式的右操作数，它也是一个 Expression 类型 + Right Expression +} + +// Format 实现了 Node 接口的 Format 方法，用于将 ComparisonExpr 结构体格式化为字符串 +func (c *ComparisonExpr) Format(buf *bytes.Buffer) { + // 先输出左操作数 + c.Left.Format(buf) + // 再输出一个空格 + buf.WriteString(" ") + // 再输出运算符 + buf.WriteString(c.Op.Literal) + // 再输出一个空格 + buf.WriteString(" ") + // 最后输出右操作数 + c.Right.Format(buf) +} + +// ParenExpr 是一个表示括号表达式的结构体 +type ParenExpr struct { + // Lparen 表示左括号，它是一个 Token 类型 + Lparen Token + // Rparen 表示右括号，它是一个 Token 类型 + Rparen Token + // Expr 表示括号内的表达式，它是一个 Expression 类型 + Expr Expression +} + +// Format 实现了接口中的 Format 方法，用于将 ParenExpr 结构体格式化为字符串 +func (p *ParenExpr) Format(buf *bytes.Buffer) { + // 使用 buf.WriteString 方法向缓冲区写入左括号的字面值 + buf.WriteString(p.Lparen.Literal) + // 调用括号内的表达式的 Format 方法，将其内容写入缓冲区 + p.Expr.Format(buf) + // 使用 buf.WriteString 方法向缓冲区写入右括号的字面值 + buf.WriteString(p.Rparen.Literal) +} + +// FunctionCall 函数 +type FunctionCall struct { + //是否是聚合函数 + IsAggregate bool + //是否是窗口函数 + IsWindow bool + Name string + Args []Expression +} + +// Format 实现了接口中的 Format 方法，用于将 ParenExpr 结构体格式化为字符串 +func (p *FunctionCall) Format(buf *bytes.Buffer) { + buf.WriteString(p.Name) + buf.WriteString("(") +} diff --git a/rsql/lexer.go b/rsql/lexer.go new file mode 100644 index 0000000..28e292b --- /dev/null +++ b/rsql/lexer.go @@ -0,0 +1,290 @@ +/* + * Copyright 2024 The RuleGo Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package rsql + +// Lexer 是词法分析器的接口，它定义了从输入中读取 Token 的方法 +type Lexer interface { + // NextToken 用于返回输入中的下一个 Token + NextToken() Token +} + +// NewLexer 是一个工厂函数，用于根据输入的字符串创建一个 Lexer 的实例 +func NewLexer(input string) Lexer { + // 创建一个 lexer 的实例，并读取第一个字符 + l := &lexer{input: input} + l.readChar() + // 返回这个实例 + return l +} + +// lexer 是 Lexer 接口的一个实现，它用一个字符串作为输入 +type lexer struct { + // input 表示输入的字符串 + input string + // position 表示当前读取的位置 + position int + // readPosition 表示下一个要读取的位置 + readPosition int + // ch 表示当前读取的字符 + ch byte +} + +// newToken 用于根据给定的类型和字符，创建一个新的 Token 实例 +func newToken(tokenType TokenType, ch byte) Token { + // 返回一个 Token 结构体，设置其类型和字面值 + return Token{Type: tokenType, Literal: string(ch)} +} + +// skipWhitespace 用于跳过空白字符 +func (l *lexer) skipWhitespace() { + // while 循环，判断当前字符是否为空白字符 + for l.ch == ' ' || l.ch == '\t' || l.ch == '\n' || l.ch == '\r' { + // 如果是，就调用 readChar 方法，读取下一个字符 + l.readChar() + } +} + +// readChar 用于读取输入中的下一个字符，并更新位置信息 +func (l *lexer) readChar() { + // 如果已经到达输入的末尾，就将 ch 设置为 0 + if l.readPosition >= len(l.input) { + l.ch = 0 + } else { + // 否则，就将 ch 设置为下一个字符 + l.ch = l.input[l.readPosition] + } + // 将当前位置设置为下一个位置，将下一个位置增加 1 + l.position = l.readPosition + l.readPosition++ +} + +// peekChar 用于返回输入中的下一个字符，但不移动位置信息 +func (l *lexer) peekChar() byte { + // 如果已经到达输入的末尾，就返回 0 + if l.readPosition >= len(l.input) { + return 0 + } else { + // 否则，就返回下一个字符，但不改变位置信息 + return l.input[l.readPosition] + } +} + +// readComment 用于读取注释的内容，直到遇到换行符或文件结束 +func (l *lexer) readComment() string { + // 记录当前位置 + position := l.position + 1 + // while 循环，判断当前字符是否是换行符或文件结束 + for l.ch != '\n' && l.ch != 0 { + // 如果不是，就读取下一个字符 + l.readChar() + } + // 返回注释的内容，不包括两个斜杠和换行符 + return l.input[position:l.position] +} + +// readNumber 用于读取数字字面量的内容，直到遇到非数字的字符 +func (l *lexer) readNumber() string { + // 记录当前位置 + position := l.position + // while 循环，判断当前字符是否是数字 + for l.isDigit(l.ch) { + // 如果是，就读取下一个字符 + l.readChar() + } + // 返回数字的内容 + return l.input[position:l.position] +} + +// isLetter 用于判断一个字符是否是字母，包括下划线 +func (l *lexer) isLetter(ch byte) bool { + // 如果是大写或小写字母，或者是下划线，就返回 true + return 'A' <= ch && ch <= 'Z' || 'a' <= ch && ch <= 'z' || ch == '_' +} + +// isDigit 用于判断一个字符是否是数字 +func (l *lexer) isDigit(ch byte) bool { + // 如果是 0 到 9 之间的数字，就返回 true + return '0' <= ch && ch <= '9' +} + +// readIdentifier 用于读取标识符的内容，直到遇到非字母或非数字的字符 +func (l *lexer) readIdentifier() string { + // 记录当前位置 + position := l.position + // while 循环，判断当前字符是否是字母或数字 + for l.isLetter(l.ch) || l.isDigit(l.ch) { + // 如果是，就读取下一个字符 + l.readChar() + } + // 返回标识符的内容 + return l.input[position:l.position] +} + +// readString 用于读取字符串字面量的内容，直到遇到另一个单引号或文件结束 +func (l *lexer) readString() string { + // 记录当前位置 + position := l.position + // while 循环，判断当前字符是否是单引号或文件结束 + for { + // 读取下一个字符 + l.readChar() + // 如果是单引号或文件结束，就跳出循环 + if l.ch == '\'' || l.ch == 0 { + break + } + } + // 读取下一个字符 + l.readChar() + // 返回字符串的内容，包括两个单引号 + return l.input[position:l.position] +} + +// NextToken 实现了 Lexer 接口的 NextToken 方法，用于返回输入中的下一个 Token +func (l *lexer) NextToken() Token { + var tok Token + + // 跳过空白字符 + l.skipWhitespace() + + // 根据当前字符，判断 Token 的类型 + switch l.ch { + case '=': + // 如果是等号，就判断是赋值符号还是等于号 + if l.peekChar() == '=' { + // 如果下一个字符也是等号，就是等于号 + ch := l.ch + // 读取下一个字符 + l.readChar() + // 设置 Token 的类型和字面值 + tok = Token{Type: EQ, Literal: string(ch) + string(l.ch)} + } else { + // 否则，就是赋值符号 + tok = newToken(ASSIGN, l.ch) + } + case '+': + // 如果是加号，就设置 Token 的类型和字面值 + tok = newToken(PLUS, l.ch) + case '-': + // 如果是减号，就设置 Token 的类型和字面值 + tok = newToken(MINUS, l.ch) + case '*': + // 如果是星号，就设置 Token 的类型和字面值 + tok = newToken(ASTERISK, l.ch) + case '/': + // 如果是斜杠，就判断是除号还是注释 + if l.peekChar() == '/' { + // 如果下一个字符也是斜杠，就是注释 + // 读取注释的内容，直到遇到换行符或文件结束 + literal := l.readComment() + // 设置 Token 的类型和字面值 + tok = Token{Type: COMMENT, Literal: literal} + } else { + // 否则，就是除号 + tok = newToken(SLASH, l.ch) + } + case ',': + // 如果是逗号，就设置 Token 的类型和字面值 + tok = newToken(COMMA, l.ch) + case ';': + // 如果是分号，就设置 Token 的类型和字面值 + tok = newToken(SEMICOLON, l.ch) + case '(': + // 如果是左括号，就设置 Token 的类型和字面值 + tok = newToken(LPAREN, l.ch) + case ')': + // 如果是右括号，就设置 Token 的类型和字面值 + tok = newToken(RPAREN, l.ch) + case '{': + // 如果是左大括号，就设置 Token 的类型和字面值 + tok = newToken(LBRACE, l.ch) + case '}': + // 如果是右大括号，就设置 Token 的类型和字面值 + tok = newToken(RBRACE, l.ch) + case '<': + // 如果是小于号，就判断是小于号、小于等于号还是不等于号 + if l.peekChar() == '=' { + // 如果下一个字符是等号，就是小于等于号 + ch := l.ch + // 读取下一个字符 + l.readChar() + // 设置 Token 的类型和字面值 + tok = Token{Type: LE, Literal: string(ch) + string(l.ch)} + } else if l.peekChar() == '>' { + // 如果下一个字符是大于号，就是不等于号 + ch := l.ch + // 读取下一个字符 + l.readChar() + // 设置 Token 的类型和字面值 + tok = Token{Type: NOT_EQ, Literal: string(ch) + string(l.ch)} + } else { + // 否则，就是小于号 + tok = newToken(LT, l.ch) + } + case '>': + // 如果是大于号，就判断是大于号还是大于等于号 + if l.peekChar() == '=' { + // 如果下一个字符是等号，就是大于等于号 + ch := l.ch + // 读取下一个字符 + l.readChar() + // 设置 Token 的类型和字面值 + tok = Token{Type: GE, Literal: string(ch) + string(l.ch)} + } else { + // 否则，就是大于号 + tok = newToken(GT, l.ch) + } + case 0: + // 如果是文件结束，就设置 Token 的类型和字面值 + tok.Literal = "" + tok.Type = EOF + default: + // 如果是其他字符，就判断是标识符、数字字面量还是字符串字面量 + if l.isLetter(l.ch) { + // 如果是字母，就是标识符 + // 读取标识符的内容 + tok.Literal = l.readIdentifier() + // 根据标识符的内容，判断是关键字还是普通的标识符 + tok.Type = LookupIdent(tok.Literal) + // 返回 Token，不需要再读取下一个字符 + return tok + } else if l.isDigit(l.ch) { + // 如果是数字，就是数字字面量 + // 读取数字的内容 + tok.Literal = l.readNumber() + // 设置 Token 的类型为 NUMBER + tok.Type = NUMBER + // 返回 Token，不需要再读取下一个字符 + return tok + } else if l.ch == '\'' { + // 如果是单引号，就是字符串字面量 + // 读取字符串的内容，包括两个单引号 + tok.Literal = l.readString() + // 设置 Token 的类型为 STRING + tok.Type = STRING + // 返回 Token，不需要再读取下一个字符 + return tok + } else { + // 否则，就是非法字符 + tok = newToken(ILLEGAL, l.ch) + } + } + + // 读取下一个字符 + l.readChar() + // 返回 Token + return tok +} diff --git a/rsql/parser.go b/rsql/parser.go new file mode 100644 index 0000000..818c019 --- /dev/null +++ b/rsql/parser.go @@ -0,0 +1,489 @@ +/* + * Copyright 2024 The RuleGo Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// parser.go 文件定义了语法分析的接口和方法 +package rsql + +import ( + "strconv" + "strings" +) + +// Parser 是语法分析器的接口，它定义了从 Lexer 中获取 Token，并构建 AST 的方法 +type Parser interface { + // ParseStatement 用于解析一个 SQL 语句，返回一个 Statement 的实例 + ParseStatement() Statement + // ParseExpression 用于解析一个 SQL 表达式，返回一个 Expression 的实例 + ParseExpression() Expression +} + +// NewParser 是一个工厂函数，用于根据一个 Lexer 的实例创建一个 Parser 的实例 +func NewParser(l Lexer) Parser { + // 创建一个 parser 的实例 + p := &parser{l: l} + // 读取两个 Token，分别存入 curToken 和 peekToken + p.nextToken() + p.nextToken() + // 返回这个实例 + return p +} + +// parser 是 Parser 接口的一个实现，它用一个 Lexer 的实例作为输入 +type parser struct { + // l 表示输入的 Lexer 的实例 + l Lexer + // curToken 表示当前读取的 Token + curToken Token + // peekToken 表示下一个要读取的 Token + peekToken Token +} + +// ParseStatement 实现了 Parser 接口的 ParseStatement 方法，用于解析一个 SQL 语句，返回一个 Statement 的实例 +func (p *parser) ParseStatement() Statement { + // 省略具体的实现细节，只是简单地根据当前的 Token 类型调用相应的方法 + switch p.curToken.Type { + case SELECT: + return p.parseSelectStatement() + case CREATE: + return p.parseCreateStatement() + default: + return nil + } +} + +// ParseExpression 实现了 Parser 接口的 ParseExpression 方法，用于解析一个 SQL 表达式，返回一个 Expression 的实例 +func (p *parser) ParseExpression() Expression { + // 省略具体的实现细节，只是简单地根据当前的 Token 类型调用相应的方法 + switch p.curToken.Type { + case STRING: + return p.parseStringLiteral() + case IDENT: + return p.parseIdentifier() + default: + return nil + } +} + +// parseCreateStatement 用于解析一个 CREATE 语句，返回一个 Create 的实例 +func (p *parser) parseCreateStatement() *Create { + //todo 暂不支持 + return &Create{} +} + +// parseSelectStatement 用于解析一个 SELECT 语句，返回一个 Select 的实例 +func (p *parser) parseSelectStatement() *Select { + // 创建一个 Select 的实例 + stmt := &Select{} + // 如果当前的 Token 不是 SELECT，就返回 nil + if p.curToken.Type != SELECT { + return nil + } + // 读取下一个 Token + p.nextToken() + // 解析选择的列或表达式，调用 parseSelectExprs 方法，赋值给 stmt.SelectExprs + stmt.SelectExprs = p.parseSelectExprs() + // 读取下一个 Token + p.nextToken() + // 如果当前的 Token 是 FROM，就读取下一个 Token + if p.curToken.Type == FROM { + p.nextToken() + // 解析来源的表或子查询，调用 parseFrom 方法，赋值给 stmt.From + stmt.From = p.parseFrom() + // 读取下一个 Token + p.nextToken() + } + // 如果当前的 Token 是 WHERE，就读取下一个 Token + if p.curToken.Type == WHERE { + p.nextToken() + // 解析条件表达式，调用 parseExpression 方法，赋值给 stmt.Where + stmt.Where = p.parseWhere() + // 读取下一个 Token + p.nextToken() + } + // 如果当前的 Token 是 GROUP BY，就读取下一个 Token + if p.curToken.Type == GROUP { + p.nextToken() + // 如果下一个 Token 不是 BY，就返回 nil + if p.curToken.Type != BY { + return nil + } + // 读取下一个 Token + p.nextToken() + // 解析分组的列或表达式，调用 parseGroupBy 方法，赋值给 stmt.GroupBy + stmt.GroupBy = p.parseGroupBy() + // 读取下一个 Token + p.nextToken() + } + // 如果当前的 Token 是 HAVING，就读取下一个 Token + if p.curToken.Type == HAVING { + p.nextToken() + // 解析条件表达式，调用 parseExpression 方法，赋值给 stmt.Having + stmt.Having = p.parseExpression() + // 读取下一个 Token + p.nextToken() + } + // 如果当前的 Token 是 ORDER BY，就读取下一个 Token + if p.curToken.Type == ORDER { + p.nextToken() + // 如果下一个 Token 不是 BY，就返回 nil + if p.peekToken.Type != BY { + return nil + } + // 读取下一个 Token + p.nextToken() + // 解析排序的项，调用 parseOrderBy 方法，赋值给 stmt.OrderBy + stmt.OrderBy = p.parseOrderBy() + // 读取下一个 Token + p.nextToken() + } + // 如果当前的 Token 是 LIMIT，就读取下一个 Token + if p.curToken.Type == LIMIT { + p.nextToken() + // 解析限制的行数，调用 parseLimit 方法，赋值给 stmt.Limit + stmt.Limit = p.parseLimit() + // 读取下一个 Token + p.nextToken() + } + // 如果当前的 Token 是 OFFSET，就读取下一个 Token + if p.curToken.Type == OFFSET { + p.nextToken() + // 解析偏移量，调用 parseExpression 方法，赋值给 stmt.Offset + stmt.Offset = p.parseExpression() + // 读取下一个 Token + p.nextToken() + } + // 返回 stmt + return stmt +} + +// parseSelectExprs 用于解析选择的列或表达式，返回一个 Expression 的切片 +func (p *parser) parseSelectExprs() []Field { + // 创建一个空的 Expression 切片 + var fields []Field + // for 循环，直到遇到非列或表达式的 Token + for { + // 解析一个列或表达式，调用 parseExpression 方法，追加到 exprs 中 + expr := p.parseExpression() + field := Field{ + Expr: expr, + } + if p.peekToken.Type == AS { + p.nextToken() + p.nextToken() + field.Alias = p.curToken.Literal + } + fields = append(fields, field) + // 如果下一个 Token 不是逗号，就跳出循环 + if p.peekToken.Type != COMMA { + break + } + // 读取下一个 Token + p.nextToken() + // 读取下一个 Token + p.nextToken() + } + // 返回 exprs + return fields +} + +// parseFrom 用于解析来源的表或子查询，返回一个 Expression 的切片 +func (p *parser) parseFrom() []Expression { + // 创建一个空的 Expression 切片 + exprs := []Expression{} + // for 循环，直到遇到非表或子查询的 Token + for { + // 解析一个表或子查询，调用 parseExpression 方法，追加到 exprs 中 + expr := p.parseExpression() + exprs = append(exprs, expr) + // 如果下一个 Token 不是逗号，就跳出循环 + if p.peekToken.Type != COMMA { + break + } + // 读取下一个 Token + p.nextToken() + } + // 返回 exprs + return exprs +} + +func (p *parser) parseWhere() *ExpressionLang { + var builder strings.Builder + // for 循环，直到遇到非条件表达式的 Token + for { + switch p.curToken.Type { + case ASSIGN: + builder.WriteString("==") + case OR, AND, NOT: + builder.WriteString(" ") + builder.WriteString(p.curToken.Literal) + builder.WriteString(" ") + default: + builder.WriteString(p.curToken.Literal) + //builder.WriteString(" ") + } + + if p.peekToken.Type == GROUP || p.peekToken.Type == LIMIT || p.peekToken.Type == HAVING { + break + } + // 读取下一个 Token + p.nextToken() + } + return &ExpressionLang{ + Val: builder.String(), + } +} + +// parseComparisonExpr 用于解析一个比较表达式，返回一个 ComparisonExpr 的实例 +func (p *parser) parseComparisonExpr() Expression { + // 获取左操作数 + left := p.parseExpression() + // 获取比较运算符 + op := p.curToken + //// 检查是否是合法的比较运算符 + //if !op.isComparisonOperator() { + // p.errorf("invalid comparison operator: %s", op.Literal) + //} + // 移动到下一个 Token + p.nextToken() + // 获取右操作数 + right := p.parseExpression() + // 返回一个 ComparisonExpr + return &ComparisonExpr{ + Left: left, + Op: op, + Right: right, + } +} + +// parseOrderBy 用于解析排序的项，返回一个 OrderByItem 的切片 +func (p *parser) parseOrderBy() []OrderByItem { + // 创建一个空的 OrderByItem 切片 + var items []OrderByItem + // for 循环，直到遇到非排序项的 Token + for { + // 创建一个 OrderByItem 的实例 + item := OrderByItem{} + // 解析排序的列或表达式，调用 parseExpression 方法，赋值给 item.Expr + item.Expr = p.parseExpression() + // 如果下一个 Token 是 ASC 或 DESC，就读取下一个 Token + if p.peekToken.Type == ASC || p.peekToken.Type == DESC { + p.nextToken() + // 如果是 DESC，就设置 item.Desc 为 true + if p.curToken.Type == DESC { + item.Desc = true + } + } + // 追加 item 到 items 中 + items = append(items, item) + // 如果下一个 Token 不是逗号，就跳出循环 + if p.peekToken.Type != COMMA { + break + } + // 读取下一个 Token + p.nextToken() + } + // 返回 items + return items +} + +// parseLimit 用于解析限制的行数，返回一个 Limit 的实例 +func (p *parser) parseLimit() *Limit { + // 创建一个 Limit 的实例 + limit := &Limit{} + // 解析限制的行数，调用 parseExpression 方法，赋值给 limit.RowCount + limit.RowCount = p.parseExpression() + // 返回 limit + return limit +} + +// parseStringLiteral 用于解析一个字符串字面量，返回一个 StringLiteral 的实例 +func (p *parser) parseStringLiteral() *StringLiteral { + // 创建一个 StringLiteral 的实例 + lit := &StringLiteral{} + // 如果当前的 Token 不是 STRING，就返回 nil + if p.curToken.Type != STRING { + return nil + } + // 将当前的 Token 的字面值赋给 lit.Val + lit.Val = p.curToken.Literal + // 返回 lit + return lit +} + +// parseIdentifier 用于解析一个标识符，返回一个 Identifier 的实例 +func (p *parser) parseIdentifier() *Identifier { + // 创建一个 Identifier 的实例 + ident := &Identifier{} + // 如果当前的 Token 不是 IDENT，就返回 nil + if p.curToken.Type != IDENT { + return nil + } + // 将当前的 Token 的字面值赋给 ident.Val + ident.Val = p.curToken.Literal + // 返回 ident + return ident +} + +func (p *parser) parseNumber() *Integer { + // 创建一个 Identifier 的实例 + ident := &Integer{} + // 如果当前的 Token 不是 IDENT，就返回 nil + if p.curToken.Type != NUMBER { + return nil + } + i, _ := strconv.ParseInt(p.curToken.Literal, 10, 64) + ident.val = i + // 返回 ident + return ident +} + +// nextToken 用于从 Lexer 中读取下一个 Token，并更新 curToken 和 peekToken +func (p *parser) nextToken() { + // 将 peekToken 赋值给 curToken + p.curToken = p.peekToken + // 从 Lexer 中读取下一个 Token，赋值给 peekToken + p.peekToken = p.l.NextToken() +} + +// parseExpression 用于解析一个 SQL 表达式，返回一个 Expression 的实例 +func (p *parser) parseExpression() Expression { + // 省略具体的实现细节，只是简单地根据当前的 Token 类型调用相应的方法 + switch p.curToken.Type { + case STRING: + return p.parseStringLiteral() + case IDENT: + if p.peekToken.Type == LPAREN { + return p.parseFunctionCall() + } else { + return p.parseIdentifier() + } + case LPAREN: + return p.parseExpressionLang() + case NUMBER: + return p.parseNumber() + default: + switch p.peekToken.Type { + case ASSIGN, PLUS, MINUS, ASTERISK, SLASH: + return p.parseExpressionLang() + } + // 如果遇到无法解析的 Token 类型，就返回 nil + return nil + } +} + +// parseFunctionCall 用于解析一个函数调用，返回一个 FunctionCall 的实例 +func (p *parser) parseFunctionCall() Expression { + // 获取函数名，它是前一个 Token 的字面值 + name := p.curToken.Literal + // 创建一个空的 Expression 切片，用于存放参数 + var args []Expression + // 移动到下一个 Token + p.nextToken() + // for 循环，直到遇到右括号或结束符 + for p.curToken.Type != RPAREN && p.curToken.Type != EOF { + arg := p.parseExpressionLang() + //// 解析一个参数，调用 parseExpression 方法，追加到 args 中 + //arg := p.parseExpression() + args = append(args, arg) + // 如果下一个 Token 是逗号，就跳过它 + if p.peekToken.Type == COMMA { + p.nextToken() + } + if p.curToken.Type == RPAREN { + break + } + // 移动到下一个 Token + p.nextToken() + } + // 返回一个 FunctionCall + return &FunctionCall{ + IsAggregate: LookupIsAggregateFunc(name), + IsWindow: LookupIsWindowFunc(name) != NOT_WINDOW, + Name: name, + Args: args, + } +} + +func (p *parser) parseExpressionLang() *ExpressionLang { + var parenNum int + var builder strings.Builder + for p.curToken.Type != EOF { + //if index > 0 { + // builder.WriteString(" ") + //} + switch p.curToken.Type { + case LPAREN: + parenNum += 1 + case RPAREN: + parenNum -= 1 + case ASSIGN: + builder.WriteString("==") + default: + builder.WriteString(p.curToken.Literal) + } + if parenNum <= 0 || p.peekToken.Type == COMMA || p.peekToken.Type == FROM || p.peekToken.Type == GROUP || p.peekToken.Type == LIMIT || p.peekToken.Type == HAVING { + break + } + // 读取下一个 Token + p.nextToken() + } + return &ExpressionLang{ + Val: builder.String(), + } +} + +// parseParenExpr 用于解析一个括号表达式，返回一个 ParenExpr 的实例 +func (p *parser) parseParenExpr() Expression { + // 创建一个 ast.ParenExpr 的实例 + expr := &ParenExpr{} + // 保存当前的 Token 作为左括号 + expr.Lparen = p.curToken + // 读取下一个 Token + p.nextToken() + // 解析括号内的表达式，调用 parseExpression 方法，赋值给 expr.X + expr.Expr = p.parseExpression() + // 读取下一个 Token + p.nextToken() + // 如果当前的 Token 不是右括号，就返回 nil + if p.curToken.Type != RPAREN { + return nil + } + // 保存当前的 Token 作为右括号 + expr.Rparen = p.curToken + // 返回 expr + return expr +} + +// parseGroupBy 用于解析分组的列或表达式，返回一个 Expression 的切片 +func (p *parser) parseGroupBy() []Expression { + // 创建一个空的 Expression 切片 + exprs := []Expression{} + // for 循环，直到遇到非列或表达式的 Token + for { + // 解析一个列或表达式，调用 parseExpression 方法，追加到 exprs 中 + expr := p.parseExpression() + exprs = append(exprs, expr) + // 如果下一个 Token 不是逗号，就跳出循环 + if p.peekToken.Type != COMMA { + break + } + // 读取下一个 Token + p.nextToken() + p.nextToken() + } + // 返回 exprs + return exprs +} diff --git a/rsql/parser_test.go b/rsql/parser_test.go new file mode 100644 index 0000000..661dbc8 --- /dev/null +++ b/rsql/parser_test.go @@ -0,0 +1,35 @@ +/* + * Copyright 2023 The RuleGo Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package rsql + +import ( + "bytes" + "fmt" + "os" + "testing" +) + +func TestNewParser(t *testing.T) { + sql := "select deviceId,avg(temperature+20/20,aa)/2 as aa from Input where deviceId='aa' OR deviceId='bb' group by deviceId,TumblingWindow(10m) ;" + newParser := NewParser(NewLexer(sql)) + + statement := newParser.ParseStatement() + buf := new(bytes.Buffer) + statement.Format(buf) + buf.WriteTo(os.Stdout) + fmt.Println(statement) +} diff --git a/rsql/token.go b/rsql/token.go new file mode 100644 index 0000000..43dac1b --- /dev/null +++ b/rsql/token.go @@ -0,0 +1,161 @@ +/* + * Copyright 2024 The RuleGo Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// token.go 定义了一些常量，表示不同的 Token 类型 +package rsql + +import "strings" + +// Token 是一个结构体，用于表示一个词法单元，包括类型和字面值 +type Token struct { + // Type 表示 Token 的类型，如 SELECT, CREATE, STRING, IDENT 等 + Type TokenType + // Literal 表示 Token 的字面值，如 "SELECT", "users", "'hello'" 等 + Literal string +} + +// TokenType 是一个字符串，用于表示 Token 的类型 +type TokenType string + +// 定义了一些常用的 Token 类型 +const ( + // 特殊类型 + ILLEGAL = "ILLEGAL" // 非法字符 + EOF = "EOF" // 文件结束 + + // 标识符和字面量 + IDENT = "IDENT" // 标识符，如 user, name 等 + STRING = "STRING" // 字符串字面量，如 'hello' 等 + + // 操作符和分隔符 + ASSIGN = "=" // 赋值符号 + PLUS = "+" // 加号 + MINUS = "-" // 减号 + ASTERISK = "*" // 星号，表示乘法或通配符 + SLASH = "/" // 斜杠，表示除法或注释 + COMMA = "," // 逗号，分隔符 + SEMICOLON = ";" // 分号，结束符 + LPAREN = "(" // 左括号 + RPAREN = ")" // 右括号 + LBRACE = "{" // 左大括号 + RBRACE = "}" // 右大括号 + LT = "<" // 小于号 + LE = "<=" // 小于等于号 + GT = ">" // 大于号 + GE = ">=" // 大于等于号 + EQ = "==" // 等于号 + NOT_EQ = "!=" // 不等于号 + NOT = "NOT" + AND = "AND" + OR = "OR" + + // 关键字 + SELECT = "SELECT" // 查询语句 + CREATE = "CREATE" // 创建语句 + TABLE = "TABLE" // 表 + FROM = "FROM" // 来源 + WHERE = "WHERE" // 条件 + GROUP = "GROUP" + BY = "BY" + ORDER = "ORDER" + LIMIT = "LIMIT" + HAVING = "HAVING" + ASC = "ASC" + DESC = "DESC" + OFFSET = "OFFSET" + AS = "AS" + COMMENT = "COMMENT" // 注释 + NUMBER = "NUMBER" // 数字 +) + +// keywords 是一个 map，用于存储所有的关键字及其对应的 Token 类型 +var keywords = map[string]TokenType{ + "SELECT": SELECT, + "CREATE": CREATE, + "TABLE": TABLE, + "FROM": FROM, + "GROUP": GROUP, + "BY": BY, + "ORDER": ORDER, + "LIMIT": LIMIT, + "HAVING": HAVING, + "ASC": ASC, + "DESC": DESC, + "OFFSET": OFFSET, + "WHERE": WHERE, + "AS": AS, + "NOT": NOT, + "AND": AND, + "OR": OR, +} + +// LookupIdent 用于根据给定的标识符，返回其对应的 Token 类型 +// 如果是一个关键字，就返回关键字的类型，否则就返回 IDENT 类型 +func LookupIdent(ident string) TokenType { + if tok, ok := keywords[strings.ToUpper(ident)]; ok { + return tok + } + return IDENT +} + +// 聚合函数 +var aggregateFunctions = map[string]string{ + "COUNT": "COUNT", + "SUM": "SUM", + "MAX": "MAX", + "MIN": "MIN", + "AVG": "AVG", + "STD": "STD", + "VAR": "VAR", + "FIRST": "FIRST", + "LAST": "LAST", + "TOP": "TOP", + "BOTTOM": "BOTTOM", +} + +// LookupIsAggregateFunc 用于判断是否是聚合函数 +func LookupIsAggregateFunc(ident string) bool { + if _, ok := aggregateFunctions[strings.ToUpper(ident)]; ok { + return true + } + return false +} + +type WindowType string + +const ( + NOT_WINDOW WindowType = "NotWindow" + TUMBLING_WINDOW = "TumblingWindow" + SLIDING_WINDOW = "SlidingWindow" + SESSION_WINDOW = "SessionWindow" + COUNT_WINDOW = "CountWindow" +) + +// 窗口函数 +var windowFunctions = map[string]WindowType{ + "TUMBLINGWINDOW": TUMBLING_WINDOW, + "SLIDINGWINDOW": SLIDING_WINDOW, + "SESSIONWINDOW": SESSION_WINDOW, + "COUNTWINDOW": COUNT_WINDOW, +} + +// LookupIsWindowFunc 用于判断是否是窗口函数 +func LookupIsWindowFunc(ident string) WindowType { + if windowType, ok := windowFunctions[strings.ToUpper(ident)]; ok { + return windowType + } + return NOT_WINDOW +} diff --git a/streamsql.go b/streamsql.go new file mode 100644 index 0000000..2f1e00c --- /dev/null +++ b/streamsql.go @@ -0,0 +1,24 @@ +/* + * Copyright 2024 The RuleGo Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package stream + +type Streamsql struct { +} + +func (c *Streamsql) OnMsg(inputJson string, metadata map[string]string) { + +} diff --git a/types/errors.go b/types/errors.go new file mode 100644 index 0000000..77bf39b --- /dev/null +++ b/types/errors.go @@ -0,0 +1,24 @@ +/* + * Copyright 2024 The RuleGo Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package types + +import "errors" + +var ( + ErrUnsupported = errors.New("unsupported operation") + ErrNotMatch = errors.New("not match") +) diff --git a/types/group_fields.go b/types/group_fields.go new file mode 100644 index 0000000..b55a305 --- /dev/null +++ b/types/group_fields.go @@ -0,0 +1,50 @@ +/* + * Copyright 2024 The RuleGo Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package types + +import "strings" + +// EmptyGroupFields 空的GroupFields +var EmptyGroupFields = NewGroupFields("") + +type GroupFields string + +func NewGroupFields(keys ...string) GroupFields { + return GroupFields(strings.Join(keys, ",")) +} + +func (g GroupFields) ToList() []string { + return strings.Split(string(g), ",") +} + +func (g GroupFields) ToMap() map[string]struct{} { + keys := g.ToList() + keysMap := make(map[string]struct{}) + for _, key := range keys { + keysMap[key] = struct{}{} + } + return keysMap +} + +// EmptyGroupValues 空的GroupValues +var EmptyGroupValues = NewGroupValues("") + +type GroupValues GroupFields + +func NewGroupValues(keys ...string) GroupValues { + return GroupValues(strings.Join(keys, ",")) +} diff --git a/types/msg.go b/types/msg.go new file mode 100644 index 0000000..a2776e6 --- /dev/null +++ b/types/msg.go @@ -0,0 +1,167 @@ +/* + * Copyright 2024 The RuleGo Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package types + +import ( + "encoding/json" + "time" +) + +// DataType 消息数据类型 +type DataType string + +const ( + JSON = DataType("JSON") + TEXT = DataType("TEXT") + BINARY = DataType("BINARY") + INTERFACE = DataType("interface") +) + +// Metadata 消息元数据 +type Metadata map[string]string + +// NewMetadata 创建一个新的消息元数据实例 +func NewMetadata() Metadata { + return make(Metadata) +} + +// BuildMetadata 通过map，创建一个新的消息元数据实例 +func BuildMetadata(data Metadata) Metadata { + metadata := make(Metadata) + for k, v := range data { + metadata[k] = v + } + return metadata +} + +// Copy 复制 +func (md Metadata) Copy() Metadata { + return BuildMetadata(md) +} + +// Has 是否存在某个key +func (md Metadata) Has(key string) bool { + _, ok := md[key] + return ok +} + +// GetValue 通过key获取值 +func (md Metadata) GetValue(key string) string { + v, _ := md[key] + return v +} + +// PutValue 设置值 +func (md Metadata) PutValue(key, value string) { + if key != "" { + md[key] = value + } +} + +// Values 获取所有值 +func (md Metadata) Values() map[string]string { + return md +} + +// Msg 消息 +type Msg struct { + // 消息时间戳 + Ts int64 `json:"ts"` + //数据类型 + DataType DataType `json:"dataType"` + //消息内容 + Data IData `json:"data"` + //消息元数据 + Metadata Metadata `json:"metadata"` +} + +func NewJsonMsg(ts int64, metaData Metadata, data string) Msg { + return newMsg(ts, JSON, metaData, JsonData(data)) +} +func NewTextMsg(ts int64, metaData Metadata, data string) Msg { + return newMsg(ts, TEXT, metaData, TextData(data)) +} +func NewBinaryMsg(ts int64, metaData Metadata, data []byte) Msg { + return newMsg(ts, BINARY, metaData, BinaryData(data)) +} +func NewInterfaceMsg(ts int64, metaData Metadata, data map[string]any) Msg { + return newMsg(ts, INTERFACE, metaData, InterfaceData(data)) +} + +func newMsg(ts int64, dataType DataType, metaData Metadata, data IData) Msg { + if ts <= 0 { + ts = time.Now().UnixMilli() + } + return Msg{ + Ts: ts, + DataType: dataType, + Data: data, + Metadata: metaData, + } +} + +// Copy 复制 +func (m *Msg) Copy() Msg { + return newMsg(m.Ts, m.DataType, m.Metadata.Copy(), m.Data) +} + +type IData interface { + Decode() (any, error) + DecodeAsMap() (map[string]any, error) +} + +type JsonData string + +func (d JsonData) Decode() (any, error) { + return d.DecodeAsMap() +} + +func (d JsonData) DecodeAsMap() (map[string]any, error) { + var result map[string]any + err := json.Unmarshal([]byte(d), &result) + return result, err +} + +type InterfaceData map[string]any + +func (d InterfaceData) Decode() (any, error) { + return d, nil +} + +func (d InterfaceData) DecodeAsMap() (map[string]any, error) { + return d, nil +} + +type TextData string + +func (d TextData) Decode() (any, error) { + return d, nil +} + +func (d TextData) DecodeAsMap() (map[string]any, error) { + return nil, ErrUnsupported +} + +type BinaryData []byte + +func (d BinaryData) Decode() (any, error) { + return string(d), ErrUnsupported +} + +func (d BinaryData) DecodeAsMap() (map[string]any, error) { + return nil, ErrUnsupported +} diff --git a/types/types.go b/types/types.go new file mode 100644 index 0000000..3e9351b --- /dev/null +++ b/types/types.go @@ -0,0 +1,122 @@ +/* + * Copyright 2024 The RuleGo Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package types + +import ( + "context" + "github.com/rulego/streamsql/dataset" + "github.com/rulego/streamsql/rsql" + "time" +) + +type Window interface { + // FieldName 获取聚合运算字段名称 + FieldName() string + // LastData 获取最后一条数据 + LastData() (float64, bool) + // StartTime 获取窗口开始时间 + StartTime() time.Time + // EndTime 获取窗口结束时间 + EndTime() time.Time + // Add 添加数据 + Add(data float64) + //Archive 保存数据 + Archive() + Stop() + ////Release 释放空间 + //Release() +} + +//// StreamWindow 流式计算window,不保存原始数据 +//type StreamWindow interface { +// Window +//} + +// StreamSqlContext 上下文 +type StreamSqlContext interface { + context.Context +} + +type StreamSqlOperatorContext interface { + StreamSqlContext + AddWindow(groupValues GroupValues, window Window) + CreteWindowObserver() WindowObserver + //GetWindow 获取窗口实例 + GetWindow(groupValues GroupValues) Window + IsInitWindow(groupValues GroupValues) bool + AddFieldAggregateValue(groupValues GroupValues, fieldId string, value float64) + GetFieldAggregateValue(groupValues GroupValues, fieldId string) []float64 + SetGroupByKey(groupByKey GroupFields) + GetRow(groupValues GroupValues) (*dataset.Row, bool) + AddColumn(groupValues GroupValues, kv dataset.KeyValue) + GetColumn(groupValues GroupValues, key dataset.Key) (dataset.KeyValue, bool) +} + +type SelectStreamSqlContext interface { + StreamSqlOperatorContext + InputAsMap() map[string]interface{} + RawInput() Msg + SetCurrentGroupValues(groupValues GroupValues) + GetCurrentGroupValues() GroupValues +} + +// WindowObserver 窗口事件观察者 +type WindowObserver struct { + // AddHandler 窗口添加数据事件 + AddHandler func(context StreamSqlOperatorContext, data float64) + ////FullHandler 空间满事件 + //FullHandler func(context StreamSqlContext, dataList []float64) + //ArchiveHandler 清除原始数据，观察者需要保存中间过程 + ArchiveHandler func(context StreamSqlOperatorContext, dataList []float64) + //StartHandler 窗口开始事件 + StartHandler func(context StreamSqlOperatorContext) + //EndHandler 窗口结束事件 + EndHandler func(context StreamSqlOperatorContext, dataList []float64) +} + +// Operator 操作器接口 +type Operator interface { + Init(context StreamSqlContext) error + // Apply 执行 + Apply(context StreamSqlContext) error +} + +type AggregateOperator interface { + Operator + AddHandler(context StreamSqlOperatorContext, data float64) + ArchiveHandler(context StreamSqlOperatorContext, dataList []float64) + StartHandler(context StreamSqlOperatorContext) + EndHandler(context StreamSqlOperatorContext, dataList []float64) +} + +// LogicalPlan 逻辑计划接口 +type LogicalPlan interface { + New() LogicalPlan + Plan(statement rsql.Statement) error + Init(context StreamSqlContext) error + Apply(context StreamSqlContext) error + Explain() string + Type() string + // AllOperators 获取所有操作器 + AllOperators() []Operator + // AggregateOperators 获取所有聚合操作器 + AggregateOperators() []AggregateOperator +} + +type Collector interface { + Collect(ctx context.Context, msg Msg) error +} diff --git a/utils/attribute/attribute.go b/utils/attribute/attribute.go new file mode 100644 index 0000000..509375d --- /dev/null +++ b/utils/attribute/attribute.go @@ -0,0 +1,109 @@ +/* + * Copyright 2024 The RuleGo Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package attribute + +import ( + "reflect" +) + +// BoolSliceValue converts a bool slice into an array with same elements as slice. +func BoolSliceValue(v []bool) interface{} { + var zero bool + cp := reflect.New(reflect.ArrayOf(len(v), reflect.TypeOf(zero))) + copy(cp.Elem().Slice(0, len(v)).Interface().([]bool), v) + return cp.Elem().Interface() +} + +// Int64SliceValue converts an int64 slice into an array with same elements as slice. +func Int64SliceValue(v []int64) interface{} { + var zero int64 + cp := reflect.New(reflect.ArrayOf(len(v), reflect.TypeOf(zero))) + copy(cp.Elem().Slice(0, len(v)).Interface().([]int64), v) + return cp.Elem().Interface() +} + +// Float64SliceValue converts a float64 slice into an array with same elements as slice. +func Float64SliceValue(v []float64) interface{} { + var zero float64 + cp := reflect.New(reflect.ArrayOf(len(v), reflect.TypeOf(zero))) + copy(cp.Elem().Slice(0, len(v)).Interface().([]float64), v) + return cp.Elem().Interface() +} + +// StringSliceValue converts a string slice into an array with same elements as slice. +func StringSliceValue(v []string) interface{} { + var zero string + cp := reflect.New(reflect.ArrayOf(len(v), reflect.TypeOf(zero))) + copy(cp.Elem().Slice(0, len(v)).Interface().([]string), v) + return cp.Elem().Interface() +} + +// AsBoolSlice converts a bool array into a slice into with same elements as array. +func AsBoolSlice(v interface{}) []bool { + rv := reflect.ValueOf(v) + if rv.Type().Kind() != reflect.Array { + return nil + } + var zero bool + correctLen := rv.Len() + correctType := reflect.ArrayOf(correctLen, reflect.TypeOf(zero)) + cpy := reflect.New(correctType) + _ = reflect.Copy(cpy.Elem(), rv) + return cpy.Elem().Slice(0, correctLen).Interface().([]bool) +} + +// AsInt64Slice converts an int64 array into a slice into with same elements as array. +func AsInt64Slice(v interface{}) []int64 { + rv := reflect.ValueOf(v) + if rv.Type().Kind() != reflect.Array { + return nil + } + var zero int64 + correctLen := rv.Len() + correctType := reflect.ArrayOf(correctLen, reflect.TypeOf(zero)) + cpy := reflect.New(correctType) + _ = reflect.Copy(cpy.Elem(), rv) + return cpy.Elem().Slice(0, correctLen).Interface().([]int64) +} + +// AsFloat64Slice converts a float64 array into a slice into with same elements as array. +func AsFloat64Slice(v interface{}) []float64 { + rv := reflect.ValueOf(v) + if rv.Type().Kind() != reflect.Array { + return nil + } + var zero float64 + correctLen := rv.Len() + correctType := reflect.ArrayOf(correctLen, reflect.TypeOf(zero)) + cpy := reflect.New(correctType) + _ = reflect.Copy(cpy.Elem(), rv) + return cpy.Elem().Slice(0, correctLen).Interface().([]float64) +} + +// AsStringSlice converts a string array into a slice into with same elements as array. +func AsStringSlice(v interface{}) []string { + rv := reflect.ValueOf(v) + if rv.Type().Kind() != reflect.Array { + return nil + } + var zero string + correctLen := rv.Len() + correctType := reflect.ArrayOf(correctLen, reflect.TypeOf(zero)) + cpy := reflect.New(correctType) + _ = reflect.Copy(cpy.Elem(), rv) + return cpy.Elem().Slice(0, correctLen).Interface().([]string) +} diff --git a/utils/cast/cast.go b/utils/cast/cast.go new file mode 100644 index 0000000..8598e0c --- /dev/null +++ b/utils/cast/cast.go @@ -0,0 +1,63 @@ +/* + * Copyright 2024 The RuleGo Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cast + +import ( + "fmt" + "strconv" +) + +func ToFloat(x any) float64 { + switch x := x.(type) { + case float32: + return float64(x) + case float64: + return x + case int: + return float64(x) + case int8: + return float64(x) + case int16: + return float64(x) + case int32: + return float64(x) + case int64: + return float64(x) + case uint: + return float64(x) + case uint8: + return float64(x) + case uint16: + return float64(x) + case uint32: + return float64(x) + case uint64: + return float64(x) + case string: + f, err := strconv.ParseFloat(x, 64) + if err != nil { + panic(fmt.Sprintf("invalid operation: float(%s)", x)) + } + return f + default: + panic(fmt.Sprintf("invalid operation: float(%T)", x)) + } +} + +func ToString(arg any) string { + return fmt.Sprintf("%v", arg) +} diff --git a/utils/compress/compress.go b/utils/compress/compress.go new file mode 100644 index 0000000..6ba13d3 --- /dev/null +++ b/utils/compress/compress.go @@ -0,0 +1,72 @@ +/* + * Copyright 2024 The RuleGo Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package compress + +import ( + "bytes" + "encoding/binary" + "fmt" + "log" + + "github.com/golang/snappy" +) + +// float64ToByte converts a slice of float64 to a slice of byte +func float64ToByte(data []float64) []byte { + buf := new(bytes.Buffer) + err := binary.Write(buf, binary.LittleEndian, data) + if err != nil { + log.Fatal(err) + } + return buf.Bytes() +} + +// byteToFloat64 converts a slice of byte to a slice of float64 +func byteToFloat64(data []byte) []float64 { + buf := bytes.NewReader(data) + var result []float64 + err := binary.Read(buf, binary.LittleEndian, &result) + if err != nil { + log.Fatal(err) + } + return result +} + +func main() { + // create a slice of float64 + data := []float64{3.14, 3.14, 3.15, 3.16, 3.17} + fmt.Println("Original data:", data) + + // convert it to []byte + dataByte := float64ToByte(data) + fmt.Println("Data as []byte:", dataByte) + + // compress it using snappy + compressed := snappy.Encode(nil, dataByte) + fmt.Println("Compressed data:", compressed) + + // decompress it using snappy + decompressed, err := snappy.Decode(nil, compressed) + if err != nil { + log.Fatal(err) + } + fmt.Println("Decompressed data:", decompressed) + + // convert it back to []float64 + dataFloat64 := byteToFloat64(decompressed) + fmt.Println("Data as []float64:", dataFloat64) +} diff --git a/utils/compress/rawhelpers.go b/utils/compress/rawhelpers.go new file mode 100644 index 0000000..35ab681 --- /dev/null +++ b/utils/compress/rawhelpers.go @@ -0,0 +1,41 @@ +package compress + +import ( + "math" + "unsafe" +) + +func BoolToRaw(b bool) uint64 { // nolint:revive // b is not a control flag. + if b { + return 1 + } + return 0 +} + +func RawToBool(r uint64) bool { + return r != 0 +} + +func Int64ToRaw(i int64) uint64 { + return uint64(i) +} + +func RawToInt64(r uint64) int64 { + return int64(r) +} + +func Float64ToRaw(f float64) uint64 { + return math.Float64bits(f) +} + +func RawToFloat64(r uint64) float64 { + return math.Float64frombits(r) +} + +func RawPtrToFloat64Ptr(r *uint64) *float64 { + return (*float64)(unsafe.Pointer(r)) +} + +func RawPtrToInt64Ptr(r *uint64) *int64 { + return (*int64)(unsafe.Pointer(r)) +} diff --git a/utils/expr/expr_test.go b/utils/expr/expr_test.go new file mode 100644 index 0000000..9617d73 --- /dev/null +++ b/utils/expr/expr_test.go @@ -0,0 +1,85 @@ +/* + * Copyright 2023 The RuleGo Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package rsql + +import ( + "fmt" + "github.com/expr-lang/expr" + "github.com/montanaflynn/stats" + "testing" + "time" +) + +//type Env struct { +// Posts []Post `expr:"posts"` +//} +// +//func (Env) Format(t time.Time) string { +// return t.Format(time.RFC822) +//} + +type Post struct { + Name string + Body string + Date time.Time +} + +func TestExpr(t *testing.T) { + + env := map[string]interface{}{ + "greet": "Hello, %v!", + "names": []string{"world", "you"}, + "sprintf": fmt.Sprintf, + "name": "WORLd", + "name2": "我们", + "name3": "5oiR5Lus", + "foo": 100.2, + "bar": 2004, + "post": Post{Name: "lala", Body: "aa"}, + } + //code := `(foo + bar)/2` + //code := `duration("1m")` + code := `fromBase64(name3)` + program, err := expr.Compile(code, expr.Env(env), expr.AllowUndefinedVariables()) + if err != nil { + panic(err) + } + + output, err := expr.Run(program, env) + if err != nil { + panic(err) + } + fmt.Println(output) +} + +func TestStat(t *testing.T) { + // start with some source data to use + data := []float64{1.0, 2.1, 3.2, 4.823, 4.1, 5.8} + //随机生成100W数据 + for i := 0; i < 10000000; i++ { + data = append(data, float64(i)) + } + // you could also use different types like this + // data := stats.LoadRawData([]int{1, 2, 3, 4, 5}) + // data := stats.LoadRawData([]interface{}{1.1, "2", 3}) + // etc... + //计算耗时 + start := time.Now() + median, _ := stats.StandardDeviation(data) + fmt.Printf("%f", median) + fmt.Println("\n", time.Since(start)) +} diff --git a/utils/queue/queue.go b/utils/queue/queue.go new file mode 100644 index 0000000..d93b090 --- /dev/null +++ b/utils/queue/queue.go @@ -0,0 +1,196 @@ +/* + * Copyright 2024 The RuleGo Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package queue + +import ( + "errors" + "fmt" + "sync/atomic" +) + +type snapshot struct { + head int32 // 队首的索引，使用int32类型，方便原子操作 + tail int32 // 队尾的索引，使用int32类型，方便原子操作 + count int32 // 队列中元素的个数，使用int32类型，方便原子操作 +} +type Queue struct { + data []float64 // 存储数据的切片 + head int32 // 队首的索引，使用int32类型，方便原子操作 + tail int32 // 队尾的索引，使用int32类型，方便原子操作 + cap int32 // 队列的大小，使用int32类型，方便原子操作 + count int32 // 队列中元素的个数，使用int32类型，方便原子操作 + buffer []float64 // 缓冲切片，用于复用 +} + +// NewCircleQueue 创建一个指定大小的环形队列 +func NewCircleQueue(size int) *Queue { + return &Queue{ + data: make([]float64, size), + head: 0, + tail: 0, + cap: int32(size), + count: 0, + buffer: make([]float64, size), // 初始化缓冲切片 + } +} + +// IsEmpty 判断队列是否为空 +func (q *Queue) IsEmpty() bool { + return atomic.LoadInt32(&q.count) == 0 // 原子读取count的值 +} + +// IsFull 判断队列是否已满 +func (q *Queue) IsFull() bool { + return atomic.LoadInt32(&q.count) == q.cap // 原子读取count和size的值 +} + +// Push 向队尾添加一个元素，如果队列已满，返回错误 +func (q *Queue) Push(x float64) error { + for { + if q.IsFull() { + // 队列满时，返回错误 + return errors.New("queue is full") + } + tail := atomic.LoadInt32(&q.tail) // 原子读取tail的值 + next := (tail + 1) % q.cap // 计算下一个tail的值 + if atomic.CompareAndSwapInt32(&q.tail, tail, next) { // 原子比较并交换tail的值，如果成功则表示没有其他协程修改过tail + q.data[tail] = x // 写入数据 + atomic.AddInt32(&q.count, 1) // 原子增加count的值 + return nil // 返回nil表示成功 + } + // 否则，表示有其他协程修改了tail，重新尝试 + } +} + +// Pop 从队首删除一个元素，并返回它 +func (q *Queue) Pop() (float64, bool) { + for { + if q.IsEmpty() { + // 队列空时，返回错误 + return 0, false + } + head := atomic.LoadInt32(&q.head) // 原子读取head的值 + next := (head + 1) % q.cap // 计算下一个head的值 + if atomic.CompareAndSwapInt32(&q.head, head, next) { // 原子比较并交换head的值，如果成功则表示没有其他协程修改过head + x := q.data[head] // 读取数据 + atomic.AddInt32(&q.count, -1) // 原子减少count的值 + return x, true // 返回数据和成功标志 + } + // 否则，表示有其他协程修改了head，重新尝试 + } +} + +// Back 返回队尾元素，不出队 +func (q *Queue) Back() (float64, bool) { + if q.IsEmpty() { + // 队列空时，返回错误 + return 0, false + } + // 队尾元素的索引是 (q.tail - 1 + q.cap) % q.cap + tail := atomic.LoadInt32(&q.tail) // 原子读取tail的值 + x := q.data[(tail-1+q.cap)%q.cap] + return x, true +} + +// PopAll 返回并删除队列中的所有元素，并重置队列的状态 +func (q *Queue) PopAll() []float64 { + if q.IsEmpty() { + // 队列空时，返回空切片 + return nil + } + // 复用缓冲切片，避免内存浪费 + slice := q.buffer + if q.head < q.tail { + // 队列中的元素是连续的，直接截取切片 + slice = slice[:q.tail-q.head] + copy(slice, q.data[q.head:q.tail]) + } else { + // 队列中的元素是环形的，需要拼接两部分切片 + slice = slice[:q.cap-q.head+q.tail] + copy(slice, append(q.data[q.head:], q.data[:q.tail]...)) + } + + //重置队列的状态 + q.Reset() + return slice +} + +// RemoveRange 删除队列中的指定范围元素 +func (q *Queue) RemoveRange(head, tail int32) { + if q.IsEmpty() { + return + } + var sliceLen int32 + if head < tail { + sliceLen = tail - head + } else { + sliceLen = q.cap - head + tail + } + + // 原子重置head，tail和count的值 + atomic.StoreInt32(&q.head, tail) + atomic.StoreInt32(&q.count, q.count-sliceLen) +} + +// Reset 清空队列中的所有元素，但不释放内存空间，只是重置队列的状态 +func (q *Queue) Reset() { + // 原子重置head，tail和count的值 + atomic.StoreInt32(&q.head, 0) + atomic.StoreInt32(&q.tail, 0) + atomic.StoreInt32(&q.count, 0) +} + +func (q *Queue) Count() int32 { + return q.count +} + +// Print 打印队列中的元素 +func (q *Queue) Print() { + fmt.Println("队列中的元素：") + for i := int32(0); i < atomic.LoadInt32(&q.count); i++ { // 原子读取count的值 + fmt.Printf("%f ", q.data[(atomic.LoadInt32(&q.head)+i)%q.cap]) // 原子读取head的值 + } + fmt.Println() +} + +func main() { + // 创建一个大小为 5 的环形队列 + q := NewCircleQueue(5) + // 向队列中添加元素 + q.Push(1.1) + q.Push(2.2) + q.Push(3.3) + q.Push(4.4) + q.Push(5.5) + // 打印队列中的元素 + q.Print() + // 从队列中删除元素 + x, ok := q.Pop() + if ok { + fmt.Println("删除的元素：", x) + } + // 打印队列中的元素 + q.Print() + // 向队列中添加元素 + q.Push(6) + // 打印队列中的元素 + q.Print() + + q.Push(7) + q.Print() + fmt.Println(q.Back()) +} diff --git a/window/count_window.go b/window/count_window.go new file mode 100644 index 0000000..08e37d9 --- /dev/null +++ b/window/count_window.go @@ -0,0 +1,148 @@ +/* + * Copyright 2024 The RuleGo Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package window + +import ( + "github.com/rulego/streamsql/types" + queue2 "github.com/rulego/streamsql/utils/queue" + "sync" + "sync/atomic" + "time" +) + +// CountWindow is a structure that represents a tumbling window +type CountWindow struct { + context types.SelectStreamSqlContext + observer types.WindowObserver + windowSize int32 // the size of the window + count int32 + lastTime time.Time // the last time the window was updated + fieldName string // the name of the field that is being aggregated + //maxCapacity 最大容量 + maxCapacity int + queue *queue2.Queue + startTime time.Time + endTime time.Time + locker sync.Mutex + ticker *time.Ticker + //退出标志 + quit chan struct{} +} + +// NewCountWindow creates a new rolling window with the given size +func NewCountWindow(fieldName string, windowSize int32, observer types.WindowObserver) *CountWindow { + maxCapacity := 100000 + w := &CountWindow{ + fieldName: fieldName, + windowSize: windowSize, + lastTime: time.Now(), + maxCapacity: maxCapacity, + queue: queue2.NewCircleQueue(maxCapacity), + observer: observer, + } + + //开始新的窗口 + w.start() + return w +} + +// 检查是否需要是下一个窗口 +func (w *CountWindow) checkNextWindow() { + w.locker.Lock() + defer w.locker.Unlock() + if atomic.LoadInt32(&w.count) >= w.windowSize { + //结束当前窗口 + w.end() + //开始新的窗口 + w.start() + } +} + +// 开始窗口事件 +func (w *CountWindow) start() { + w.startTime = time.Now() + atomic.StoreInt32(&w.count, 0) + w.queue.Reset() + + if w.observer.StartHandler != nil { + w.observer.StartHandler(w.context) + } +} + +// 结束窗口事件 +func (w *CountWindow) end() { + w.endTime = time.Now() + if w.observer.EndHandler != nil { + w.observer.EndHandler(w.context, w.queue.PopAll()) + } +} + +// 队列满，触发full事件，并重置队列 +func (w *CountWindow) full() { + if w.observer.ArchiveHandler != nil { + w.observer.ArchiveHandler(w.context, w.queue.PopAll()) + } + //重置队列 + w.queue.Reset() +} + +// Add 添加数据 +func (w *CountWindow) Add(data float64) { + if atomic.LoadInt32(&w.count) >= w.windowSize { + w.checkNextWindow() + } + if w.queue.IsFull() { + w.full() + } + atomic.AddInt32(&w.count, 1) + _ = w.queue.Push(data) + if w.observer.AddHandler != nil { + w.observer.AddHandler(w.context, data) + } +} + +// FieldName 获取聚合运算字段名称 +func (w *CountWindow) FieldName() string { + return w.fieldName +} + +// LastData 获取最后一条数据 +func (w *CountWindow) LastData() (float64, bool) { + return w.queue.Back() +} + +// StartTime 获取窗口开始时间 +func (w *CountWindow) StartTime() time.Time { + return w.startTime +} + +// EndTime 获取窗口结束时间 +func (w *CountWindow) EndTime() time.Time { + return w.endTime +} + +// Archive 保存数据 +func (w *CountWindow) Archive() { + +} +func (w *CountWindow) Stop() { + + if w.ticker != nil { + w.ticker.Stop() + } + w.quit <- struct{}{} +} diff --git a/window/session_window.go b/window/session_window.go new file mode 100644 index 0000000..eda0119 --- /dev/null +++ b/window/session_window.go @@ -0,0 +1,160 @@ +/* + * Copyright 2024 The RuleGo Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package window + +import ( + "github.com/rulego/streamsql/types" + queue2 "github.com/rulego/streamsql/utils/queue" + "sync" + "time" +) + +// SessionWindow is a structure that represents a tumbling window +type SessionWindow struct { + context types.SelectStreamSqlContext + observer types.WindowObserver + windowSize time.Duration // the size of the window + timeout time.Duration + lastTime time.Time // the last time the window was updated + fieldName string // the name of the field that is being aggregated + //maxCapacity 最大容量 + maxCapacity int + queue *queue2.Queue + startTime time.Time + endTime time.Time + locker sync.Mutex + ticker *time.Ticker + //退出标志 + quit chan struct{} +} + +// NewSessionWindow creates a new rolling window with the given size +func NewSessionWindow(fieldName string, windowSize time.Duration, observer types.WindowObserver) *SessionWindow { + maxCapacity := 100000 + w := &SessionWindow{ + fieldName: fieldName, + windowSize: windowSize, + lastTime: time.Now(), + maxCapacity: maxCapacity, + queue: queue2.NewCircleQueue(maxCapacity), + observer: observer, + } + + //开始新的窗口 + w.start() + // 创建一个通道，用于通知ticker的结束 + w.quit = make(chan struct{}) + //开始定时器 + w.ticker = time.NewTicker(windowSize) + go func() { + for { + select { + case <-w.ticker.C: + w.checkNextWindow() + case <-w.quit: + return + } + } + }() + return w +} + +// 检查是否需要是下一个窗口 +func (w *SessionWindow) checkNextWindow() { + w.locker.Lock() + defer w.locker.Unlock() + if time.Now().Sub(w.startTime) > w.windowSize { + //结束当前窗口 + w.end() + //开始新的窗口 + w.start() + } +} + +// 开始窗口事件 +func (w *SessionWindow) start() { + w.startTime = time.Now() + w.queue.Reset() + + if w.observer.StartHandler != nil { + w.observer.StartHandler(w.context) + } +} + +// 结束窗口事件 +func (w *SessionWindow) end() { + w.endTime = time.Now() + if w.observer.EndHandler != nil { + w.observer.EndHandler(w.context, w.queue.PopAll()) + } +} + +// 队列满，触发full事件，并重置队列 +func (w *SessionWindow) full() { + if w.observer.ArchiveHandler != nil { + w.observer.ArchiveHandler(w.context, w.queue.PopAll()) + } + //重置队列 + w.queue.Reset() +} + +// Add 添加数据 +func (w *SessionWindow) Add(data float64) { + if time.Now().Sub(w.startTime) > w.windowSize { + w.checkNextWindow() + } + if w.queue.IsFull() { + w.full() + } + w.lastTime = time.Now() + _ = w.queue.Push(data) + if w.observer.AddHandler != nil { + w.observer.AddHandler(w.context, data) + } +} + +// FieldName 获取聚合运算字段名称 +func (w *SessionWindow) FieldName() string { + return w.fieldName +} + +// LastData 获取最后一条数据 +func (w *SessionWindow) LastData() (float64, bool) { + return w.queue.Back() +} + +// StartTime 获取窗口开始时间 +func (w *SessionWindow) StartTime() time.Time { + return w.startTime +} + +// EndTime 获取窗口结束时间 +func (w *SessionWindow) EndTime() time.Time { + return w.endTime +} + +// Archive 保存数据 +func (w *SessionWindow) Archive() { + +} +func (w *SessionWindow) Stop() { + + if w.ticker != nil { + w.ticker.Stop() + } + w.quit <- struct{}{} +} diff --git a/window/sliding_window.go b/window/sliding_window.go new file mode 100644 index 0000000..47c73d2 --- /dev/null +++ b/window/sliding_window.go @@ -0,0 +1,175 @@ +/* + * Copyright 2024 The RuleGo Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package window + +import ( + "github.com/rulego/streamsql/types" + queue2 "github.com/rulego/streamsql/utils/queue" + "sync" + "time" +) + +// SidingWindow 滑动窗口 +type SidingWindow struct { + context types.SelectStreamSqlContext + observer types.WindowObserver + windowSize time.Duration // the size of the window + slide time.Duration // the slide of the window + + fieldName string // the name of the field that is being aggregated + //maxCapacity 最大容量 + maxCapacity int + queue *queue2.Queue + startTime time.Time + endTime time.Time + locker sync.Mutex + windowTimer *time.Timer + slideTicker *time.Ticker + // + slideSnapshot int32 + //退出标志 + quit chan struct{} +} + +// NewSlidingWindow creates a new siding window with the given size and slide +func NewSlidingWindow(fieldName string, windowSize, slide time.Duration, observer types.WindowObserver) *SidingWindow { + maxCapacity := 100000 + w := &SidingWindow{ + fieldName: fieldName, + observer: observer, + windowSize: windowSize, + slide: slide, + maxCapacity: maxCapacity, + queue: queue2.NewCircleQueue(maxCapacity), + } + + //开始新的窗口 + w.start() + // 创建一个通道，用于通知ticker的结束 + w.quit = make(chan struct{}) + + w.windowTimer = time.AfterFunc(windowSize, func() { + + }) + //时间窗口定时器 + //w.windowTicker = time.NewTicker(windowSize) + w.slideTicker = time.NewTicker(slide) + go func() { + for { + select { + //case <-w.windowTicker.C: + // w.checkNextWindow() + case <-w.slideTicker.C: + w.checkNextWindow() + case <-w.quit: + return + } + } + }() + return w +} + +// 清理滑动窗口队列元素 +func (w *SidingWindow) slideClean() { + //w.queue.RemoveRange(w.queue.head, w.queue.tail) +} + +// 检查是否需要是下一个窗口 +func (w *SidingWindow) checkNextWindow() { + w.locker.Lock() + defer w.locker.Unlock() + if time.Now().Sub(w.startTime) > w.windowSize { + //结束当前窗口 + w.end() + //开始新的窗口 + w.start() + } +} + +// 开始窗口事件 +func (w *SidingWindow) start() { + w.startTime = time.Now() + w.queue.Reset() + + if w.observer.StartHandler != nil { + w.observer.StartHandler(w.context) + } +} + +// 结束窗口事件 +func (w *SidingWindow) end() { + w.endTime = time.Now() + if w.observer.EndHandler != nil { + w.observer.EndHandler(w.context, w.queue.PopAll()) + } +} + +// 队列满，触发full事件，并重置队列 +func (w *SidingWindow) full() { + if w.observer.ArchiveHandler != nil { + w.observer.ArchiveHandler(w.context, w.queue.PopAll()) + } + //重置队列 + w.queue.Reset() +} + +// Add 添加数据 +func (w *SidingWindow) Add(data float64) { + if time.Now().Sub(w.startTime) > w.windowSize { + w.checkNextWindow() + } + if w.queue.IsFull() { + w.full() + } + + _ = w.queue.Push(data) + if w.observer.AddHandler != nil { + w.observer.AddHandler(w.context, data) + } +} + +// FieldName 获取聚合运算字段名称 +func (w *SidingWindow) FieldName() string { + return w.fieldName +} + +// LastData 获取最后一条数据 +func (w *SidingWindow) LastData() (float64, bool) { + return w.queue.Back() +} + +// StartTime 获取窗口开始时间 +func (w *SidingWindow) StartTime() time.Time { + return w.startTime +} + +// EndTime 获取窗口结束时间 +func (w *SidingWindow) EndTime() time.Time { + return w.endTime +} + +// Archive 保存数据 +func (w *SidingWindow) Archive() { + +} +func (w *SidingWindow) Stop() { + + //if w.windowTicker != nil { + // w.windowTicker.Stop() + //} + w.quit <- struct{}{} +} diff --git a/window/tumbling_window.go b/window/tumbling_window.go new file mode 100644 index 0000000..dbb2c9a --- /dev/null +++ b/window/tumbling_window.go @@ -0,0 +1,166 @@ +/* + * Copyright 2024 The RuleGo Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package window + +import ( + "github.com/rulego/streamsql/types" + queue2 "github.com/rulego/streamsql/utils/queue" + "sync" + "time" +) + +// TumblingWindow is a structure that represents a tumbling window +type TumblingWindow struct { + context types.StreamSqlOperatorContext + observer types.WindowObserver + windowSize time.Duration // the size of the window + + lastTime time.Time // the last time the window was updated + fieldName string // the name of the field that is being aggregated + //maxCapacity 最大容量 + maxCapacity int + queue *queue2.Queue + startTime time.Time + endTime time.Time + locker sync.Mutex + ticker *time.Ticker + //退出标志 + quit chan struct{} +} + +// NewTumblingWindow creates a new rolling window with the given size +func NewTumblingWindow(context types.StreamSqlOperatorContext, fieldName string, windowSize time.Duration, observer types.WindowObserver) *TumblingWindow { + maxCapacity := 100000 + w := &TumblingWindow{ + context: context, + fieldName: fieldName, + windowSize: windowSize, + lastTime: time.Now(), + maxCapacity: maxCapacity, + queue: queue2.NewCircleQueue(maxCapacity), + observer: observer, + } + + //开始新的窗口 + w.start() + // 创建一个通道，用于通知ticker的结束 + w.quit = make(chan struct{}) + //开始定时器 + w.ticker = time.NewTicker(windowSize) + go func() { + for { + select { + case <-w.ticker.C: + w.checkNextWindow() + case <-w.quit: + return + } + } + }() + return w +} + +// 检查是否需要是下一个窗口 +func (w *TumblingWindow) checkNextWindow() { + //w.locker.Lock() + //defer w.locker.Unlock() + if time.Now().Sub(w.startTime) > w.windowSize { + //结束当前窗口 + w.end() + //开始新的窗口 + w.start() + } +} + +// 开始窗口事件 +func (w *TumblingWindow) start() { + w.locker.Lock() + w.startTime = time.Now() + w.queue.Reset() + w.locker.Unlock() + if w.observer.StartHandler != nil { + w.observer.StartHandler(w.context) + } +} + +// 结束窗口事件 +func (w *TumblingWindow) end() { + w.endTime = time.Now() + if w.observer.EndHandler != nil { + w.locker.Lock() + values := w.queue.PopAll() + w.locker.Unlock() + + w.observer.EndHandler(w.context, values) + } +} + +// 队列满，触发full事件，并重置队列 +func (w *TumblingWindow) full() { + if w.observer.ArchiveHandler != nil { + w.observer.ArchiveHandler(w.context, w.queue.PopAll()) + } + //重置队列 + w.queue.Reset() +} + +// Add 添加数据 +func (w *TumblingWindow) Add(data float64) { + if time.Now().Sub(w.startTime) > w.windowSize { + w.checkNextWindow() + } + if w.queue.IsFull() { + w.full() + } + + _ = w.queue.Push(data) + if w.observer.AddHandler != nil { + w.observer.AddHandler(w.context, data) + } +} + +// FieldName 获取聚合运算字段名称 +func (w *TumblingWindow) FieldName() string { + return w.fieldName +} + +// LastData 获取最后一条数据 +func (w *TumblingWindow) LastData() (float64, bool) { + return w.queue.Back() +} + +// StartTime 获取窗口开始时间 +func (w *TumblingWindow) StartTime() time.Time { + return w.startTime +} + +// EndTime 获取窗口结束时间 +func (w *TumblingWindow) EndTime() time.Time { + return w.endTime +} + +// Archive 保存数据 +func (w *TumblingWindow) Archive() { + +} +func (w *TumblingWindow) Stop() { + + if w.ticker != nil { + w.ticker.Stop() + } + w.quit <- struct{}{} +} diff --git a/window/window.go b/window/window.go new file mode 100644 index 0000000..1972249 --- /dev/null +++ b/window/window.go @@ -0,0 +1,125 @@ +/* + * Copyright 2024 The RuleGo Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package window + +import ( + "github.com/rulego/streamsql/types" + queue2 "github.com/rulego/streamsql/utils/queue" + "sync" + "sync/atomic" + "time" +) + +type Window struct { + context types.SelectStreamSqlContext + observer types.WindowObserver + windowSize int32 // the size of the window + count int32 + lastTime time.Time // the last time the window was updated + fieldName string // the name of the field that is being aggregated + //maxCapacity 最大容量 + maxCapacity int + queue *queue2.Queue + startTime time.Time + endTime time.Time + //退出标志 + quit chan struct{} + locker sync.Mutex +} + +// 检查是否需要是下一个窗口 +func (w *Window) checkNextWindow() { + w.locker.Lock() + defer w.locker.Unlock() + if atomic.LoadInt32(&w.count) >= w.windowSize { + //结束当前窗口 + w.end() + //开始新的窗口 + w.start() + } +} + +// 开始窗口事件 +func (w *Window) start() { + w.startTime = time.Now() + atomic.StoreInt32(&w.count, 0) + w.queue.Reset() + + if w.observer.StartHandler != nil { + w.observer.StartHandler(w.context) + } +} + +// 结束窗口事件 +func (w *Window) end() { + w.endTime = time.Now() + if w.observer.EndHandler != nil { + w.observer.EndHandler(w.context, w.queue.PopAll()) + } +} + +// 队列满，触发full事件，并重置队列 +func (w *Window) full() { + if w.observer.ArchiveHandler != nil { + w.observer.ArchiveHandler(w.context, w.queue.PopAll()) + } + //重置队列 + w.queue.Reset() +} + +// Add 添加数据 +func (w *Window) Add(data float64) { + if atomic.LoadInt32(&w.count) >= w.windowSize { + w.checkNextWindow() + } + if w.queue.IsFull() { + w.full() + } + atomic.AddInt32(&w.count, 1) + _ = w.queue.Push(data) + if w.observer.AddHandler != nil { + w.observer.AddHandler(w.context, data) + } +} + +// FieldName 获取聚合运算字段名称 +func (w *Window) FieldName() string { + return w.fieldName +} + +// LastData 获取最后一条数据 +func (w *Window) LastData() (float64, bool) { + return w.queue.Back() +} + +// StartTime 获取窗口开始时间 +func (w *Window) StartTime() time.Time { + return w.startTime +} + +// EndTime 获取窗口结束时间 +func (w *Window) EndTime() time.Time { + return w.endTime +} + +// Archive 保存数据 +func (w *Window) Archive() { + +} +func (w *Window) Stop() { + w.quit <- struct{}{} +}