perf(rule-engine, eventsource): Implement more efficient event bitsets

internal/etw/consumer.go

@@ -74,22 +74,21 @@ func (c *Consumer) ProcessEvent(ev *etw.EventRecord) error {
 	if c.isClosing {
 		return nil
 	}
-	if event.IsCurrentProcDropped(ev.Header.ProcessID) {
+
+	if !c.config.EventSource.EventExists(ev.ID()) {
+		eventsUnknown.Add(1)
 		return nil
 	}
-	if c.config.EventSource.ExcludeEvent(ev.Header.ProviderID, ev.HookID()) {
-		eventsExcluded.Add(1)
+	if event.IsCurrentProcDropped(ev.Header.ProcessID) {
 		return nil
 	}
-
-	etype := event.NewFromEventRecord(ev)
-	if !etype.Exists() {
-		eventsUnknown.Add(1)
+	if c.config.EventSource.ExcludeEvent(ev.ID()) {
+		eventsExcluded.Add(1)
 		return nil
 	}
 
 	eventsProcessed.Add(1)
-	evt := event.New(c.sequencer.Get(), etype, ev)
+	evt := event.New(c.sequencer.Get(), ev)
 
 	// Dispatch each event to the processor chain.
 	// Processors may further augment the event with

internal/etw/source_test.go

@@ -123,6 +123,7 @@ func TestEventSourceStartTraces(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
+			tt.cfg.EventSource.Init()
 			evs := NewEventSource(psnap, hsnap, tt.cfg, nil)
 			require.NoError(t, evs.Open(tt.cfg))
 			defer evs.Close()
@@ -193,6 +194,7 @@ func TestEventSourceEnableFlagsDynamically(t *testing.T) {
 		Filters: &config.Filters{},
 	}
 
+	cfg.EventSource.Init()
 	evs := NewEventSource(psnap, hsnap, cfg, r)
 	require.NoError(t, evs.Open(cfg))
 	defer evs.Close()
@@ -277,6 +279,7 @@ func TestEventSourceEnableFlagsDynamicallyWithYaraEnabled(t *testing.T) {
 		},
 	}
 
+	cfg.EventSource.Init()
 	evs := NewEventSource(psnap, hsnap, cfg, r)
 	require.NoError(t, evs.Open(cfg))
 	defer evs.Close()
@@ -328,6 +331,7 @@ func TestEventSourceRundownEvents(t *testing.T) {
 		Filters:     &config.Filters{},
 	}
 
+	cfg.EventSource.Init()
 	evs := NewEventSource(psnap, hsnap, cfg, nil)
 
 	l := &MockListener{}
@@ -741,6 +745,7 @@ func TestEventSourceAllEvents(t *testing.T) {
 		StackEnrichment:      false,
 	}
 
+	evsConfig.Init()
 	cfg := &config.Config{EventSource: evsConfig, Filters: &config.Filters{}}
 	evs := NewEventSource(psnap, hsnap, cfg, nil)
 
@@ -1226,6 +1231,8 @@ func testCallstackEnrichment(t *testing.T, hsnap handle.Snapshotter, psnap ps.Sn
 		FlushTimer:           1,
 	}
 
+	evsConfig.Init()
+
 	cfg := &config.Config{
 		EventSource:              evsConfig,
 		Filters:                  &config.Filters{},

internal/etw/stackext_test.go

@@ -38,6 +38,9 @@ func TestStackExtensions(t *testing.T) {
 			FlushTimer:         time.Millisecond * 2300,
 		},
 	}
+
+	cfg.EventSource.Init()
+
 	exts := NewStackExtensions(cfg.EventSource)
 	assert.Len(t, exts.EventIds(), 0)
 

internal/etw/trace_test.go

@@ -36,6 +36,8 @@ func TestStartTrace(t *testing.T) {
 		},
 	}
 
+	cfg.EventSource.Init()
+
 	trace := NewKernelTrace(cfg)
 	require.NoError(t, trace.Start())
 	require.True(t, trace.IsStarted())

pkg/config/eventsource.go

@@ -23,7 +23,7 @@ package config
 
 import (
 	"github.com/rabbitstack/fibratus/pkg/event"
-	"golang.org/x/sys/windows"
+	"github.com/rabbitstack/fibratus/pkg/util/bitmask"
 	"runtime"
 	"time"
 
@@ -102,7 +102,8 @@ type EventSourceConfig struct {
 	// ExcludedImages are process image names that will be rejected if they generate a kernel event.
 	ExcludedImages []string `json:"blacklist.images" yaml:"blacklist.images"`
 
-	dropMasks event.EventsetMasks
+	dropMasks *bitmask.Bitmask
+	allMasks  *bitmask.Bitmask
 
 	excludedImages map[string]bool
 }
@@ -127,13 +128,21 @@ func (c *EventSourceConfig) initFromViper(v *viper.Viper) {
 	c.ExcludedEvents = v.GetStringSlice(excludedEvents)
 	c.ExcludedImages = v.GetStringSlice(excludedImages)
 
+	c.dropMasks = bitmask.New()
+	c.allMasks = bitmask.New()
+
 	c.excludedImages = make(map[string]bool)
 
 	for _, name := range c.ExcludedEvents {
 		if typ := event.NameToType(name); typ != event.UnknownType {
-			c.dropMasks.Set(typ)
+			c.dropMasks.Set(typ.ID())
 		}
 	}
+
+	for _, typ := range event.AllWithState() {
+		c.allMasks.Set(typ.ID())
+	}
+
 	for _, name := range c.ExcludedImages {
 		c.excludedImages[name] = true
 	}
@@ -142,35 +151,54 @@ func (c *EventSourceConfig) initFromViper(v *viper.Viper) {
 // Init is an exported method to allow initializing exclusion maps from external modules.
 func (c *EventSourceConfig) Init() {
 	c.excludedImages = make(map[string]bool)
+
+	if c.dropMasks == nil {
+		c.dropMasks = bitmask.New()
+	}
 	for _, name := range c.ExcludedEvents {
 		for _, typ := range event.NameToTypes(name) {
 			if typ != event.UnknownType {
-				c.dropMasks.Set(typ)
+				c.dropMasks.Set(typ.ID())
 			}
 		}
 	}
+
 	for _, name := range c.ExcludedImages {
 		c.excludedImages[name] = true
 	}
+
+	if c.allMasks == nil {
+		c.allMasks = bitmask.New()
+	}
+	for _, typ := range event.AllWithState() {
+		c.allMasks.Set(typ.ID())
+	}
 }
 
 // SetDropMask inserts the event mask in the bitset to
 // instruct the given event type should be dropped from
 // the event stream.
-func (c *EventSourceConfig) SetDropMask(Type event.Type) {
-	c.dropMasks.Set(Type)
+func (c *EventSourceConfig) SetDropMask(typ event.Type) {
+	c.dropMasks.Set(typ.ID())
 }
 
 // TestDropMask checks if the specified event type has
 // the drop mask in the bitset.
-func (c *EventSourceConfig) TestDropMask(Type event.Type) bool {
-	return c.dropMasks.Test(Type.GUID(), Type.HookID())
+func (c *EventSourceConfig) TestDropMask(typ event.Type) bool {
+	return c.dropMasks.IsSet(typ.ID())
+}
+
+// ExcludeEvent determines whether the supplied short
+// event ID exists in the bitset of excluded events.
+func (c *EventSourceConfig) ExcludeEvent(id uint) bool {
+	return c.dropMasks.IsSet(id)
 }
 
-// ExcludeEvent determines whether the supplied provider GUID
-// and the hook identifier are in the bitset of excluded events.
-func (c *EventSourceConfig) ExcludeEvent(guid windows.GUID, hookID uint16) bool {
-	return c.dropMasks.Test(guid, hookID)
+// EventExists determines if the provided event ID exists
+// in the internal event catalog by checking the event ID
+// bitmask.
+func (c *EventSourceConfig) EventExists(id uint) bool {
+	return c.allMasks.IsSet(id)
 }
 
 // ExcludeImage determines whether the process generating event is present in the

pkg/config/eventsource_test.go

@@ -55,8 +55,8 @@ func TestEventSourceConfig(t *testing.T) {
 	assert.False(t, c.EventSource.EnableImageEvents)
 	assert.False(t, c.EventSource.EnableFileIOEvents)
 
-	assert.True(t, c.EventSource.ExcludeEvent(event.CloseHandle.GUID(), event.CloseHandle.HookID()))
-	assert.False(t, c.EventSource.ExcludeEvent(event.CreateProcess.GUID(), event.CreateProcess.HookID()))
+	assert.True(t, c.EventSource.ExcludeEvent(event.CloseHandle.ID()))
+	assert.False(t, c.EventSource.ExcludeEvent(event.CreateProcess.ID()))
 
 	assert.True(t, c.EventSource.ExcludeImage(&pstypes.PS{Name: "svchost.exe"}))
 	assert.False(t, c.EventSource.ExcludeImage(&pstypes.PS{Name: "explorer.exe"}))

pkg/event/bitset.go

@@ -0,0 +1,94 @@
+/*
+ * Copyright 2021-2022 by Nedim Sabic Sabic
+ * https://www.fibratus.io
+ * All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package event
+
+import (
+	"github.com/bits-and-blooms/bitset"
+	"github.com/rabbitstack/fibratus/pkg/util/bitmask"
+)
+
+// BitSetType defines the bitset type
+type BitSetType uint8
+
+const (
+	// BitmaskBitSet designates the mask-based event id bitset
+	BitmaskBitSet BitSetType = iota + 1
+	// TypeBitSet designates the uint16 number space event type bitset
+	TypeBitSet
+	// CategoryBitSet designates the event category bitset
+	CategoryBitSet
+)
+
+// BitSets handles the group of category/event type bitsets
+// and the bitmask for evaluating event ids bits.
+type BitSets struct {
+	bitmask *bitmask.Bitmask
+	cats    *bitset.BitSet
+	types   *bitset.BitSet
+}
+
+// SetBit sets the bit dictated by the bitset type.
+func (b *BitSets) SetBit(bs BitSetType, typ Type) {
+	switch bs {
+	case BitmaskBitSet:
+		if b.bitmask == nil {
+			b.bitmask = bitmask.New()
+		}
+		b.bitmask.Set(typ.ID())
+
+	case TypeBitSet:
+		if b.types == nil {
+			b.types = bitset.New(uint(MaxTypeID() + 1))
+		}
+		b.types.Set(uint(typ.HookID()))
+
+	case CategoryBitSet:
+		if b.cats == nil {
+			b.cats = bitset.New(MaxCategoryIndex + 1)
+		}
+		b.cats.Set(uint(typ.Category().Index()))
+	}
+}
+
+// SetCategoryBit toggles the category bit in the bitset.
+func (b *BitSets) SetCategoryBit(c Category) {
+	if b.cats == nil {
+		b.cats = bitset.New(MaxCategoryIndex + 1)
+	}
+	b.cats.Set(uint(c.Index()))
+}
+
+// IsBitSet checks if any of the populated bitsets
+// contain the type, event ID, or category bit.
+// This method evaluates first the event type bitset.
+// The event type bitset should only be initialized
+// if all event types pertain to the same category.
+// Otherwise, event id bitset and last category bitset
+// are tested for respective bits.
+func (b *BitSets) IsBitSet(evt *Event) bool {
+	if b.types != nil && b.types.Test(uint(evt.Type.HookID())) {
+		return true
+	}
+	return (b.bitmask != nil && b.bitmask.IsSet(evt.Type.ID())) ||
+		(b.cats != nil && b.cats.Test(uint(evt.Category.Index())))
+}
+
+func (b *BitSets) IsBitmaskInitialized() bool  { return b.bitmask != nil }
+func (b *BitSets) IsTypesInitialized() bool    { return b.types != nil }
+func (b *BitSets) IsCategoryInitialized() bool { return b.cats != nil }